diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 9362e7a..2302c1a 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -3,9 +3,9 @@ run-name: dbt_run_dev_refresh on: workflow_dispatch: - schedule: - # Runs "at 9:00 UTC" (see https://crontab.guru) - - cron: '0 9 * * *' + # schedule: + # # Runs "at 9:00 UTC" (see https://crontab.guru) + # - cron: '0 9 * * *' env: USE_VARS: "${{ vars.USE_VARS }}" diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml deleted file mode 100644 index 1c5e21d..0000000 --- a/.github/workflows/dbt_run_incremental.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: dbt_run_scheduled -run-name: dbt_run_scheduled - -on: - workflow_dispatch: - schedule: - - cron: '0 1,7,13,19 * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run -s ./models - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit - diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml deleted file mode 100644 index 5b3254b..0000000 --- a/.github/workflows/dbt_run_streamline_history.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_streamline_history -run-name: dbt_run_streamline_history - -on: - workflow_dispatch: - schedule: - # Runs "every 6 hours" (see https://crontab.guru) - - cron: '0 1-23/6 * * *' - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/streamline/realtime \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml deleted file mode 100644 index 67414b1..0000000 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_streamline_realtime -run-name: dbt_run_streamline_realtime - -on: - workflow_dispatch: - schedule: - # Runs "every 1 hour at min 40" (see https://crontab.guru) - - cron: '40 */1 * * *' - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/realtime \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml deleted file mode 100644 index ec7ff03..0000000 --- a/.github/workflows/dbt_test.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_test_scheduled -run-name: dbt_test_scheduled - -on: - workflow_dispatch: - schedule: - - cron: '0 4 * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v1 - with: - python-version: "3.7.x" - - - name: install dependencies - run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click - dbt deps - - name: Run DBT Jobs - run: | - dbt test -m ./models diff --git a/README.md b/README.md index 4cef631..1528017 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ + +# Please find and replace all instances of `xyz` with your project name. + ## Profile Set Up #### Use the following within profiles.yml ---- ```yml -evmos: +xyz: target: dev outputs: dev: @@ -14,7 +17,7 @@ evmos: user: password: region: - database: evmos_DEV + database: xyz_DEV warehouse: schema: silver threads: 4 @@ -70,5 +73,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s ``` select * -from table(evmos.information_schema.tag_references('evmos.core.fact_blocks', 'table')); -``` +from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table')); +``` \ No newline at end of file diff --git a/analyses/sample_tendermint_rpc_calls.sql b/analyses/sample_tendermint_rpc_calls.sql deleted file mode 100644 index 8e36edc..0000000 --- a/analyses/sample_tendermint_rpc_calls.sql +++ /dev/null @@ -1,47 +0,0 @@ --- get chainhead -select ETHEREUM.STREAMLINE.UDF_API('GET','https://rpc-evmos.imperator.co/abci_info',{},{} ) - ---sample gen for last 1k blocks - create table sample_block_ids as ( - with gen as ( -select - row_number() over ( - order by - seq4() - ) as block_height -from - table(generator(rowcount => 100000000)) - ) - - - select top 10000 block_height from gen - -- except select 12872988 from sample_blocks - where block_height <= 12873408 - order by 1 desc) - - ---pull one block -select -block_height, -ETHEREUM.STREAMLINE.UDF_JSON_RPC_CALL('https://rpc-evmos.imperator.co/',{}, - [ - { 'id': block_height, 'jsonrpc': '2.0', 'method': 'block', 'params': [ block_height::STRING ] } - ] - ) data, - getdate() as _inserted_timestamp - from - (select 12889280 as block_height ) - - ---pull one block's transactions -select -block_height, -ETHEREUM.STREAMLINE.UDF_JSON_RPC_CALL('https://rpc-evmos.imperator.co/',{}, - [ - { 'id': block_height, 'jsonrpc': '2.0', 'method': 'tx_search', 'params': [ 'tx.height='||block_height::STRING , true, '1', '1000', 'asc' ] } - ] - ) data, - getdate() as _inserted_timestamp - from - (select 12889280 as block_height ) - \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 3b9ebd4..3a5b5f4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,14 +1,14 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "evmos_models" +name: "xyz_models" version: "1.0.0" config-version: 2 require-dbt-version: ">=1.4.0" # This setting configures which "profile" dbt uses for this project. -profile: "evmos" +profile: "xyz" # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be @@ -41,12 +41,7 @@ models: vars: "dbt_date:time_zone": GMT - STREAMLINE_INVOKE_STREAMS: False - STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False - UPDATE_UDFS_AND_SPS: False - STREAMLINE_RUN_HISTORY: False - UPDATE_SNOWFLAKE_TAGS: True - WAIT: 0 + "UPDATE_SNOWFLAKE_TAGS": TRUE tests: +store_failures: true # all tests diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 04ae357..8577716 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,5 +1,5 @@ {% macro create_sps() %} - {% if target.database == 'EVMOS' %} + {% if target.database == 'xyz' %} CREATE SCHEMA IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 1f1e9bb..56fe0ff 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,9 +1,2 @@ {% macro create_udfs() %} - {{ create_udtf_get_base_table( - schema = "streamline" - ) }} - {{ create_udf_get_chainhead() }} - {{ create_udf_json_rpc() }} - {{ create_udf_get_tendermint_transactions() }} - {{ create_udf_get_tendermint_validators() }} {% endmacro %} diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 4074835..1868683 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -1,6 +1,6 @@ {% macro run_sp_create_prod_clone() %} {% set clone_query %} -call evmos._internal.create_prod_clone('evmos', 'evmos_dev', 'internal_dev'); +call xyz._internal.create_prod_clone('xyz', 'xyz_dev', 'internal_dev'); {% endset %} {% do run_query(clone_query) %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql deleted file mode 100644 index 1c374b3..0000000 --- a/macros/streamline/api_integrations.sql +++ /dev/null @@ -1,11 +0,0 @@ -{% macro create_aws_ethereum_api() %} - {% if target.name == "prod" %} - {% set sql %} - CREATE api integration IF NOT EXISTS aws_evmos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/snowflake-api-evmos' api_allowed_prefixes = ( - 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/', - 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/' - ) enabled = TRUE; -{% endset %} - {% do run_query(sql) %} - {% endif %} -{% endmacro %} diff --git a/macros/streamline/get_base_table_udtf.sql b/macros/streamline/get_base_table_udtf.sql deleted file mode 100644 index a488d14..0000000 --- a/macros/streamline/get_base_table_udtf.sql +++ /dev/null @@ -1,24 +0,0 @@ -{% macro create_udtf_get_base_table(schema) %} -create or replace function {{ schema }}.udtf_get_base_table(max_height integer) -returns table (height number) -as -$$ - with base as ( - select - row_number() over ( - order by - seq4() - ) as id - from - table(generator(rowcount => 100000000)) - ) -select - id as height -from - base -where - id <= max_height -$$ -; - -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql deleted file mode 100644 index 02cf854..0000000 --- a/macros/streamline/models.sql +++ /dev/null @@ -1,171 +0,0 @@ - -{% macro decode_logs_history( - start, - stop - ) %} - WITH look_back AS ( - SELECT - block_number - FROM - {{ ref("_max_block_by_date") }} - qualify ROW_NUMBER() over ( - ORDER BY - block_number DESC - ) = 1 - ) -SELECT - l.block_number, - l._log_id, - abi.data AS abi, - l.data -FROM - {{ ref("streamline__decode_logs") }} - l - INNER JOIN {{ ref("silver__abis") }} - abi - ON l.abi_address = abi.contract_address -WHERE - ( - l.block_number BETWEEN {{ start }} - AND {{ stop }} - ) - AND l.block_number <= ( - SELECT - block_number - FROM - look_back - ) - AND _log_id NOT IN ( - SELECT - _log_id - FROM - {{ ref("streamline__complete_decode_logs") }} - WHERE - ( - block_number BETWEEN {{ start }} - AND {{ stop }} - ) - AND block_number <= ( - SELECT - block_number - FROM - look_back - ) - ) -{% endmacro %} - -{% macro streamline_external_table_query( - model, - partition_function, - partition_name, - unique_key - ) %} - WITH meta AS ( - SELECT - job_created_time AS _inserted_timestamp, - file_name, - {{ partition_function }} AS {{ partition_name }} - FROM - TABLE( - information_schema.external_table_file_registration_history( - start_time => DATEADD('day', -7, CURRENT_TIMESTAMP()), - table_name => '{{ source( "bronze_streamline", model) }}') - ) A - ) - SELECT - {{ unique_key }}, - DATA, - _inserted_timestamp, - MD5( - CAST( - COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text - ) - ) AS id, - s.{{ partition_name }}, - s.value AS VALUE - FROM - {{ source( - "bronze_streamline", - model - ) }} - s - JOIN meta b - ON b.file_name = metadata$filename - AND b.{{ partition_name }} = s.{{ partition_name }} - WHERE - b.{{ partition_name }} = s.{{ partition_name }} - AND ( - DATA :error :code IS NULL - OR DATA :error :code NOT IN ( - '-32000', - '-32001', - '-32002', - '-32003', - '-32004', - '-32005', - '-32006', - '-32007', - '-32008', - '-32009', - '-32010' - ) - ) -{% endmacro %} - -{% macro streamline_external_table_FR_query( - model, - partition_function, - partition_name, - unique_key - ) %} - WITH meta AS ( - SELECT - registered_on AS _inserted_timestamp, - file_name, - {{ partition_function }} AS {{ partition_name }} - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "bronze_streamline", model) }}' - ) - ) A - ) -SELECT - {{ unique_key }}, - DATA, - _inserted_timestamp, - MD5( - CAST( - COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text - ) - ) AS id, - s.{{ partition_name }}, - s.value AS VALUE -FROM - {{ source( - "bronze_streamline", - model - ) }} - s - JOIN meta b - ON b.file_name = metadata$filename - AND b.{{ partition_name }} = s.{{ partition_name }} -WHERE - b.{{ partition_name }} = s.{{ partition_name }} - AND ( - DATA :error :code IS NULL - OR DATA :error :code NOT IN ( - '-32000', - '-32001', - '-32002', - '-32003', - '-32004', - '-32005', - '-32006', - '-32007', - '-32008', - '-32009', - '-32010' - ) - ) -{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql deleted file mode 100644 index df13c22..0000000 --- a/macros/streamline/streamline_udfs.sql +++ /dev/null @@ -1,37 +0,0 @@ -{% macro create_udf_get_chainhead() %} - CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.udf_get_chainhead() returns variant api_integration = aws_evmos_api AS {% if target.name == "prod" %} - 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/get_chainhead' - {% else %} - 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/get_chainhead' - {%- endif %}; -{% endmacro %} - -{% macro create_udf_json_rpc() %} - CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.udf_json_rpc( - json OBJECT - ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} - 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_json_rpc' - {% else %} - 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/bulk_get_json_rpc' - {%- endif %}; -{% endmacro %} - -{% macro create_udf_get_tendermint_transactions() %} - CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.bulk_get_tendermint_transactions( - json OBJECT - ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} - 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_tendermint_transactions' - {% else %} - 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/bulk_get_tendermint_transactions' - {%- endif %}; -{% endmacro %} - -{% macro create_udf_get_tendermint_validators() %} - CREATE EXTERNAL FUNCTION IF NOT EXISTS streamline.bulk_get_tendermint_validators( - json OBJECT - ) returns ARRAY api_integration = aws_evmos_api AS {% if target.name == "prod" %} - 'https://n0reh6ugbf.execute-api.us-east-1.amazonaws.com/prod/bulk_get_tendermint_validators' - {% else %} - 'https://55h4rahr50.execute-api.us-east-1.amazonaws.com/dev/bulk_get_tendermint_validators' - {%- endif %}; -{% endmacro %} \ No newline at end of file diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql index cf8abf3..184e0bd 100644 --- a/macros/tags/add_database_or_schema_tags.sql +++ b/macros/tags/add_database_or_schema_tags.sql @@ -1,7 +1,7 @@ {% macro add_database_or_schema_tags() %} {{ set_database_tag_value( 'BLOCKCHAIN_NAME', - 'EVMOS' + 'xyz' ) }} {{ set_database_tag_value( 'BLOCKCHAIN_TYPE', diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql index 9425003..84a9aa9 100644 --- a/macros/tests/sequence_gaps.sql +++ b/macros/tests/sequence_gaps.sql @@ -1,34 +1,37 @@ -{% test sequence_gaps( - model, - partition_by, - column_name -) %} -{%- set partition_sql = partition_by | join(", ") -%} -{%- set previous_column = "prev_" ~ column_name -%} -WITH source AS ( - SELECT - {{ partition_sql + "," if partition_sql }} - {{ column_name }}, - LAG( - {{ column_name }}, - 1 - ) over ( - {{ "PARTITION BY " ~ partition_sql if partition_sql }} - ORDER BY - {{ column_name }} ASC - ) AS {{ previous_column }} - FROM - {{ model }} -) +{% macro sequence_gaps( + table, + partition_by, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + {{ table }} + WHERE + block_timestamp::date <= current_date - 1 + ) SELECT {{ partition_sql + "," if partition_sql }} {{ previous_column }}, - {{ column_name }}, - {{ column_name }} - {{ previous_column }} + {{ column }}, + {{ column }} - {{ previous_column }} - 1 AS gap FROM source WHERE - {{ column_name }} - {{ previous_column }} <> 1 + {{ column }} - {{ previous_column }} <> 1 ORDER BY - gap DESC {% endtest %} + gap DESC +{% endmacro %} diff --git a/macros/utils.sql b/macros/utils.sql deleted file mode 100644 index 0b800b8..0000000 --- a/macros/utils.sql +++ /dev/null @@ -1,78 +0,0 @@ -{% macro if_data_call_function( - func, - target - ) %} - {% if var( - "STREAMLINE_INVOKE_STREAMS" - ) %} - {% if execute %} - {{ log( - "Running macro `if_data_call_function`: Calling udf " ~ func ~ " on " ~ target, - True - ) }} - {% endif %} - SELECT - {{ func }} - WHERE - EXISTS( - SELECT - 1 - FROM - {{ target }} - LIMIT - 1 - ) - {% else %} - {% if execute %} - {{ log( - "Running macro `if_data_call_function`: NOOP", - False - ) }} - {% endif %} - SELECT - NULL - {% endif %} -{% endmacro %} - -{% macro if_data_call_wait() %} - {% if var( - "STREAMLINE_INVOKE_STREAMS" - ) %} - {% set query %} - SELECT - 1 - WHERE - EXISTS( - SELECT - 1 - FROM - {{ model.schema ~ "." ~ model.alias }} - LIMIT - 1 - ) {% endset %} - {% if execute %} - {% set results = run_query( - query - ) %} - {% if results %} - {{ log( - "Waiting...", - info = True - ) }} - - {% set wait_query %} - SELECT - system$wait( - {{ var( - "WAIT", - 600 - ) }} - ) {% endset %} - {% do run_query(wait_query) %} - {% else %} - SELECT - NULL; - {% endif %} - {% endif %} - {% endif %} -{% endmacro %} diff --git a/models/bronze/bronze__tendermint_blocks.sql b/models/bronze/bronze__tendermint_blocks.sql deleted file mode 100644 index 40ec94e..0000000 --- a/models/bronze/bronze__tendermint_blocks.sql +++ /dev/null @@ -1,60 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'block_id', - cluster_by = ['_inserted_timestamp::date'], - merge_update_columns = ["block_id"], -) }} - -WITH meta AS ( - - SELECT - registered_on, - last_modified, - LEAST( - last_modified, - registered_on - ) AS _inserted_timestamp, - file_name - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "streamline", "tendermint_blocks") }}' - ) - ) A - -{% if is_incremental() %} -WHERE - LEAST( - registered_on, - last_modified - ) >= ( - SELECT - COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP - FROM - {{ this }}) - ) -{% else %} -) -{% endif %} - -SELECT - value, - _partition_by_block_id, - block_number AS block_id, - DATA, - TO_TIMESTAMP( - m._inserted_timestamp - ) AS _inserted_timestamp -FROM - {{ source( - 'streamline', - 'tendermint_blocks' - ) }} -JOIN meta m -ON m.file_name = metadata$filename -WHERE - DATA: error IS NULL - -qualify(ROW_NUMBER() over (PARTITION BY block_number -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/bronze/bronze__tendermint_transactions.sql b/models/bronze/bronze__tendermint_transactions.sql deleted file mode 100644 index 9dc7827..0000000 --- a/models/bronze/bronze__tendermint_transactions.sql +++ /dev/null @@ -1,60 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = 'tx_id', - cluster_by = ['_inserted_timestamp::date'], - merge_update_columns = ["data", "_inserted_timestamp"], -) }} - -WITH meta AS ( - - SELECT - registered_on, - last_modified, - LEAST( - last_modified, - registered_on - ) AS _inserted_timestamp, - file_name - FROM - TABLE( - information_schema.external_table_files( - table_name => '{{ source( "streamline", "tendermint_transactions") }}' - ) - ) A - -{% if is_incremental() %} -WHERE - LEAST( - registered_on, - last_modified - ) >= ( - SELECT - COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP - FROM - {{ this }}) - ) -{% else %} -) -{% endif %} - -SELECT - value, - _partition_by_block_id, - block_number as block_id, - value :data :hash :: STRING AS tx_id, - metadata, - DATA, - TO_TIMESTAMP( - m._inserted_timestamp - ) AS _inserted_timestamp -FROM - {{ source( - 'streamline', - 'tendermint_transactions' - ) }} -JOIN meta m -ON m.file_name = metadata$filename -WHERE - DATA: error IS NULL qualify(ROW_NUMBER() over (PARTITION BY value :data :hash :: STRING -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql b/models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql deleted file mode 100644 index c190e9e..0000000 --- a/models/bronze/streamline/bronze__streamline_FR_eth_blocks.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -{{ streamline_external_table_FR_query( - "eth_blocks", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql b/models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql deleted file mode 100644 index 8c54a36..0000000 --- a/models/bronze/streamline/bronze__streamline_FR_eth_transactions.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -{{ streamline_external_table_FR_query( - "eth_transactions", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql b/models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql deleted file mode 100644 index c199622..0000000 --- a/models/bronze/streamline/bronze__streamline_FR_tendermint_blocks.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -{{ streamline_external_table_FR_query( - "tendermint_blocks", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql b/models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql deleted file mode 100644 index 85ea40f..0000000 --- a/models/bronze/streamline/bronze__streamline_FR_tendermint_transactions.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -{{ streamline_external_table_FR_query( - "tendermint_transactions", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql b/models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql deleted file mode 100644 index a186098..0000000 --- a/models/bronze/streamline/bronze__streamline_FR_tendermint_validators.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config ( - materialized = 'view' -) }} - -{{ streamline_external_table_FR_query( - "tendermint_validators", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_eth_blocks.sql b/models/bronze/streamline/bronze__streamline_eth_blocks.sql deleted file mode 100644 index 9de31ba..0000000 --- a/models/bronze/streamline/bronze__streamline_eth_blocks.sql +++ /dev/null @@ -1,12 +0,0 @@ - -{{ config ( - materialized = 'view' -) }} - -{% set model = this.identifier.split("_") [-1] %} -{{ streamline_external_table_query( - "eth_blocks", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_eth_transactions.sql b/models/bronze/streamline/bronze__streamline_eth_transactions.sql deleted file mode 100644 index 927fd0c..0000000 --- a/models/bronze/streamline/bronze__streamline_eth_transactions.sql +++ /dev/null @@ -1,12 +0,0 @@ - -{{ config ( - materialized = 'view' -) }} - -{% set model = this.identifier.split("_") [-1] %} -{{ streamline_external_table_query( - "eth_transactions", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_tendermint_blocks.sql b/models/bronze/streamline/bronze__streamline_tendermint_blocks.sql deleted file mode 100644 index e21f1f7..0000000 --- a/models/bronze/streamline/bronze__streamline_tendermint_blocks.sql +++ /dev/null @@ -1,12 +0,0 @@ - -{{ config ( - materialized = 'view' -) }} - -{% set model = this.identifier.split("_") [-1] %} -{{ streamline_external_table_query( - "tendermint_blocks", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_tendermint_transactions.sql b/models/bronze/streamline/bronze__streamline_tendermint_transactions.sql deleted file mode 100644 index 0a66bd1..0000000 --- a/models/bronze/streamline/bronze__streamline_tendermint_transactions.sql +++ /dev/null @@ -1,12 +0,0 @@ - -{{ config ( - materialized = 'view' -) }} - -{% set model = this.identifier.split("_") [-1] %} -{{ streamline_external_table_query( - "tendermint_transactions", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze/streamline/bronze__streamline_tendermint_validators.sql b/models/bronze/streamline/bronze__streamline_tendermint_validators.sql deleted file mode 100644 index b19ed78..0000000 --- a/models/bronze/streamline/bronze__streamline_tendermint_validators.sql +++ /dev/null @@ -1,12 +0,0 @@ - -{{ config ( - materialized = 'view' -) }} - -{% set model = this.identifier.split("_") [-1] %} -{{ streamline_external_table_query( - "tendermint_validators", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "block_number" -) }} \ No newline at end of file diff --git a/models/bronze_api/bronze_api__get_validator_metadata_lcd.sql b/models/bronze_api/bronze_api__get_validator_metadata_lcd.sql deleted file mode 100644 index 42cf493..0000000 --- a/models/bronze_api/bronze_api__get_validator_metadata_lcd.sql +++ /dev/null @@ -1,46 +0,0 @@ -{{ config( - materialized = 'table' -) }} - -WITH call AS ( - - SELECT - ethereum.streamline.udf_api( - 'GET', - 'https://lcd-evmos.keplr.app/cosmos/staking/v1beta1/validators?pagination.limit=5000',{},{} - ) AS resp, - SYSDATE() AS _inserted_timestamp -), -keep_last_if_failed AS ( - SELECT - i.value :operator_address :: STRING AS address, - i.value AS DATA, - _inserted_timestamp, - 2 AS RANK - FROM - call, - LATERAL FLATTEN( - input => resp :data :validators - ) i - UNION ALL - SELECT - address, - DATA, - _inserted_timestamp, - 1 AS RANK - FROM - bronze_api.get_validator_metadata_lcd -) -SELECT - address, - DATA, - _inserted_timestamp -FROM - keep_last_if_failed A - JOIN ( - SELECT - MAX(RANK) max_rank - FROM - keep_last_if_failed - ) b - ON A.rank = b.max_rank \ No newline at end of file diff --git a/models/core/core__dim_labels.sql b/models/core/core__dim_labels.sql deleted file mode 100644 index 6d0a065..0000000 --- a/models/core/core__dim_labels.sql +++ /dev/null @@ -1,49 +0,0 @@ -{{ config( - materialized = 'table' -) }} - -SELECT - blockchain, - creator, - address, - label_type, - label_subtype, - project_name AS label, - address_name AS address_name, - NULL AS raw_metadata -FROM - {{ source( - 'crosschain', - 'address_labels' - ) }} -WHERE - blockchain = 'evmos' - -UNION -SELECT - blockchain, - creator, - address, - label_type, - label_subtype, - project_name AS label, - label AS address_name, - raw_metadata -FROM - {{ ref('core__dim_tokens') }} -WHERE - blockchain = 'evmos' -UNION -SELECT - blockchain, - creator, - address, - label_type, - label_subtype, - project_name AS label, - label AS address_name, - NULL AS raw_metadata -FROM - {{ ref('core__fact_validators') }} -WHERE - blockchain = 'evmos' \ No newline at end of file diff --git a/models/core/core__dim_labels.yml b/models/core/core__dim_labels.yml deleted file mode 100644 index 7c3a2f2..0000000 --- a/models/core/core__dim_labels.yml +++ /dev/null @@ -1,33 +0,0 @@ -version: 2 -models: - - name: core__dim_labels - description: A hand curated table containing address names / labels for popular contracts, validators, tokens, etc. - columns: - - name: BLOCKCHAIN - description: The name of the blockchain - tests: - - not_null - - name: CREATOR - description: The name of the creator of the label - tests: - - not_null - - name: LABEL_TYPE - description: A high-level category describing the addresses main function or ownership - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['flotsam', 'nft', 'defi', 'dex', 'cex', 'dapp', 'token', 'operator', 'layer2', 'chadmin'] - - name: LABEL_SUBTYPE - description: A sub-category nested within label type providing further detail - tests: - - not_null - - name: LABEL - description: Name of the controlling entity of the address - tests: - - not_null - - name: ADDRESS - description: Address that the label is for. This is the field that should be used to join other tables with labels. - tests: - - not_null - - name: RAW_METADATA - description: A field available for tokens that contains decimal information \ No newline at end of file diff --git a/models/core/core__dim_tokens.sql b/models/core/core__dim_tokens.sql deleted file mode 100644 index 0fe04b0..0000000 --- a/models/core/core__dim_tokens.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config( - materialized = 'table' -) }} - -SELECT - 'evmos' AS blockchain, - address, - creator, - label_type, - label_subtype, - label, - project_name, - alias, - DECIMAL, - raw_metadata, - concat_ws( - '-', - address, - creator, - blockchain - ) AS unique_key -FROM - {{ source( - 'osmo', - 'asset_metadata' - ) }} diff --git a/models/core/core__dim_tokens.yml b/models/core/core__dim_tokens.yml deleted file mode 100644 index 272b770..0000000 --- a/models/core/core__dim_tokens.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 2 -models: - - name: core__dim_tokens - description: Contains token metadata for assets on the Evmos blockchain. This table was sourced from an API and may not contain every token. - columns: - - name: ADDRESS - description: "{{ doc('address') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCKCHAIN - description: "{{ doc('blockchain') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: CREATOR - description: "{{ doc('creator') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: LABEL_TYPE - description: "{{ doc('label_type') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: LABEL_SUBTYPE - description: "{{ doc('label_subtype') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: LABEL - description: "{{ doc('label') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: PROJECT_NAME - description: "{{ doc('project_name') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: ALIAS - description: A secondary address for the token, where available - tests: - - dbt_expectations.expect_column_to_exist - - name: DECIMAL - description: "{{ doc('decimal') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: RAW_METADATA - description: "{{ doc('raw_metadata') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: UNIQUE_KEY - description: The unique key of the table - tests: - - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_blocks.sql b/models/core/core__fact_blocks.sql deleted file mode 100644 index 9a6fc9c..0000000 --- a/models/core/core__fact_blocks.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'view' -) }} - -SELECT - block_id, - block_timestamp, - chain_id, - tx_count, - proposer_address, - validator_hash -FROM - {{ ref('silver__blocks') }} \ No newline at end of file diff --git a/models/core/core__fact_blocks.yml b/models/core/core__fact_blocks.yml deleted file mode 100644 index c05502c..0000000 --- a/models/core/core__fact_blocks.yml +++ /dev/null @@ -1,29 +0,0 @@ -version: 2 -models: - - name: core__fact_blocks - description: Records of all blocks that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: CHAIN_ID - description: "{{ doc('chain_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_COUNT - description: "{{ doc('tx_count') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: PROPOSER_ADDRESS - description: "{{ doc('proposer_address') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: VALIDATOR_HASH - description: "{{ doc('validator_hash') }}" - tests: - - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_msg_attributes.sql b/models/core/core__fact_msg_attributes.sql deleted file mode 100644 index 13e7743..0000000 --- a/models/core/core__fact_msg_attributes.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - materialized = 'view' -) }} - -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - CONCAT( - msg_group, - ':', - msg_sub_group - ) AS msg_group, - msg_index, - msg_type, - attribute_index, - attribute_key, - attribute_value -FROM - {{ ref('silver__msg_attributes') }} diff --git a/models/core/core__fact_msg_attributes.yml b/models/core/core__fact_msg_attributes.yml deleted file mode 100644 index c996d68..0000000 --- a/models/core/core__fact_msg_attributes.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: 2 -models: - - name: core__fact_msg_attributes - description: Records of all message attributes associated to messages that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG_GROUP - description: "{{ doc('msg_group') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG_INDEX - description: "{{ doc('msg_index') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG_TYPE - description: "{{ doc('msg_type') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: ATTRIBUTE_KEY - description: "They key from the key-value pair from the message attribute" - tests: - - dbt_expectations.expect_column_to_exist - - name: ATTRIBUTE_VALUE - description: "They value from the key-value pair from the message attribute" - tests: - - dbt_expectations.expect_column_to_exist - diff --git a/models/core/core__fact_msgs.sql b/models/core/core__fact_msgs.sql deleted file mode 100644 index 3908d01..0000000 --- a/models/core/core__fact_msgs.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - materialized = 'view' -) }} - -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - CONCAT( - msg_group, - ':', - msg_sub_group - ) AS msg_group, - msg_index, - msg_type, - msg -FROM - {{ ref('silver__msgs') }} diff --git a/models/core/core__fact_msgs.yml b/models/core/core__fact_msgs.yml deleted file mode 100644 index da1b8a4..0000000 --- a/models/core/core__fact_msgs.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: 2 -models: - - name: core__fact_msgs - description: Records of all message attributes associated to messages that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG_GROUP - description: "{{ doc('msg_group') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG_INDEX - description: "{{ doc('msg_index') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG_TYPE - description: "{{ doc('msg_type') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSG - description: "A block of json that contains the message attributes in base64 encoding." - tests: - - dbt_expectations.expect_column_to_exist - diff --git a/models/core/core__fact_transactions.sql b/models/core/core__fact_transactions.sql deleted file mode 100644 index 104068e..0000000 --- a/models/core/core__fact_transactions.sql +++ /dev/null @@ -1,74 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "tx_id", - incremental_strategy = 'merge', - cluster_by = ['block_timestamp::DATE'], -) }} - -WITH fee AS ( - - SELECT - tx_id, - attribute_value AS fee - FROM - {{ ref('silver__msg_attributes') }} - WHERE - attribute_key = 'fee' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 -{% endif %} -qualify(ROW_NUMBER() over(PARTITION BY tx_id -ORDER BY - msg_index)) = 1 -), -spender AS ( - SELECT - tx_id, - SPLIT_PART( - attribute_value, - '/', - 0 - ) AS tx_from - FROM - {{ ref('silver__msg_attributes') }} - WHERE - attribute_key = 'acc_seq' - OR (msg_type = 'transfer' - AND attribute_key = 'sender') - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= CURRENT_DATE -2 -{% endif %} - -qualify(ROW_NUMBER() over(PARTITION BY tx_id -ORDER BY - msg_index)) = 1 -) -SELECT - t.block_id, - t.block_timestamp, - t.tx_id, - s.tx_from, - tx_succeeded, - codespace, - COALESCE( - fee, - '0aevmos' - ) AS fee, - gas_used, - gas_wanted, - tx_code - msgs -FROM - {{ ref('silver__transactions') }} - t - LEFT OUTER JOIN fee f - ON t.tx_id = f.tx_id - LEFT OUTER JOIN spender s - ON t.tx_id = s.tx_id - -{% if is_incremental() %} -WHERE - _inserted_timestamp :: DATE >= CURRENT_DATE -2 -{% endif %} diff --git a/models/core/core__fact_transactions.yml b/models/core/core__fact_transactions.yml deleted file mode 100644 index 4461e0e..0000000 --- a/models/core/core__fact_transactions.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: 2 -models: - - name: core__fact_transactions - description: Records of all transactions that have occurred on Evmos. This schema is currently in lite mode, and contains data back to DATE. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TX_ID - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_FROM - description: "{{ doc('tx_from') }}" - tests: - - not_null: - where: tx_succeeded = 'TRUE' - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: CODESPACE - description: "{{ doc('codespace') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: FEE - description: "{{ doc('fee') }}" - tests: - - not_null - - name: GAS_USED - description: "{{ doc('gas_used') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: GAS_WANTED - description: "{{ doc('gas_wanted') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: MSGS - description: "The underlying json from the messages or events within the transactions" - tests: - - dbt_expectations.expect_column_to_exist diff --git a/models/core/core__fact_transfers.sql b/models/core/core__fact_transfers.sql deleted file mode 100644 index d7bc2dc..0000000 --- a/models/core/core__fact_transfers.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - materialized = 'view' -) }} - -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - transfer_type, - sender, - amount, - currency, - receiver -FROM - {{ ref('silver__transfers') }} \ No newline at end of file diff --git a/models/core/core__fact_transfers.yml b/models/core/core__fact_transfers.yml deleted file mode 100644 index a19e550..0000000 --- a/models/core/core__fact_transfers.yml +++ /dev/null @@ -1,41 +0,0 @@ -version: 2 -models: - - name: core__fact_transfers - description: Records of all transfers on Evmos. This schema is currently in lite mode and contains data back to DATE. - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: TRANSFER_TYPE - description: "{{ doc('transfer_type') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: SENDER - description: "{{ doc('sender') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: AMOUNT - description: "{{ doc('amount') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: CURRENCY - description: "{{ doc('currency') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: RECEIVER - description: "{{ doc('receiver') }}" - tests: - - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/core/core__fact_validators.sql b/models/core/core__fact_validators.sql deleted file mode 100644 index 559f7f5..0000000 --- a/models/core/core__fact_validators.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - materialized = 'view' -) }} - -SELECT - address, - blockchain, - creator, - label_type, - label_subtype, - label, - project_name, - delegator_shares, - jailed, - rate, - max_change_rate, - max_rate, - min_self_delegation, - RANK, - raw_metadata -FROM - {{ ref('silver__validators') }} diff --git a/models/core/core__fact_validators.yml b/models/core/core__fact_validators.yml deleted file mode 100644 index 8c65e3d..0000000 --- a/models/core/core__fact_validators.yml +++ /dev/null @@ -1,69 +0,0 @@ -version: 2 -models: - - name: core__fact_validators - description: Information about all active and inactive set validators on Evmos. - columns: - - name: ADDRESS - description: "{{ doc('address') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: BLOCKCHAIN - description: "{{ doc('blockchain') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: CREATOR - description: "{{ doc('creator') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: LABEL_TYPE - description: "{{ doc('label_type') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: LABEL_SUBTYPE - description: "{{ doc('label_subtype') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: LABEL - description: "{{ doc('label') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: PROJECT_NAME - description: "{{ doc('project_name') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: DELEGATOR_SHARES - description: The number of tokens staked to the validator. - tests: - - dbt_expectations.expect_column_to_exist - - name: JAILED - description: FALSE when a validator is not jailed, TRUE when a validator is jailed - tests: - - dbt_expectations.expect_column_to_exist - - name: RATE - description: The current commission rate the validator is charging stakers. - tests: - - dbt_expectations.expect_column_to_exist - - name: MAX_CHANGE_RATE - description: The maximum rate at which a validator can change their commission per day. - tests: - - dbt_expectations.expect_column_to_exist - - name: MAX_RATE - description: The maximum commission rate that the validator can charge - tests: - - dbt_expectations.expect_column_to_exist - - name: MIN_SELF_DELEGATION - description: The minimum number of Evmos tokens that the operator must be staking with their own validator - tests: - - dbt_expectations.expect_column_to_exist - - name: RANK - description: The rank of the validator in the validator set. Rank is determined by the number of Evmos tokens staked to the validator - tests: - - dbt_expectations.expect_column_to_exist - - name: RAW_METADATA - description: "{{ doc('raw_metadata') }}" - tests: - - dbt_expectations.expect_column_to_exist - - name: UNIQUE_KEY - description: The unique key of the table - tests: - - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index b3e273a..588ffbd 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -1,10 +1,10 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto EVMOS Models Documentation +# Welcome to the Flipside Crypto xyz Models Documentation ## **What does this documentation cover?** -The documentation included here details the design of the evmos - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/evmos-models/) +The documentation included here details the design of the xyz + tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/xyz-models/) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -17,7 +17,7 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (`evmos`.`CORE`.``) +### Core Tables (`xyz`.`CORE`.``) **Dimension Tables:** @@ -33,7 +33,8 @@ There is more information on how to use dbt docs in the last section of this doc ## **Data Model Overview** -The evmos models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +The xyz + models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -67,7 +68,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/evmos-models) +- [Github](https://github.com/FlipsideCrypto/xyz-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/models/descriptions/_unique_key.md b/models/descriptions/_unique_key.md deleted file mode 100644 index f602192..0000000 --- a/models/descriptions/_unique_key.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs _unique_key %} - -The unique key for the table. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address.md b/models/descriptions/address.md deleted file mode 100644 index 36215fe..0000000 --- a/models/descriptions/address.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs address %} - -Address unique to an individual wallet, validator, or token. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/amount.md b/models/descriptions/amount.md deleted file mode 100644 index 1c7f7a0..0000000 --- a/models/descriptions/amount.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs amount %} - -The amount that was used in the transaction message. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_id.md b/models/descriptions/block_id.md deleted file mode 100644 index 95c8389..0000000 --- a/models/descriptions/block_id.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs block_id %} - -The block height the block was recorded at. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md deleted file mode 100644 index 907f6a3..0000000 --- a/models/descriptions/block_timestamp.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs block_timestamp %} - -The date and time at which the block began. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/blockchain.md b/models/descriptions/blockchain.md deleted file mode 100644 index dc243a7..0000000 --- a/models/descriptions/blockchain.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs blockchain %} - -In this table, always Cosmos. Used to join to cross-chain tables. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/chain_id.md b/models/descriptions/chain_id.md deleted file mode 100644 index 0981eb6..0000000 --- a/models/descriptions/chain_id.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs chain_id %} - -The name and version of the blockchain. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/codespace.md b/models/descriptions/codespace.md deleted file mode 100644 index 7537e19..0000000 --- a/models/descriptions/codespace.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs codespace %} - -Namespace for the code. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/creator.md b/models/descriptions/creator.md deleted file mode 100644 index ff72c36..0000000 --- a/models/descriptions/creator.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs creator %} - -Name of the label creator - for now, this will always be "Flipside." - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/currency.md b/models/descriptions/currency.md deleted file mode 100644 index 93f0327..0000000 --- a/models/descriptions/currency.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs currency %} - -The currency that was used in the transaction message. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/decimal.md b/models/descriptions/decimal.md deleted file mode 100644 index eb9c154..0000000 --- a/models/descriptions/decimal.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs decimal %} - -Divide the amount by POW(10, decimal) to get the amount used in the transaction. This value can be NULL, as decimals are hand curated from an outside source and not found on-chain. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/fee.md b/models/descriptions/fee.md deleted file mode 100644 index 8247bd4..0000000 --- a/models/descriptions/fee.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs fee %} - -The fee is paid by the initiator of the transaction. Fee = gas * gas price and is given in micro-OSMO. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_used.md b/models/descriptions/gas_used.md deleted file mode 100644 index ae688ae..0000000 --- a/models/descriptions/gas_used.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs gas_used %} - -The amount of gas consumed by the transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_wanted.md b/models/descriptions/gas_wanted.md deleted file mode 100644 index 937d08a..0000000 --- a/models/descriptions/gas_wanted.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs gas_wanted %} - -Amount of gas requested for a transaction. It is provided by users when the transaction is generated. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md deleted file mode 100644 index 374d1f5..0000000 --- a/models/descriptions/inserted_timestamp.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs inserted_timestamp %} - -The date and time at which the block or transaction was inserted in the bronze tables. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label.md b/models/descriptions/label.md deleted file mode 100644 index 51af3dc..0000000 --- a/models/descriptions/label.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs label %} - -The label or name of the address. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label_subtype.md b/models/descriptions/label_subtype.md deleted file mode 100644 index 113de57..0000000 --- a/models/descriptions/label_subtype.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs label_subtype %} - -Adds more detail to the label type. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/label_type.md b/models/descriptions/label_type.md deleted file mode 100644 index c81472b..0000000 --- a/models/descriptions/label_type.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs label_type %} - -A broad category that describes what a label is representing. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_group.md b/models/descriptions/msg_group.md deleted file mode 100644 index ef4c5c1..0000000 --- a/models/descriptions/msg_group.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs msg_group %} - -Value grouping different messages together to represent a single action. Format will include the numeric msg_group and msg_sub_group with a ":" seperator. The subgroup will always be 0 except for "Exec" actions. NULL group means messages are related to the header (overall transaction) - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_index.md b/models/descriptions/msg_index.md deleted file mode 100644 index 8af9065..0000000 --- a/models/descriptions/msg_index.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs msg_index %} - -Short for "message index," the position in which messages occur in a transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_sub_group.md b/models/descriptions/msg_sub_group.md deleted file mode 100644 index 8b5a36c..0000000 --- a/models/descriptions/msg_sub_group.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs msg_sub_group %} - -Silver only -- Numeric value grouping different messages together to represent a single action within a group. This is relevent for exec actions that contain mutiple underlying actions. NULL sub group means messages are related to the header (overall transaction) - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/msg_type.md b/models/descriptions/msg_type.md deleted file mode 100644 index e16302b..0000000 --- a/models/descriptions/msg_type.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs msg_type %} - -A string containing information about the type of message occurring. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/project_name.md b/models/descriptions/project_name.md deleted file mode 100644 index a8a553d..0000000 --- a/models/descriptions/project_name.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs project_name %} - -The name of the project the label belongs to. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/proposer_address.md b/models/descriptions/proposer_address.md deleted file mode 100644 index 0dcaa94..0000000 --- a/models/descriptions/proposer_address.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs proposer_address %} - -The address of the validator that proposed the block. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/raw_metadata.md b/models/descriptions/raw_metadata.md deleted file mode 100644 index 539e992..0000000 --- a/models/descriptions/raw_metadata.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs raw_metadata %} - -Additional details about the validator or token in json format. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/receiver.md b/models/descriptions/receiver.md deleted file mode 100644 index e82f2da..0000000 --- a/models/descriptions/receiver.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs receiver %} - -The wallet address of the individual received tokens in the transfer. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sender.md b/models/descriptions/sender.md deleted file mode 100644 index 1b55aab..0000000 --- a/models/descriptions/sender.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs sender %} - -The wallet address of the individual sent tokens in the transfer. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/transfer_type.md b/models/descriptions/transfer_type.md deleted file mode 100644 index b8630ce..0000000 --- a/models/descriptions/transfer_type.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs transfer_type %} - -Details on the type of transfer occurring during the transaction. "Transfer_In" = depositing tokens onto the Cosmos Hub. "Transfer_out" = withdrawing tokens from the Cosmos Hub. "Cosmos" = wallet to wallet transfer on the Cosmos Hub. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_code.md b/models/descriptions/tx_code.md deleted file mode 100644 index 5200c7e..0000000 --- a/models/descriptions/tx_code.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs tx_code %} - -A number that corresponds to various error codes. When "0", the transaction is successful. Non-zero numbers signify different types of transaction failures. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_count.md b/models/descriptions/tx_count.md deleted file mode 100644 index 750dccb..0000000 --- a/models/descriptions/tx_count.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs tx_count %} - -The number of transactions that occurred during a block. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_from.md b/models/descriptions/tx_from.md deleted file mode 100644 index d9fd624..0000000 --- a/models/descriptions/tx_from.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs tx_from %} - -The wallet address of the individual who initiated the transaction - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_id.md b/models/descriptions/tx_id.md deleted file mode 100644 index 9232e98..0000000 --- a/models/descriptions/tx_id.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs tx_id %} - -A unique key that identifies a transaction. Called "TxHash" on block explorers. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_log.md b/models/descriptions/tx_log.md deleted file mode 100644 index f4024b9..0000000 --- a/models/descriptions/tx_log.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs tx_log %} - -A string that contains the transaction logs, which are logs written by the program interacted with during the transaction. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_succeeded.md b/models/descriptions/tx_succeeded.md deleted file mode 100644 index a429659..0000000 --- a/models/descriptions/tx_succeeded.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs tx_succeeded %} - -Transaction status is "TRUE" if the transaction went through, "FALSE" if the transaction failed. - -{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/validator_hash.md b/models/descriptions/validator_hash.md deleted file mode 100644 index 4536422..0000000 --- a/models/descriptions/validator_hash.md +++ /dev/null @@ -1,5 +0,0 @@ -{% docs validator_hash %} - -The root hash of the new validator set. - -{% enddocs %} \ No newline at end of file diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql deleted file mode 100644 index ef8856a..0000000 --- a/models/silver/silver__blocks.sql +++ /dev/null @@ -1,48 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', chain_id, block_id)", - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::DATE'], -) }} - -SELECT - block_id, - COALESCE( - DATA [0] :result :block :header :time :: TIMESTAMP, - DATA :result :block :header :time :: TIMESTAMP - ) AS block_timestamp, - COALESCE( - DATA [0] :result :block :header :chain_id :: STRING, - DATA :result :block :header :chain_id :: STRING - ) AS chain_id, - ARRAY_SIZE( - COALESCE( - data [0] :result :block :data :txs, - data :result :block :data :txs - ) - ) AS tx_count, - COALESCE( - data [0] :result :block :header :proposer_address :: STRING, - data :result :block :header :proposer_address :: STRING - ) AS proposer_address, - COALESCE( - DATA [0] :result :block :header :validators_hash :: STRING, - DATA :result :block :header :validators_hash :: STRING - ) AS validator_hash, - _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp -FROM - {{ ref('bronze__tendermint_blocks') }} -WHERE - data [0] :error IS NULL - AND DATA :error IS NULL - AND DATA :result :block :header :chain_id :: STRING IS NOT NULL - AND DATA[0] :result :block :header :chain_id :: STRING IS NOT NULL - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} diff --git a/models/silver/silver__blocks.yml b/models/silver/silver__blocks.yml deleted file mode 100644 index 532fafa..0000000 --- a/models/silver/silver__blocks.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 -models: - - name: silver__blocks - description: Records of all blocks that have occurred on Evmos, dating back to the genesis block. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - CHAIN_ID - - BLOCK_ID - - sequence_gaps: - column_name: block_ID - where: BLOCK_TIMESTAMP <= CURRENT_DATE - 1 - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - name: CHAIN_ID - description: "{{ doc('chain_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: TX_COUNT - description: "{{ doc('tx_count') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: PROPOSER_ADDRESS - description: "{{ doc('proposer_address') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: VALIDATOR_HASH - description: "{{ doc('validator_hash') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: _INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver/silver__msg_attributes.sql b/models/silver/silver__msg_attributes.sql deleted file mode 100644 index 4885a9b..0000000 --- a/models/silver/silver__msg_attributes.sql +++ /dev/null @@ -1,41 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', tx_id, msg_index, attribute_index)", - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", -) }} - -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - msg_group, - msg_sub_group, - msg_index, - msg_type, - b.index AS attribute_index, - TRY_BASE64_DECODE_STRING( - b.value :key :: STRING - ) AS attribute_key, - TRY_BASE64_DECODE_STRING( - b.value :value :: STRING - ) AS attribute_value, - _inserted_timestamp -FROM - {{ ref('silver__msgs') }} A, - LATERAL FLATTEN( - input => A.msg, - path => 'attributes' - ) b - -{% if is_incremental() %} -WHERE - _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} - ) -{% endif %} diff --git a/models/silver/silver__msg_attributes.yml b/models/silver/silver__msg_attributes.yml deleted file mode 100644 index 913079c..0000000 --- a/models/silver/silver__msg_attributes.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 -models: - - name: silver__msg_attributes - description: Records of all messages associated to transactions that have occurred on Evmos, dating back to the genesis block. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TX_ID - - MSG_INDEX - - ATTRIBUTE_INDEX - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: MSG_GROUP - description: "{{ doc('msg_group') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSG_SUB_GROUP - description: "{{ doc('msg_sub_group') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSG_INDEX - description: "{{ doc('msg_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSG_TYPE - description: "{{ doc('msg_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: MSG - description: "The underlying json from the message or event within the transactions" - \ No newline at end of file diff --git a/models/silver/silver__msgs.sql b/models/silver/silver__msgs.sql deleted file mode 100644 index 7023b86..0000000 --- a/models/silver/silver__msgs.sql +++ /dev/null @@ -1,134 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', tx_id, msg_index)", - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::DATE'], -) }} - -WITH base AS ( - - SELECT - t.block_id, - t.block_timestamp, - t.tx_id, - t.gas_used, - t.gas_wanted, - t.tx_succeeded, - f.value AS msg, - f.index :: INT AS msg_index, - msg :type :: STRING AS msg_type, - IFF( - TRY_BASE64_DECODE_STRING( - msg :attributes [0] :key :: STRING - ) = 'action', - TRUE, - FALSE - ) AS is_action, - NULLIF( - (conditional_true_event(is_action) over (PARTITION BY tx_id - ORDER BY - msg_index ASC) -1), - -1 - ) AS msg_group, - IFF( - TRY_BASE64_DECODE_STRING( - msg :attributes [0] :key :: STRING - ) = 'module', - TRUE, - FALSE - ) AS is_module, - TRY_BASE64_DECODE_STRING( - msg :attributes [0] :key :: STRING - ) AS attribute_key, - TRY_BASE64_DECODE_STRING( - msg :attributes [0] :value :: STRING - ) AS attribute_value, - t._inserted_timestamp - FROM - {{ ref('silver__transactions') }} t, - LATERAL FLATTEN(input => msgs) f - -{% if is_incremental() %} -WHERE - _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} - ) -{% endif %} -), -exec_actions AS ( - SELECT - DISTINCT tx_id, - msg_group - FROM - base - WHERE - msg_type = 'message' - AND attribute_key = 'action' - AND LOWER(attribute_value) LIKE '%exec%' -), -GROUPING AS ( - SELECT - base.tx_id, - base.msg_index, - RANK() over( - PARTITION BY base.tx_id, - base.msg_group - ORDER BY - base.msg_index - ) -1 AS msg_sub_group - FROM - base - INNER JOIN exec_actions e - ON base.tx_id = e.tx_id - AND base.msg_group = e.msg_group - WHERE - base.is_module = 'TRUE' - AND base.msg_type = 'message' -), -FINAL AS ( - SELECT - block_id, - block_timestamp, - A.tx_id, - tx_succeeded, - msg_group, - CASE - WHEN msg_group IS NULL THEN NULL - ELSE COALESCE( - LAST_VALUE( - b.msg_sub_group ignore nulls - ) over( - PARTITION BY A.tx_id, - msg_group - ORDER BY - A.msg_index DESC rows unbounded preceding - ), - 0 - ) - END AS msg_sub_group, - A.msg_index, - msg_type, - msg, - _inserted_timestamp - FROM - base A - LEFT JOIN GROUPING b - ON A.tx_id = b.tx_id - AND A.msg_index = b.msg_index -) -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - msg_group, - msg_sub_group, - msg_index, - msg_type, - msg, - _inserted_timestamp -FROM - FINAL diff --git a/models/silver/silver__msgs.yml b/models/silver/silver__msgs.yml deleted file mode 100644 index fc07ac5..0000000 --- a/models/silver/silver__msgs.yml +++ /dev/null @@ -1,76 +0,0 @@ -version: 2 -models: - - name: silver__msgs - description: Records of all messages associated to transactions that have occurred on Evmos, dating back to the genesis block. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TX_ID - - MSG_INDEX - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - name: MSG_GROUP - description: "{{ doc('msg_group') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSG_SUB_GROUP - description: "{{ doc('msg_sub_group') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSG_INDEX - description: "{{ doc('msg_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSG_TYPE - description: "{{ doc('msg_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: MSG - description: "The underlying json from the message or event within the transactions" - \ No newline at end of file diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql deleted file mode 100644 index e49a910..0000000 --- a/models/silver/silver__transactions.sql +++ /dev/null @@ -1,75 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "tx_id", - incremental_strategy = 'delete+insert', - cluster_by = 'block_timestamp::DATE', -) }} - -WITH base_transactions AS ( - - SELECT - block_id, - t.value :hash :: STRING AS tx_id, - t.value :tx_result :codespace AS codespace, - t.value :tx_result :gas_used :: NUMBER AS gas_used, - t.value :tx_result :gas_wanted :: NUMBER AS gas_wanted, - CASE - WHEN t.value :tx_result :code :: NUMBER = 0 THEN TRUE - ELSE FALSE - END AS tx_succeeded, - t.value :tx_result :code :: NUMBER AS tx_code, - t.value :tx_result :events AS msgs, - t.value :tx_result :log :: STRING AS tx_log, - _inserted_timestamp - FROM - {{ ref('bronze__tendermint_transactions') }}, - TABLE(FLATTEN(DATA :result :txs)) t - -{% if is_incremental() %} -WHERE - _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} - ) -{% endif %} -) -SELECT - t.block_id, - b.block_timestamp, - tx_id, - codespace, - gas_used, - gas_wanted, - tx_succeeded, - tx_code, - msgs, - tx_log, - t._inserted_timestamp, - concat_ws( - '-', - t.block_id, - tx_id - ) AS unique_key -FROM - base_transactions t - LEFT OUTER JOIN {{ ref('silver__blocks') }} - b - ON t.block_id = b.block_id - -{% if is_incremental() %} -WHERE - b._inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} - ) -{% endif %} - -qualify ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - t._inserted_timestamp DESC -) = 1 diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml deleted file mode 100644 index a792c9b..0000000 --- a/models/silver/silver__transactions.yml +++ /dev/null @@ -1,87 +0,0 @@ -version: 2 -models: - - name: silver__transactions - description: Records of all transactions that have occurred on Evmos, dating back to the genesis block. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TX_ID - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: CODESPACE - description: "{{ doc('codespace') }}" - - name: GAS_USED - description: "{{ doc('gas_used') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: GAS_WANTED - description: "{{ doc('gas_wanted') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - name: TX_CODE - description: "{{ doc('tx_code') }}" - tests: - - not_null: - where: TX_SUCCEEDED - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MSGS - description: "The underlying json from the messages or events within the transactions" - tests: - - not_null - - name: TX_TYPE - description: "The transaction type" - - name: TX_LOG - description: "{{ doc('tx_log') }}" - tests: - - not_null: - where: TX_SUCCEEDED - - name: _INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql deleted file mode 100644 index 24458ee..0000000 --- a/models/silver/silver__transfers.sql +++ /dev/null @@ -1,362 +0,0 @@ -{{ config( - materialized = 'incremental', - unique_key = "CONCAT_WS('-', tx_id, block_id, msg_index, currency)", - incremental_strategy = 'delete+insert', - cluster_by = 'block_timestamp::DATE', -) }} - -WITH evmos_txs AS ( - - SELECT - DISTINCT tx_id - FROM - {{ ref('silver__msg_attributes') }} - WHERE - attribute_value IN ( - '/cosmos.bank.v1beta1.MsgSend', - '/cosmos.bank.v1beta1.MsgMultiSend', - '/ibc.applications.transfer.v1.MsgTransfer' - ) - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -sender_index AS ( - SELECT - tx_id, - MIN(msg_index) AS msg_index - FROM - {{ ref('silver__msg_attributes') }} - WHERE - msg_type = 'tx' - AND attribute_key = 'acc_seq' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -GROUP BY - tx_id -), -sender AS ( - SELECT - m.block_id, - m.tx_id, - s.msg_index, - SPLIT_PART( - attribute_value, - '/', - 0 - ) AS sender - FROM - {{ ref('silver__msg_attributes') }} - m - INNER JOIN sender_index s - ON m.tx_id = s.tx_id - AND m.msg_index = s.msg_index - WHERE - msg_type = 'tx' - AND attribute_key = 'acc_seq' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -msg_index AS ( - SELECT - m.block_id, - v.tx_id, - attribute_key, - m.msg_index - FROM - evmos_txs v - LEFT OUTER JOIN {{ ref('silver__msg_attributes') }} - m - ON v.tx_id = m.tx_id - INNER JOIN sender s - ON v.tx_id = s.tx_id - AND m.block_id = s.block_id - WHERE - msg_type = 'transfer' - AND attribute_key = 'amount' - AND m.msg_index > s.msg_index - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -receiver AS ( - SELECT - m.block_id, - v.tx_id, - m.msg_index, - attribute_value AS receiver - FROM - evmos_txs v - LEFT OUTER JOIN {{ ref('silver__msg_attributes') }} - m - ON v.tx_id = m.tx_id - INNER JOIN sender s - ON v.tx_id = s.tx_id - AND m.block_id = s.block_id - WHERE - msg_type = 'ibc_transfer' - OR msg_type = 'transfer' - AND attribute_key = 'recipient' - AND m.msg_index > s.msg_index - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -amount AS ( - SELECT - m.block_id, - v.tx_id, - m.msg_index, - COALESCE( - SPLIT_PART( - TRIM( - REGEXP_REPLACE( - attribute_value, - '[^[:digit:]]', - ' ' - ) - ), - ' ', - 0 - ), - TRY_PARSE_JSON(attribute_value) :amount - ) AS amount, - COALESCE( - RIGHT(attribute_value, LENGTH(attribute_value) - LENGTH(SPLIT_PART(TRIM(REGEXP_REPLACE(attribute_value, '[^[:digit:]]', ' ')), ' ', 0))), - TRY_PARSE_JSON(attribute_value) [1] :denom - ) AS currency - FROM - evmos_txs v - LEFT OUTER JOIN {{ ref('silver__msg_attributes') }} - m - ON v.tx_id = m.tx_id - INNER JOIN sender s - ON v.tx_id = s.tx_id - AND m.block_id = s.block_id - WHERE - msg_type = 'transfer' - AND attribute_key = 'amount' - AND m.msg_index > s.msg_index - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -evmos_txs_final AS ( - SELECT - r.block_id, - block_timestamp, - r.tx_id, - tx_succeeded, - 'EVMOS' AS transfer_type, - r.msg_index, - sender, - amount, - currency, - receiver, - _inserted_timestamp - FROM - receiver r - LEFT OUTER JOIN amount C - ON r.tx_id = C.tx_id - AND r.block_id = C.block_id - AND r.msg_index = C.msg_index - LEFT OUTER JOIN sender s - ON r.tx_id = s.tx_id - AND r.block_id = s.block_id - LEFT OUTER JOIN {{ ref('silver__transactions') }} - t - ON r.tx_id = t.tx_id - AND r.block_id = t.block_id - WHERE - amount IS NOT NULL - AND sender IS NOT NULL - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -ibc_in_tx AS ( - SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - msg_index, - 'IBC_TRANSFER_IN' AS transfer_type, - TRY_PARSE_JSON(attribute_value) :sender :: STRING AS sender, - TRY_PARSE_JSON(attribute_value) :amount :: INT AS amount, - CASE - WHEN TRY_PARSE_JSON(attribute_value) :denom :: STRING LIKE '%/%' THEN SPLIT(TRY_PARSE_JSON(attribute_value) :denom :: STRING, '/') [array_size(split(try_parse_json(attribute_value):denom::string, '/')) - 1] - ELSE TRY_PARSE_JSON(attribute_value) :denom :: STRING - END AS currency, - TRY_PARSE_JSON(attribute_value) :receiver :: STRING AS receiver, - _inserted_timestamp - FROM - {{ ref('silver__msg_attributes') }} - WHERE - msg_type = 'write_acknowledgement' - AND attribute_key = 'packet_data' - AND TRY_PARSE_JSON(attribute_value): amount IS NOT NULL - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -ibc_out_txid AS ( - SELECT - tx_id - FROM - {{ ref('silver__msg_attributes') }} - WHERE - msg_type = 'ibc_transfer' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -ibc_out_tx AS ( - SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - msg_index, - 'IBC_TRANSFER_OUT' AS transfer_type, - TRY_PARSE_JSON(attribute_value) :sender :: STRING AS sender, - TRY_PARSE_JSON(attribute_value) :amount :: INT AS amount, - CASE - WHEN TRY_PARSE_JSON(attribute_value) :denom :: STRING LIKE '%/%' THEN SPLIT(TRY_PARSE_JSON(attribute_value) :denom :: STRING, '/') [array_size(split(try_parse_json(attribute_value):denom::string, '/')) - 1] - ELSE TRY_PARSE_JSON(attribute_value) :denom :: STRING - END AS currency, - TRY_PARSE_JSON(attribute_value) :receiver :: STRING AS receiver, - _inserted_timestamp - FROM - {{ ref('silver__msg_attributes') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - ibc_out_txid - ) - AND msg_type = 'send_packet' - AND attribute_key = 'packet_data' - -{% if is_incremental() %} -AND _inserted_timestamp :: DATE >= ( - SELECT - MAX(_inserted_timestamp) :: DATE - 2 - FROM - {{ this }} -) -{% endif %} -), -ibc_transfers_agg AS ( - SELECT - * - FROM - ibc_out_tx - UNION ALL - SELECT - * - FROM - ibc_in_tx -), -ibc_tx_final AS ( - SELECT - i.block_id, - i.block_timestamp, - i.tx_id, - i.tx_succeeded, - i.transfer_type, - i.sender, - i.amount, - i.currency, - i.receiver, - msg_index, - _inserted_timestamp - FROM - ibc_transfers_agg i -) -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - transfer_type, - sender, - amount, - currency, - receiver, - msg_index, - _inserted_timestamp -FROM - ibc_tx_final -UNION ALL -SELECT - block_id, - block_timestamp, - tx_id, - tx_succeeded, - transfer_type, - sender, - amount, - currency, - receiver, - msg_index, - _inserted_timestamp -FROM - evmos_txs_final diff --git a/models/silver/silver__transfers.yml b/models/silver/silver__transfers.yml deleted file mode 100644 index c2bf4a0..0000000 --- a/models/silver/silver__transfers.yml +++ /dev/null @@ -1,92 +0,0 @@ -version: 2 -models: - - name: silver__transfers - description: Records of all transfers on the Evmos, including IBC transfers as on- and off-ramps to Evmos and wallet to wallet transfers. This schema is in lite mode and contains data back to DATE. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - TX_ID - - BLOCK_ID - - MSG_INDEX - - CURRENCY - columns: - - name: BLOCK_ID - description: "{{ doc('block_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: BLOCK_TIMESTAMP - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - name: TX_ID - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - name: TRANSFER_TYPE - description: "{{ doc('transfer_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: SENDER - description: "{{ doc('sender') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: AMOUNT - description: "{{ doc('amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: CURRENCY - description: "{{ doc('currency') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: RECEIVER - description: "{{ doc('receiver') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: _INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/silver__validators.sql b/models/silver/silver__validators.sql deleted file mode 100644 index 242640d..0000000 --- a/models/silver/silver__validators.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config( - materialized = 'table' -) }} - -SELECT - address, - 'evmos' AS blockchain, - 'flipside' AS creator, - 'operator' AS label_type, - 'validator' AS label_subtype, - DATA :description :moniker :: STRING AS label, - DATA :description :identity :: STRING AS project_name, - DATA :delegator_shares :: INT AS delegator_shares, - DATA :jailed :: BOOLEAN AS jailed, - DATA :commission :commission_rates :rate :: FLOAT AS rate, - DATA :commission :commission_rates :max_change_rate :: FLOAT AS max_change_rate, - DATA :commission :commission_rates :max_rate :: FLOAT AS max_rate, - DATA :min_self_delegation :: INT AS min_self_delegation, - RANK() over ( - PARTITION BY address - ORDER BY - DATA :delegator_shares :: INT DESC - ) AS RANK, - DATA AS raw_metadata -FROM - {{ ref('bronze_api__get_validator_metadata_lcd') }} diff --git a/models/silver/silver__validators.yml b/models/silver/silver__validators.yml deleted file mode 100644 index 94b3bdb..0000000 --- a/models/silver/silver__validators.yml +++ /dev/null @@ -1,139 +0,0 @@ -version: 2 -models: - - name: silver__validators - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - ADDRESS - - CREATOR - - BLOCKCHAIN - columns: - - name: ADDRESS - description: "{{ doc('address') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: BLOCKCHAIN - description: "{{ doc('blockchain') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['evmos'] - - name: CREATOR - description: "{{ doc('creator') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: LABEL_TYPE - description: "{{ doc('label_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['operator'] - - name: LABEL_SUBTYPE - description: "{{ doc('label_subtype') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - dbt_expectations.expect_column_values_to_be_in_set: - value_set: ['validator'] - - name: LABEL - description: "{{ doc('label') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: PROJECT_NAME - description: "{{ doc('project_name') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - name: DELEGATOR_SHARES - description: The number of tokens staked to the validator. - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: JAILED - description: FALSE when a validator is not jailed, TRUE when a validator is jailed - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - name: RATE - description: The current commission rate the validator is charging stakers. - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MAX_CHANGE_RATE - description: The maximum rate at which a validator can change their commission per day. - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MAX_RATE - description: The maximum commission rate that the validator can charge - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: MIN_SELF_DELEGATION - description: The minimum number of Evmos tokens that the operator must be staking with their own validator - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: RANK - description: The rank of the validator in the validator set. Rank is determined by the number of Evmos tokens staked to the validator - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - name: RAW_METADATA - description: "{{ doc('raw_metadata') }}" - tests: - - not_null - - name: UNIQUE_KEY - description: The unique key of the table - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 04155ed..e31d5cc 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -2,38 +2,9 @@ version: 2 sources: - name: crosschain - database: "{{ 'crosschain' if target.database == 'evmos' else 'crosschain_dev' }}" + database: "{{ 'crosschain' if target.database == 'xyz' else 'crosschain_dev' }}" schema: core tables: - name: dim_date_hours - name: address_tags - - name: address_labels - name: dim_dates - - name: bronze_streamline - database: streamline - schema: | - {{ "EVMOS_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "EVMOS" }} - tables: - - name: eth_blocks - - name: eth_transactions - - name: tendermint_blocks - - name: tendermint_transactions - - name: tendermint_validators - - name: address_labels - - name: bronze - database: evmos - schema: bronze - tables: - - name: sample_blocks - - name: sample_txs - - name: streamline - database: streamline - schema: evmos - tables: - - name: tendermint_blocks - - name: tendermint_transactions - - name: osmo - database: osmosis - schema: silver - tables: - - name: asset_metadata diff --git a/models/streamline/complete/streamline__complete_eth_blocks.sql b/models/streamline/complete/streamline__complete_eth_blocks.sql deleted file mode 100644 index 8b25bf3..0000000 --- a/models/streamline/complete/streamline__complete_eth_blocks.sql +++ /dev/null @@ -1,30 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_eth_blocks') }} -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" -) }} - -SELECT - id, - block_number, - _inserted_timestamp -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_eth_blocks') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) -{% else %} - {{ ref('bronze__streamline_FR_eth_blocks') }} -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 diff --git a/models/streamline/complete/streamline__complete_eth_transactions.sql b/models/streamline/complete/streamline__complete_eth_transactions.sql deleted file mode 100644 index 95b14f4..0000000 --- a/models/streamline/complete/streamline__complete_eth_transactions.sql +++ /dev/null @@ -1,30 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_eth_transactions') }} -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" -) }} - -SELECT - id, - block_number, - _inserted_timestamp -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_eth_transactions') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) -{% else %} - {{ ref('bronze__streamline_FR_eth_transactions') }} -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/complete/streamline__complete_tendermint_blocks.sql b/models/streamline/complete/streamline__complete_tendermint_blocks.sql deleted file mode 100644 index f7cc942..0000000 --- a/models/streamline/complete/streamline__complete_tendermint_blocks.sql +++ /dev/null @@ -1,30 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_tendermint_blocks') }} -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" -) }} - -SELECT - id, - block_number, - _inserted_timestamp -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_tendermint_blocks') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) -{% else %} - {{ ref('bronze__streamline_FR_tendermint_blocks') }} -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/complete/streamline__complete_tendermint_transactions.sql b/models/streamline/complete/streamline__complete_tendermint_transactions.sql deleted file mode 100644 index 02d4bde..0000000 --- a/models/streamline/complete/streamline__complete_tendermint_transactions.sql +++ /dev/null @@ -1,30 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_tendermint_transactions') }} -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" -) }} - -SELECT - id, - block_number, - _inserted_timestamp -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_tendermint_transactions') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) -{% else %} - {{ ref('bronze__streamline_FR_tendermint_transactions') }} -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/complete/streamline__complete_tendermint_validators.sql b/models/streamline/complete/streamline__complete_tendermint_validators.sql deleted file mode 100644 index b0697ba..0000000 --- a/models/streamline/complete/streamline__complete_tendermint_validators.sql +++ /dev/null @@ -1,30 +0,0 @@ --- depends_on: {{ ref('bronze__streamline_tendermint_validators') }} -{{ config ( - materialized = "incremental", - unique_key = "id", - cluster_by = "ROUND(block_number, -3)", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)" -) }} - -SELECT - id, - block_number, - _inserted_timestamp -FROM - -{% if is_incremental() %} -{{ ref('bronze__streamline_tendermint_validators') }} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) _inserted_timestamp - FROM - {{ this }} - ) -{% else %} - {{ ref('bronze__streamline_FR_tendermint_validators') }} -{% endif %} - -qualify(ROW_NUMBER() over (PARTITION BY id -ORDER BY - _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/streamline/realtime/streamline__eth_blocks_realtime.sql b/models/streamline/realtime/streamline__eth_blocks_realtime.sql deleted file mode 100644 index 6506ca6..0000000 --- a/models/streamline/realtime/streamline__eth_blocks_realtime.sql +++ /dev/null @@ -1,62 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','eth_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'eth_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 2000000, 'worker_batch_size',100))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} - - SELECT - 0 AS block_number - {% else %} - SELECT - MAX(block_number) - 100000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number, - block_number_hex - FROM - {{ ref("streamline__blocks") }} - WHERE - ( - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) - AND block_number IS NOT NULL - EXCEPT - SELECT - block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex - FROM - {{ ref("streamline__complete_eth_blocks") }} - WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) -SELECT - block_number, - 'eth_getBlockByNumber' AS method, - CONCAT( - block_number_hex, - '_-_', - 'false' - ) AS params -FROM - tbl diff --git a/models/streamline/realtime/streamline__eth_transactions_realtime.sql b/models/streamline/realtime/streamline__eth_transactions_realtime.sql deleted file mode 100644 index 72ff6ed..0000000 --- a/models/streamline/realtime/streamline__eth_transactions_realtime.sql +++ /dev/null @@ -1,62 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'eth_transactions', 'exploded_key','[\"result\", \"transactions\"]', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 2000000, 'worker_batch_size',100))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} - - SELECT - 0 AS block_number - {% else %} - SELECT - MAX(block_number) - 100000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number, - block_number_hex - FROM - {{ ref("streamline__blocks") }} - WHERE - ( - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) - AND block_number IS NOT NULL - EXCEPT - SELECT - block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex - FROM - {{ ref("streamline__complete_eth_transactions") }} - WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) -SELECT - block_number, - 'eth_getBlockByNumber' AS method, - CONCAT( - block_number_hex, - '_-_', - 'true' - ) AS params -FROM - tbl diff --git a/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql b/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql deleted file mode 100644 index 00d96f1..0000000 --- a/models/streamline/realtime/streamline__tendermint_blocks_realtime.sql +++ /dev/null @@ -1,52 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_json_rpc(object_construct('url_route','tendermint_rpc', 'sql_source', '{{this.identifier}}', 'external_table', 'tendermint_blocks', 'method', 'eth_getBlockByNumber', 'producer_batch_size',10000, 'producer_limit_size', 2000000, 'worker_batch_size',100, 'producer_batch_chunks_size', 1000))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} - - SELECT - 0 AS block_number - {% else %} - SELECT - MAX(block_number) - 100000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number - FROM - {{ ref("streamline__blocks") }} - WHERE - ( - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) - AND block_number IS NOT NULL - EXCEPT - SELECT - block_number - FROM - {{ ref("streamline__complete_tendermint_blocks") }} - WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) -SELECT - block_number, - 'block' AS method, - block_number :: STRING AS params -FROM - tbl diff --git a/models/streamline/realtime/streamline__tendermint_transactions_realtime.sql b/models/streamline/realtime/streamline__tendermint_transactions_realtime.sql deleted file mode 100644 index 174b520..0000000 --- a/models/streamline/realtime/streamline__tendermint_transactions_realtime.sql +++ /dev/null @@ -1,50 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.bulk_get_tendermint_transactions(object_construct('sql_source', '{{this.identifier}}'))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} - - SELECT - 0 AS block_number - {% else %} - SELECT - MAX(block_number) - 100000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number - FROM - {{ ref("streamline__blocks") }} - WHERE - ( - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) - AND block_number IS NOT NULL - EXCEPT - SELECT - block_number - FROM - {{ ref("streamline__complete_tendermint_transactions") }} - WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) -SELECT - block_number -FROM - tbl diff --git a/models/streamline/realtime/streamline__tendermint_validators_realtime.sql b/models/streamline/realtime/streamline__tendermint_validators_realtime.sql deleted file mode 100644 index 9f53896..0000000 --- a/models/streamline/realtime/streamline__tendermint_validators_realtime.sql +++ /dev/null @@ -1,50 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.bulk_get_tendermint_validators(object_construct('sql_source', '{{this.identifier}}'))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH last_3_days AS ({% if var('STREAMLINE_RUN_HISTORY') %} - - SELECT - 0 AS block_number - {% else %} - SELECT - MAX(block_number) - 100000 AS block_number --aprox 3 days - FROM - {{ ref("streamline__blocks") }} - {% endif %}), - tbl AS ( - SELECT - block_number - FROM - {{ ref("streamline__blocks") }} - WHERE - ( - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) - AND block_number IS NOT NULL - EXCEPT - SELECT - block_number - FROM - {{ ref("streamline__complete_tendermint_validators") }} - WHERE - block_number >= ( - SELECT - block_number - FROM - last_3_days - ) - ) -SELECT - block_number -FROM - tbl diff --git a/models/streamline/streamline__blocks.sql b/models/streamline/streamline__blocks.sql deleted file mode 100644 index d43cee8..0000000 --- a/models/streamline/streamline__blocks.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config ( - materialized = "view", - tags = ['streamline_view'] -) }} - -{% if execute %} - {% set height = run_query('SELECT streamline.udf_get_chainhead()') %} - {% set block_height = height.columns [0].values() [0] %} -{% else %} - {% set block_height = 0 %} -{% endif %} - -SELECT - height AS block_number, - REPLACE( - concat_ws('', '0x', to_char(block_number, 'XXXXXXXX')), - ' ', - '' - ) AS block_number_hex -FROM - TABLE(streamline.udtf_get_base_table({{ block_height }})) diff --git a/packages.yml b/packages.yml index 58d1209..9159ebb 100644 --- a/packages.yml +++ b/packages.yml @@ -1,7 +1,3 @@ packages: - package: calogica/dbt_expectations - version: [">=0.8.0", "<0.9.0"] - - package: dbt-labs/dbt_external_tables - version: [">=0.8.0", "<0.9.0"] - - package: dbt-labs/dbt_utils - version: [">=1.0.0", "<1.1.0"] + version: [">=0.4.0", "<0.9.0"] diff --git a/profiles.yml b/profiles.yml index 6ff3582..0661f98 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,4 +1,4 @@ -evmos: +xyz: target: dev outputs: dev: