From 7eb5832b2b334cefd3cc7b32faa4ecec70428665 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Wed, 22 Jun 2022 17:04:36 -0400 Subject: [PATCH] Initial/setup (#1) * initial commit * inserted at * overview updates * labels and docs * docs --- .gitignore | 16 ++ README.md | 46 +++- analysis/.gitkeep | 0 data/.gitkeep | 0 dbt_project.yml | 39 ++++ macros/.gitkeep | 0 macros/create_sps.sql | 6 + macros/create_udfs.sql | 11 + macros/custom_naming_macros.sql | 11 + macros/js_hextoint.sql | 6 + macros/python/udfs.sql | 21 ++ macros/run_sp_create_prod_clone.sql | 10 + macros/sp_create_prod_clone.sql | 76 ++++++ macros/tests/sequence_gaps.sql | 34 +++ macros/tests/tx_gaps.sql | 33 +++ models/bronze/bronze__blocks.sql | 23 ++ models/bronze/bronze__transactions.sql | 21 ++ .../blocks/avax_block_header_json.md | 5 + .../blocks/avax_blockchain.md | 5 + .../blocks/avax_blocks_hash.md | 5 + .../blocks/avax_blocks_nonce.md | 5 + .../blocks/avax_blocks_table_doc.md | 5 + .../blocks/avax_difficulty.md | 5 + .../blocks/avax_extra_data.md | 5 + .../doc_descriptions/blocks/avax_gas_limit.md | 5 + .../doc_descriptions/blocks/avax_gas_used.md | 5 + models/doc_descriptions/blocks/avax_miner.md | 5 + .../doc_descriptions/blocks/avax_network.md | 5 + .../blocks/avax_parent_hash.md | 5 + .../blocks/avax_receipts_root.md | 5 + .../blocks/avax_sha3_uncles.md | 5 + models/doc_descriptions/blocks/avax_size.md | 5 + .../blocks/avax_total_difficulty.md | 5 + .../doc_descriptions/blocks/avax_tx_count.md | 5 + .../blocks/avax_uncle_blocks.md | 5 + .../event_logs/avax_event_index.md | 5 + .../event_logs/avax_event_inputs.md | 5 + .../event_logs/avax_event_name.md | 5 + .../event_logs/avax_event_removed.md | 5 + .../event_logs/avax_event_sig.md | 5 + .../event_logs/avax_log_id_events.md | 5 + .../event_logs/avax_logs_contract_address.md | 5 + .../event_logs/avax_logs_contract_name.md | 5 + .../event_logs/avax_logs_data.md | 5 + .../event_logs/avax_logs_table_doc.md | 5 + .../event_logs/avax_logs_tx_hash.md | 5 + .../event_logs/avax_origin_from.md | 5 + .../event_logs/avax_origin_to.md | 5 + .../event_logs/avax_topics.md | 5 + .../doc_descriptions/general/__overview__.md | 79 +++++++ .../general/avax_block_number.md | 5 + .../general/avax_block_timestamp.md | 5 + .../doc_descriptions/general/avax_decimals.md | 5 + .../general/avax_from_address.md | 5 + .../general/avax_ingested_at.md | 5 + .../general/avax_to_address.md | 5 + models/doc_descriptions/labels/avax_label.md | 5 + .../labels/avax_label_address.md | 5 + .../labels/avax_label_address_name.md | 5 + .../labels/avax_label_blockchain.md | 5 + .../labels/avax_label_creator.md | 5 + .../labels/avax_label_subtype.md | 5 + .../labels/avax_label_type.md | 5 + .../labels/avax_labels_table_doc.md | 5 + .../traces/avax_traces_block_no.md | 5 + .../traces/avax_traces_blocktime.md | 5 + .../traces/avax_traces_call_data.md | 5 + .../traces/avax_traces_from.md | 5 + .../traces/avax_traces_gas.md | 5 + .../traces/avax_traces_gas_used.md | 5 + .../traces/avax_traces_identifier.md | 5 + .../traces/avax_traces_input.md | 5 + .../traces/avax_traces_output.md | 5 + .../traces/avax_traces_sub.md | 5 + .../traces/avax_traces_table_doc.md | 5 + .../doc_descriptions/traces/avax_traces_to.md | 5 + .../traces/avax_traces_tx_hash.md | 5 + .../traces/avax_traces_type.md | 5 + .../traces/avax_traces_value.md | 5 + .../transactions/avax_cumulative_gas_used.md | 5 + .../transactions/avax_tx_block_hash.md | 5 + .../transactions/avax_tx_fee.md | 5 + .../transactions/avax_tx_gas_limit.md | 5 + .../transactions/avax_tx_gas_price.md | 5 + .../transactions/avax_tx_gas_used.md | 5 + .../transactions/avax_tx_hash.md | 5 + .../transactions/avax_tx_input_data.md | 5 + .../transactions/avax_tx_json.md | 5 + .../transactions/avax_tx_nonce.md | 5 + .../transactions/avax_tx_origin_sig.md | 5 + .../transactions/avax_tx_position.md | 5 + .../transactions/avax_tx_status.md | 5 + .../transactions/avax_tx_table_doc.md | 5 + .../transactions/avax_value.md | 5 + models/gold/core__dim_labels.sql | 22 ++ models/gold/core__dim_labels.yml | 57 +++++ models/gold/core__fact_blocks.sql | 26 +++ models/gold/core__fact_blocks.yml | 40 ++++ models/gold/core__fact_event_logs.sql | 25 ++ models/gold/core__fact_event_logs.yml | 38 +++ models/gold/core__fact_traces.sql | 24 ++ models/gold/core__fact_traces.yml | 40 ++++ models/gold/core__fact_transactions.sql | 27 +++ models/gold/core__fact_transactions.yml | 42 ++++ models/silver/silver__blocks.sql | 77 ++++++ models/silver/silver__blocks.yml | 123 ++++++++++ models/silver/silver__logs.sql | 111 +++++++++ models/silver/silver__logs.yml | 81 +++++++ models/silver/silver__traces.sql | 221 ++++++++++++++++++ models/silver/silver__traces.yml | 54 +++++ models/silver/silver__transactions.sql | 110 +++++++++ models/silver/silver__transactions.yml | 124 ++++++++++ models/sources.yml | 15 ++ snapshots/.gitkeep | 0 tests/.gitkeep | 0 .../optimism/silver__transactions__tx-gap.sql | 2 + 116 files changed, 2070 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100644 analysis/.gitkeep create mode 100644 data/.gitkeep create mode 100644 dbt_project.yml create mode 100644 macros/.gitkeep create mode 100644 macros/create_sps.sql create mode 100644 macros/create_udfs.sql create mode 100644 macros/custom_naming_macros.sql create mode 100644 macros/js_hextoint.sql create mode 100644 macros/python/udfs.sql create mode 100644 macros/run_sp_create_prod_clone.sql create mode 100644 macros/sp_create_prod_clone.sql create mode 100644 macros/tests/sequence_gaps.sql create mode 100644 macros/tests/tx_gaps.sql create mode 100644 models/bronze/bronze__blocks.sql create mode 100644 models/bronze/bronze__transactions.sql create mode 100644 models/doc_descriptions/blocks/avax_block_header_json.md create mode 100644 models/doc_descriptions/blocks/avax_blockchain.md create mode 100644 models/doc_descriptions/blocks/avax_blocks_hash.md create mode 100644 models/doc_descriptions/blocks/avax_blocks_nonce.md create mode 100644 models/doc_descriptions/blocks/avax_blocks_table_doc.md create mode 100644 models/doc_descriptions/blocks/avax_difficulty.md create mode 100644 models/doc_descriptions/blocks/avax_extra_data.md create mode 100644 models/doc_descriptions/blocks/avax_gas_limit.md create mode 100644 models/doc_descriptions/blocks/avax_gas_used.md create mode 100644 models/doc_descriptions/blocks/avax_miner.md create mode 100644 models/doc_descriptions/blocks/avax_network.md create mode 100644 models/doc_descriptions/blocks/avax_parent_hash.md create mode 100644 models/doc_descriptions/blocks/avax_receipts_root.md create mode 100644 models/doc_descriptions/blocks/avax_sha3_uncles.md create mode 100644 models/doc_descriptions/blocks/avax_size.md create mode 100644 models/doc_descriptions/blocks/avax_total_difficulty.md create mode 100644 models/doc_descriptions/blocks/avax_tx_count.md create mode 100644 models/doc_descriptions/blocks/avax_uncle_blocks.md create mode 100644 models/doc_descriptions/event_logs/avax_event_index.md create mode 100644 models/doc_descriptions/event_logs/avax_event_inputs.md create mode 100644 models/doc_descriptions/event_logs/avax_event_name.md create mode 100644 models/doc_descriptions/event_logs/avax_event_removed.md create mode 100644 models/doc_descriptions/event_logs/avax_event_sig.md create mode 100644 models/doc_descriptions/event_logs/avax_log_id_events.md create mode 100644 models/doc_descriptions/event_logs/avax_logs_contract_address.md create mode 100644 models/doc_descriptions/event_logs/avax_logs_contract_name.md create mode 100644 models/doc_descriptions/event_logs/avax_logs_data.md create mode 100644 models/doc_descriptions/event_logs/avax_logs_table_doc.md create mode 100644 models/doc_descriptions/event_logs/avax_logs_tx_hash.md create mode 100644 models/doc_descriptions/event_logs/avax_origin_from.md create mode 100644 models/doc_descriptions/event_logs/avax_origin_to.md create mode 100644 models/doc_descriptions/event_logs/avax_topics.md create mode 100644 models/doc_descriptions/general/__overview__.md create mode 100644 models/doc_descriptions/general/avax_block_number.md create mode 100644 models/doc_descriptions/general/avax_block_timestamp.md create mode 100644 models/doc_descriptions/general/avax_decimals.md create mode 100644 models/doc_descriptions/general/avax_from_address.md create mode 100644 models/doc_descriptions/general/avax_ingested_at.md create mode 100644 models/doc_descriptions/general/avax_to_address.md create mode 100644 models/doc_descriptions/labels/avax_label.md create mode 100644 models/doc_descriptions/labels/avax_label_address.md create mode 100644 models/doc_descriptions/labels/avax_label_address_name.md create mode 100644 models/doc_descriptions/labels/avax_label_blockchain.md create mode 100644 models/doc_descriptions/labels/avax_label_creator.md create mode 100644 models/doc_descriptions/labels/avax_label_subtype.md create mode 100644 models/doc_descriptions/labels/avax_label_type.md create mode 100644 models/doc_descriptions/labels/avax_labels_table_doc.md create mode 100644 models/doc_descriptions/traces/avax_traces_block_no.md create mode 100644 models/doc_descriptions/traces/avax_traces_blocktime.md create mode 100644 models/doc_descriptions/traces/avax_traces_call_data.md create mode 100644 models/doc_descriptions/traces/avax_traces_from.md create mode 100644 models/doc_descriptions/traces/avax_traces_gas.md create mode 100644 models/doc_descriptions/traces/avax_traces_gas_used.md create mode 100644 models/doc_descriptions/traces/avax_traces_identifier.md create mode 100644 models/doc_descriptions/traces/avax_traces_input.md create mode 100644 models/doc_descriptions/traces/avax_traces_output.md create mode 100644 models/doc_descriptions/traces/avax_traces_sub.md create mode 100644 models/doc_descriptions/traces/avax_traces_table_doc.md create mode 100644 models/doc_descriptions/traces/avax_traces_to.md create mode 100644 models/doc_descriptions/traces/avax_traces_tx_hash.md create mode 100644 models/doc_descriptions/traces/avax_traces_type.md create mode 100644 models/doc_descriptions/traces/avax_traces_value.md create mode 100644 models/doc_descriptions/transactions/avax_cumulative_gas_used.md create mode 100644 models/doc_descriptions/transactions/avax_tx_block_hash.md create mode 100644 models/doc_descriptions/transactions/avax_tx_fee.md create mode 100644 models/doc_descriptions/transactions/avax_tx_gas_limit.md create mode 100644 models/doc_descriptions/transactions/avax_tx_gas_price.md create mode 100644 models/doc_descriptions/transactions/avax_tx_gas_used.md create mode 100644 models/doc_descriptions/transactions/avax_tx_hash.md create mode 100644 models/doc_descriptions/transactions/avax_tx_input_data.md create mode 100644 models/doc_descriptions/transactions/avax_tx_json.md create mode 100644 models/doc_descriptions/transactions/avax_tx_nonce.md create mode 100644 models/doc_descriptions/transactions/avax_tx_origin_sig.md create mode 100644 models/doc_descriptions/transactions/avax_tx_position.md create mode 100644 models/doc_descriptions/transactions/avax_tx_status.md create mode 100644 models/doc_descriptions/transactions/avax_tx_table_doc.md create mode 100644 models/doc_descriptions/transactions/avax_value.md create mode 100644 models/gold/core__dim_labels.sql create mode 100644 models/gold/core__dim_labels.yml create mode 100644 models/gold/core__fact_blocks.sql create mode 100644 models/gold/core__fact_blocks.yml create mode 100644 models/gold/core__fact_event_logs.sql create mode 100644 models/gold/core__fact_event_logs.yml create mode 100644 models/gold/core__fact_traces.sql create mode 100644 models/gold/core__fact_traces.yml create mode 100644 models/gold/core__fact_transactions.sql create mode 100644 models/gold/core__fact_transactions.yml create mode 100644 models/silver/silver__blocks.sql create mode 100644 models/silver/silver__blocks.yml create mode 100644 models/silver/silver__logs.sql create mode 100644 models/silver/silver__logs.yml create mode 100644 models/silver/silver__traces.sql create mode 100644 models/silver/silver__traces.yml create mode 100644 models/silver/silver__transactions.sql create mode 100644 models/silver/silver__transactions.yml create mode 100644 models/sources.yml create mode 100644 snapshots/.gitkeep create mode 100644 tests/.gitkeep create mode 100644 tests/optimism/silver__transactions__tx-gap.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fec6df8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ + +target/ +dbt_modules/ +# newer versions of dbt use this directory instead of dbt_modules for test dependencies +dbt_packages/ +logs/ + +.venv/ +.python-version + +# Visual Studio Code files +*/.vscode +*.code-workspace +.history/ +**/.DS_Store +.vscode/ diff --git a/README.md b/README.md index f02ecf1..301302b 100644 --- a/README.md +++ b/README.md @@ -1 +1,45 @@ -# avalanche-models \ No newline at end of file +## Profile Set Up + +#### Use the following within profiles.yml +---- + +```yml +avalanche: + target: dev + outputs: + dev: + type: snowflake + account: + role: + user: + password: + region: + database: AVALANCHE_DEV + warehouse: + schema: silver + threads: 12 + client_session_keep_alive: False + query_tag: + prod: + type: snowflake + account: + role: + user: + password: + region: + database: AVALANCHE + warehouse: + schema: silver + threads: 12 + client_session_keep_alive: False + query_tag: +``` + + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/analysis/.gitkeep b/analysis/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..0f90d35 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,39 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "avalanche_models" +version: "1.0.0" +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: "avalanche" + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + - "dbt_packages" + +on-run-start: + - "{{ create_sps() }}" + - "{{ create_udfs() }}" + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. + +vars: + "dbt_date:time_zone": GMT diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/create_sps.sql b/macros/create_sps.sql new file mode 100644 index 0000000..35ba27f --- /dev/null +++ b/macros/create_sps.sql @@ -0,0 +1,6 @@ +{% macro create_sps() %} + {% if target.database == 'AVALANCHE' %} + CREATE schema IF NOT EXISTS _internal; +{{ sp_create_prod_clone('_internal') }}; + {% endif %} +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql new file mode 100644 index 0000000..cc04a84 --- /dev/null +++ b/macros/create_udfs.sql @@ -0,0 +1,11 @@ +{% macro create_udfs() %} + {% set sql %} + CREATE schema if NOT EXISTS silver; +{{ create_js_hex_to_int() }}; +{{ create_udf_hex_to_int( + schema = "public" + ) }} + + {% endset %} + {% do run_query(sql) %} +{% endmacro %} diff --git a/macros/custom_naming_macros.sql b/macros/custom_naming_macros.sql new file mode 100644 index 0000000..0f4a72c --- /dev/null +++ b/macros/custom_naming_macros.sql @@ -0,0 +1,11 @@ +{% macro generate_schema_name(custom_schema_name=none, node=none) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name[0] | trim }} +{%- endmacro %} + +{% macro generate_alias_name(custom_alias_name=none, node=none) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name[1] | trim }} +{%- endmacro %} diff --git a/macros/js_hextoint.sql b/macros/js_hextoint.sql new file mode 100644 index 0000000..8742ac3 --- /dev/null +++ b/macros/js_hextoint.sql @@ -0,0 +1,6 @@ +{% macro create_js_hex_to_int() %} + CREATE + OR REPLACE FUNCTION {{ target.schema }}.js_hex_to_int ( + s STRING + ) returns DOUBLE LANGUAGE javascript AS 'if (S !== null) { yourNumber = parseInt(S, 16); } return yourNumber' +{% endmacro %} diff --git a/macros/python/udfs.sql b/macros/python/udfs.sql new file mode 100644 index 0000000..df0489e --- /dev/null +++ b/macros/python/udfs.sql @@ -0,0 +1,21 @@ +{% macro create_udf_hex_to_int(schema) %} +create or replace function {{ schema }}.udf_hex_to_int(hex string) +returns string +language python +runtime_version = '3.8' +handler = 'hex_to_int' +as +$$ +def hex_to_int(hex) -> str: + """ + Converts hex (of any size) to int (as a string). Snowflake and java script can only handle up to 64-bit (38 digits of precision) + select hex_to_int('200000000000000000000000000000211'); + >> 680564733841876926926749214863536423441 + select hex_to_int('0x200000000000000000000000000000211'); + >> 680564733841876926926749214863536423441 + select hex_to_int(NULL); + >> NULL + """ + return str(int(hex, 16)) if hex else None +$$; +{% endmacro %} \ No newline at end of file diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql new file mode 100644 index 0000000..0fd79c8 --- /dev/null +++ b/macros/run_sp_create_prod_clone.sql @@ -0,0 +1,10 @@ +{% macro run_sp_create_prod_clone() %} + {% set clone_query %} + call avalanche._internal.create_prod_clone( + 'avalanche', + 'avalanche_dev', + 'internal_dev' + ); +{% endset %} + {% do run_query(clone_query) %} +{% endmacro %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql new file mode 100644 index 0000000..5dcee63 --- /dev/null +++ b/macros/sp_create_prod_clone.sql @@ -0,0 +1,76 @@ +{% macro sp_create_prod_clone(target_schema) -%} + +create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string) +returns boolean +language javascript +execute as caller +as +$$ + snowflake.execute({sqlText: `BEGIN TRANSACTION;`}); + try { + snowflake.execute({sqlText: `DROP DATABASE IF EXISTS ${DESTINATION_DB_NAME}`}); + snowflake.execute({sqlText: `CREATE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); + snowflake.execute({sqlText: `DROP SCHEMA ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ + + var existing_schemas = snowflake.execute({sqlText: `SELECT table_schema + FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.TABLE_PRIVILEGES + WHERE grantor IS NOT NULL + GROUP BY 1 + UNION + SELECT 'PUBLIC';`}); + + while (existing_schemas.next()) { + var schema = existing_schemas.getColumnValue(1); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE FUNCTIONS IN SCHEMA ${DESTINATION_DB_NAME}.${schema} FROM ROLE DBT_CLOUD_AVALANCHE`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE PROCEDURES IN SCHEMA ${DESTINATION_DB_NAME}.${schema} FROM ROLE DBT_CLOUD_AVALANCHE`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE TABLES IN SCHEMA ${DESTINATION_DB_NAME}.${schema} FROM ROLE DBT_CLOUD_AVALANCHE`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE VIEWS IN SCHEMA ${DESTINATION_DB_NAME}.${schema} FROM ROLE DBT_CLOUD_AVALANCHE`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE FUNCTIONS IN SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE PROCEDURES IN SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME};`}); + } + + var existing_tables = snowflake.execute({sqlText: `SELECT table_schema, table_name + FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.TABLE_PRIVILEGES + WHERE grantor IS NOT NULL + GROUP BY 1,2;`}); + + while (existing_tables.next()) { + var schema = existing_tables.getColumnValue(1); + var table_name = existing_tables.getColumnValue(2); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON TABLE ${DESTINATION_DB_NAME}.${schema}.${table_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + } + + var existing_functions = snowflake.execute({sqlText: `SELECT function_schema, function_name, concat('(',array_to_string(regexp_substr_all(argument_signature, 'VARCHAR|NUMBER|FLOAT|ARRAY|VARIANT|OBJECT|DOUBLE'),','),')') as argument_signature + FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.FUNCTIONS;`}); + + while (existing_functions.next()) { + var schema = existing_functions.getColumnValue(1); + var function_name = existing_functions.getColumnValue(2); + var argument_signature = existing_functions.getColumnValue(3); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUNCTION ${DESTINATION_DB_NAME}.${schema}.${function_name}${argument_signature} to role ${ROLE_NAME} REVOKE CURRENT GRANTS;`}); + } + + var existing_procedures = snowflake.execute({sqlText: `SELECT procedure_schema, procedure_name, concat('(',array_to_string(regexp_substr_all(argument_signature, 'VARCHAR|NUMBER|FLOAT|ARRAY|VARIANT|OBJECT|DOUBLE'),','),')') as argument_signature + FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.PROCEDURES;`}); + + while (existing_procedures.next()) { + var schema = existing_procedures.getColumnValue(1); + var procedure_name = existing_procedures.getColumnValue(2); + var argument_signature = existing_procedures.getColumnValue(3); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON PROCEDURE ${DESTINATION_DB_NAME}.${schema}.${procedure_name}${argument_signature} to role ${ROLE_NAME} REVOKE CURRENT GRANTS;`}); + } + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}) + snowflake.execute({sqlText: `COMMIT;`}); + } catch (err) { + snowflake.execute({sqlText: `ROLLBACK;`}); + throw(err); + } + + return true +$$ + +{%- endmacro %} \ No newline at end of file diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql new file mode 100644 index 0000000..9425003 --- /dev/null +++ b/macros/tests/sequence_gaps.sql @@ -0,0 +1,34 @@ +{% test sequence_gaps( + model, + partition_by, + column_name +) %} +{%- set partition_sql = partition_by | join(", ") -%} +{%- set previous_column = "prev_" ~ column_name -%} +WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column_name }}, + LAG( + {{ column_name }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column_name }} ASC + ) AS {{ previous_column }} + FROM + {{ model }} +) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column_name }}, + {{ column_name }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column_name }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC {% endtest %} diff --git a/macros/tests/tx_gaps.sql b/macros/tests/tx_gaps.sql new file mode 100644 index 0000000..f7d687c --- /dev/null +++ b/macros/tests/tx_gaps.sql @@ -0,0 +1,33 @@ +{% macro tx_gaps( + model + ) %} + WITH block_base AS ( + SELECT + block_number, + tx_count + FROM + {{ ref('silver__blocks') }} + ), + model_name AS ( + SELECT + block_number, + COUNT( + DISTINCT tx_hash + ) AS model_tx_count + FROM + {{ model }} + GROUP BY + block_number + ) +SELECT + block_base.block_number, + tx_count, + model_name.block_number, + model_tx_count +FROM + block_base + LEFT JOIN model_name + ON block_base.block_number = model_name.block_number +WHERE + tx_count <> model_tx_count +{% endmacro %} diff --git a/models/bronze/bronze__blocks.sql b/models/bronze/bronze__blocks.sql new file mode 100644 index 0000000..8b50a62 --- /dev/null +++ b/models/bronze/bronze__blocks.sql @@ -0,0 +1,23 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + record_id, + offset_id, + block_id, + block_timestamp, + network, + chain_id, + tx_count, + header, + ingested_at, + _inserted_timestamp +FROM + {{ source( + 'prod', + 'avalanche_blocks' + ) }} + qualify(ROW_NUMBER() over(PARTITION BY block_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/bronze/bronze__transactions.sql b/models/bronze/bronze__transactions.sql new file mode 100644 index 0000000..5c24692 --- /dev/null +++ b/models/bronze/bronze__transactions.sql @@ -0,0 +1,21 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + record_id, + tx_id, + tx_block_index, + offset_id, + block_id, + block_timestamp, + network, + chain_id, + tx, + ingested_at, + _inserted_timestamp +FROM + {{ source( + 'prod', + 'avalanche_txs' + ) }} diff --git a/models/doc_descriptions/blocks/avax_block_header_json.md b/models/doc_descriptions/blocks/avax_block_header_json.md new file mode 100644 index 0000000..3ef36a9 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_block_header_json.md @@ -0,0 +1,5 @@ +{% docs avax_block_header_json %} + +This JSON column contains the block header details. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_blockchain.md b/models/doc_descriptions/blocks/avax_blockchain.md new file mode 100644 index 0000000..df3aa76 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_blockchain.md @@ -0,0 +1,5 @@ +{% docs avax_blockchain %} + +The blockchain on which transactions are being confirmed. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_blocks_hash.md b/models/doc_descriptions/blocks/avax_blocks_hash.md new file mode 100644 index 0000000..6e6ff1a --- /dev/null +++ b/models/doc_descriptions/blocks/avax_blocks_hash.md @@ -0,0 +1,5 @@ +{% docs avax_blocks_hash %} + +The hash of the block header for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_blocks_nonce.md b/models/doc_descriptions/blocks/avax_blocks_nonce.md new file mode 100644 index 0000000..f9e73bd --- /dev/null +++ b/models/doc_descriptions/blocks/avax_blocks_nonce.md @@ -0,0 +1,5 @@ +{% docs avax_blocks_nonce %} + +Block nonce is a value used during mining to demonstrate proof of work for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_blocks_table_doc.md b/models/doc_descriptions/blocks/avax_blocks_table_doc.md new file mode 100644 index 0000000..5479418 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_blocks_table_doc.md @@ -0,0 +1,5 @@ +{% docs avax_blocks_table_doc %} + +This table contains block level data for the Avalanche Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/) + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_difficulty.md b/models/doc_descriptions/blocks/avax_difficulty.md new file mode 100644 index 0000000..1f83004 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_difficulty.md @@ -0,0 +1,5 @@ +{% docs avax_difficulty %} + +The effort required to mine the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_extra_data.md b/models/doc_descriptions/blocks/avax_extra_data.md new file mode 100644 index 0000000..2dcfe87 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_extra_data.md @@ -0,0 +1,5 @@ +{% docs avax_extra_data %} + +Any data included by the validator for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_gas_limit.md b/models/doc_descriptions/blocks/avax_gas_limit.md new file mode 100644 index 0000000..201504a --- /dev/null +++ b/models/doc_descriptions/blocks/avax_gas_limit.md @@ -0,0 +1,5 @@ +{% docs avax_gas_limit %} + +Total gas limit provided by all transactions in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_gas_used.md b/models/doc_descriptions/blocks/avax_gas_used.md new file mode 100644 index 0000000..51b95af --- /dev/null +++ b/models/doc_descriptions/blocks/avax_gas_used.md @@ -0,0 +1,5 @@ +{% docs avax_gas_used %} + +Total gas used in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_miner.md b/models/doc_descriptions/blocks/avax_miner.md new file mode 100644 index 0000000..f852957 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_miner.md @@ -0,0 +1,5 @@ +{% docs avax_miner %} + +Miner who successfully added a given block to the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_network.md b/models/doc_descriptions/blocks/avax_network.md new file mode 100644 index 0000000..62c18a1 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_network.md @@ -0,0 +1,5 @@ +{% docs avax_network %} + +The network on the blockchain used by a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_parent_hash.md b/models/doc_descriptions/blocks/avax_parent_hash.md new file mode 100644 index 0000000..0d4f93d --- /dev/null +++ b/models/doc_descriptions/blocks/avax_parent_hash.md @@ -0,0 +1,5 @@ +{% docs avax_parent_hash %} + +The hash of the block from which a given block is generated. Also known as the parent block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_receipts_root.md b/models/doc_descriptions/blocks/avax_receipts_root.md new file mode 100644 index 0000000..8aa66aa --- /dev/null +++ b/models/doc_descriptions/blocks/avax_receipts_root.md @@ -0,0 +1,5 @@ +{% docs avax_receipts_root %} + +The root of the state trie. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_sha3_uncles.md b/models/doc_descriptions/blocks/avax_sha3_uncles.md new file mode 100644 index 0000000..8204c7b --- /dev/null +++ b/models/doc_descriptions/blocks/avax_sha3_uncles.md @@ -0,0 +1,5 @@ +{% docs avax_sha3_uncles %} + +The mechanism which Ethereum Javascript RLP encodes an empty string. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_size.md b/models/doc_descriptions/blocks/avax_size.md new file mode 100644 index 0000000..ea315f5 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_size.md @@ -0,0 +1,5 @@ +{% docs avax_size %} + +Block size, which is determined by a given block's gas limit. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_total_difficulty.md b/models/doc_descriptions/blocks/avax_total_difficulty.md new file mode 100644 index 0000000..dfeaa06 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_total_difficulty.md @@ -0,0 +1,5 @@ +{% docs avax_total_difficulty %} + +Total difficulty of the chain at a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_tx_count.md b/models/doc_descriptions/blocks/avax_tx_count.md new file mode 100644 index 0000000..66d138a --- /dev/null +++ b/models/doc_descriptions/blocks/avax_tx_count.md @@ -0,0 +1,5 @@ +{% docs avax_tx_count %} + +Total number of transactions within a block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/blocks/avax_uncle_blocks.md b/models/doc_descriptions/blocks/avax_uncle_blocks.md new file mode 100644 index 0000000..d726fd5 --- /dev/null +++ b/models/doc_descriptions/blocks/avax_uncle_blocks.md @@ -0,0 +1,5 @@ +{% docs avax_uncle_blocks %} + +Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_event_index.md b/models/doc_descriptions/event_logs/avax_event_index.md new file mode 100644 index 0000000..1199c30 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_event_index.md @@ -0,0 +1,5 @@ +{% docs avax_event_index %} + +Event number within a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_event_inputs.md b/models/doc_descriptions/event_logs/avax_event_inputs.md new file mode 100644 index 0000000..caa9bb0 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_event_inputs.md @@ -0,0 +1,5 @@ +{% docs avax_event_inputs %} + +The decoded event inputs for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_event_name.md b/models/doc_descriptions/event_logs/avax_event_name.md new file mode 100644 index 0000000..63dc280 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_event_name.md @@ -0,0 +1,5 @@ +{% docs avax_event_name %} + +The decoded event name for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_event_removed.md b/models/doc_descriptions/event_logs/avax_event_removed.md new file mode 100644 index 0000000..e0e1b3d --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_event_removed.md @@ -0,0 +1,5 @@ +{% docs avax_event_removed %} + +Whether the event has been removed from the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_event_sig.md b/models/doc_descriptions/event_logs/avax_event_sig.md new file mode 100644 index 0000000..2e38d08 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_event_sig.md @@ -0,0 +1,5 @@ +{% docs avax_origin_sig %} + +The function signature of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_log_id_events.md b/models/doc_descriptions/event_logs/avax_log_id_events.md new file mode 100644 index 0000000..f894a4d --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_log_id_events.md @@ -0,0 +1,5 @@ +{% docs avax_log_id_events %} + +This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_logs_contract_address.md b/models/doc_descriptions/event_logs/avax_logs_contract_address.md new file mode 100644 index 0000000..4234f83 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_logs_contract_address.md @@ -0,0 +1,5 @@ +{% docs avax_logs_contract_address %} + +The address interacted with for a given event. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_logs_contract_name.md b/models/doc_descriptions/event_logs/avax_logs_contract_name.md new file mode 100644 index 0000000..60e3579 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_logs_contract_name.md @@ -0,0 +1,5 @@ +{% docs avax_logs_contract_name %} + +The name of the contract or token, where possible. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_logs_data.md b/models/doc_descriptions/event_logs/avax_logs_data.md new file mode 100644 index 0000000..9eb17e0 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_logs_data.md @@ -0,0 +1,5 @@ +{% docs avax_logs_data %} + +The un-decoded event data. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_logs_table_doc.md b/models/doc_descriptions/event_logs/avax_logs_table_doc.md new file mode 100644 index 0000000..8e62e0f --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_logs_table_doc.md @@ -0,0 +1,5 @@ +{% docs avax_logs_table_doc %} + +This table contains flattened event logs from transactions on the Avalanche Blockchain. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index. Event names are decoded in this table where possible. The event inputs column will contain the log details in JSON format. Specific fields can be pulled from this column using the following sample format: ```event_inputs::: as ```. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_logs_tx_hash.md b/models/doc_descriptions/event_logs/avax_logs_tx_hash.md new file mode 100644 index 0000000..a8fd912 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_logs_tx_hash.md @@ -0,0 +1,5 @@ +{% docs avax_logs_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_origin_from.md b/models/doc_descriptions/event_logs/avax_origin_from.md new file mode 100644 index 0000000..e6bd337 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_origin_from.md @@ -0,0 +1,5 @@ +{% docs avax_origin_from %} + +The from address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_origin_to.md b/models/doc_descriptions/event_logs/avax_origin_to.md new file mode 100644 index 0000000..7c0c8ed --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_origin_to.md @@ -0,0 +1,5 @@ +{% docs avax_origin_to %} + +The to address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/event_logs/avax_topics.md b/models/doc_descriptions/event_logs/avax_topics.md new file mode 100644 index 0000000..6961b21 --- /dev/null +++ b/models/doc_descriptions/event_logs/avax_topics.md @@ -0,0 +1,5 @@ +{% docs avax_topics %} + +The un-decoded event input topics. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/__overview__.md b/models/doc_descriptions/general/__overview__.md new file mode 100644 index 0000000..4c65293 --- /dev/null +++ b/models/doc_descriptions/general/__overview__.md @@ -0,0 +1,79 @@ +{% docs __overview__ %} + +# Welcome to the Flipside Crypto Avalanche C-Chain Models Documentation! + +# NOTE: Data is in 'lite mode' - meaning, histrorical data has not yet been backfilled. Please see `min(block_timestamp)` + +## **What is Flipside?** + +[Flipside Crypto](https://flipsidecrypto.xyz/earn) provides Community Enabled Crypto Analytics, allowing our users to create and share data insights on the crypto projects they care most about. + +**Flipside Crypto puts pre-modeled and labeled blockchain data in the hands of communities.** + +Through dashboard and visualization tools, as well as auto-generated API endpoints, data analysts can easily create queries that answer any question via a tool called [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover). + +**Community members earn bounties for answering questions with data.** + +Bounties provide incentive and direction, so crypto projects can quickly source the data insights they need in order to grow. + +**Flipside works directly with leading crypto projects to reward on-demand analytics through structured bounty programs.** + +Questions sourced directly from the community provide insight into what communities care about as well as analytics needed to drive ecosystem engagement and growth. + +## **What does this documentation cover?** +The documentation included here details the design of the Avalanche tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/earn) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/avalanche-models) + +### **Quick Links to Table Documentation** + +- [fact_blocks](https://flipsidecrypto.github.io/avalanche-models/#!/model/model.avalanche_models.core__fact_blocks) +- [fact_event_logs](https://flipsidecrypto.github.io/avalanche-models/#!/model/model.avalanche_models.core__fact_event_logs) +- [fact_traces](https://flipsidecrypto.github.io/avalanche-models/#!/model/model.avalanche_models.core__fact_traces) +- [fact_transactions](https://flipsidecrypto.github.io/avalanche-models/#!/model/model.avalanche_models.core__fact_transactions) +- [dim_lables](https://flipsidecrypto.github.io/avalanche-models/#!/model/model.avalanche_models.core__dim_labels) + +## **Data Model Overview** + +The Avalanche models are built a few different ways, but the core fact table are built using three layers of sql models: **bronze, silver, and gold (or core).** + +- Bronze: Data is loaded in from the source as a view +- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here +- Gold (or core): Final views and tables that are available in Velocity + +The dimension tables are sourced from a variety of on-chain and off-chain sources. + +Convenience views (denoted ez_) are a combination of different fact and dimension tables. + +A user-defined-function (UDF) is available to decode hex encoded values to integers in this database. You can call this UDF by using `avalanche.public.udf_hex_to_int(FIELD)`. + + +## **Using dbt docs** +### Navigation + +You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project. + +### Database Tab + +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database. + +### Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. + +Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + + +### **More information** +- [Flipside](https://flipsidecrypto.xyz/earn) +- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) +- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Github](https://github.com/FlipsideCrypto/avalanche-models) +- [Query Editor Shortcuts](https://docs.flipsidecrypto.com/velocity/query-editor-shortcuts) +- [What is dbt?](https://docs.getdbt.com/docs/introduction) + + + +{% enddocs %} diff --git a/models/doc_descriptions/general/avax_block_number.md b/models/doc_descriptions/general/avax_block_number.md new file mode 100644 index 0000000..b2f2b45 --- /dev/null +++ b/models/doc_descriptions/general/avax_block_number.md @@ -0,0 +1,5 @@ +{% docs avax_block_number %} + +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. + +{% enddocs %} diff --git a/models/doc_descriptions/general/avax_block_timestamp.md b/models/doc_descriptions/general/avax_block_timestamp.md new file mode 100644 index 0000000..40ed0cd --- /dev/null +++ b/models/doc_descriptions/general/avax_block_timestamp.md @@ -0,0 +1,5 @@ +{% docs avax_block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/avax_decimals.md b/models/doc_descriptions/general/avax_decimals.md new file mode 100644 index 0000000..80633d9 --- /dev/null +++ b/models/doc_descriptions/general/avax_decimals.md @@ -0,0 +1,5 @@ +{% docs avax_decimals %} + +The number of decimal places this contract needs adjusted where token values exist. For example, use the decimal field to correctly transform raw amounts in ```fact_token_transfers```. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/avax_from_address.md b/models/doc_descriptions/general/avax_from_address.md new file mode 100644 index 0000000..76e25a4 --- /dev/null +++ b/models/doc_descriptions/general/avax_from_address.md @@ -0,0 +1,5 @@ +{% docs avax_from_address %} + +The sending address of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/avax_ingested_at.md b/models/doc_descriptions/general/avax_ingested_at.md new file mode 100644 index 0000000..398a5a9 --- /dev/null +++ b/models/doc_descriptions/general/avax_ingested_at.md @@ -0,0 +1,5 @@ +{% docs avax_ingested_at %} + +Internal column. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/general/avax_to_address.md b/models/doc_descriptions/general/avax_to_address.md new file mode 100644 index 0000000..3f34e01 --- /dev/null +++ b/models/doc_descriptions/general/avax_to_address.md @@ -0,0 +1,5 @@ +{% docs avax_to_address %} + +The receiving address of this transaction. This can be a contract address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label.md b/models/doc_descriptions/labels/avax_label.md new file mode 100644 index 0000000..ee33800 --- /dev/null +++ b/models/doc_descriptions/labels/avax_label.md @@ -0,0 +1,5 @@ +{% docs avax_project_name %} + +The name of the project for this address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label_address.md b/models/doc_descriptions/labels/avax_label_address.md new file mode 100644 index 0000000..812c0fc --- /dev/null +++ b/models/doc_descriptions/labels/avax_label_address.md @@ -0,0 +1,5 @@ +{% docs avax_label_address %} + +Address that the label is for. This is the field that should be used to join other tables with labels. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label_address_name.md b/models/doc_descriptions/labels/avax_label_address_name.md new file mode 100644 index 0000000..fa303ea --- /dev/null +++ b/models/doc_descriptions/labels/avax_label_address_name.md @@ -0,0 +1,5 @@ +{% docs avax_label_address_name %} + +The most granular label for this address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label_blockchain.md b/models/doc_descriptions/labels/avax_label_blockchain.md new file mode 100644 index 0000000..79a4dcc --- /dev/null +++ b/models/doc_descriptions/labels/avax_label_blockchain.md @@ -0,0 +1,5 @@ +{% docs avax_label_blockchain %} + +The name of the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label_creator.md b/models/doc_descriptions/labels/avax_label_creator.md new file mode 100644 index 0000000..c9e5153 --- /dev/null +++ b/models/doc_descriptions/labels/avax_label_creator.md @@ -0,0 +1,5 @@ +{% docs avax_label_creator %} + +The name of the creator of the label. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label_subtype.md b/models/doc_descriptions/labels/avax_label_subtype.md new file mode 100644 index 0000000..52978e3 --- /dev/null +++ b/models/doc_descriptions/labels/avax_label_subtype.md @@ -0,0 +1,5 @@ +{% docs avax_label_subtype %} + +A sub-category nested within label type providing further detail. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_label_type.md b/models/doc_descriptions/labels/avax_label_type.md new file mode 100644 index 0000000..2751f13 --- /dev/null +++ b/models/doc_descriptions/labels/avax_label_type.md @@ -0,0 +1,5 @@ +{% docs avax_label_type %} + +A high-level category describing the addresses main function or ownership. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/labels/avax_labels_table_doc.md b/models/doc_descriptions/labels/avax_labels_table_doc.md new file mode 100644 index 0000000..3e88b42 --- /dev/null +++ b/models/doc_descriptions/labels/avax_labels_table_doc.md @@ -0,0 +1,5 @@ +{% docs avax_labels_table %} + +This table contains labels for addresses on the Avalanche Blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_block_no.md b/models/doc_descriptions/traces/avax_traces_block_no.md new file mode 100644 index 0000000..2fb5e93 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_block_no.md @@ -0,0 +1,5 @@ +{% docs avax_traces_block_no %} + +The block number of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_blocktime.md b/models/doc_descriptions/traces/avax_traces_blocktime.md new file mode 100644 index 0000000..e248b67 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_blocktime.md @@ -0,0 +1,5 @@ +{% docs avax_traces_blocktime %} + +The block timestamp of this transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_call_data.md b/models/doc_descriptions/traces/avax_traces_call_data.md new file mode 100644 index 0000000..cb835a7 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_call_data.md @@ -0,0 +1,5 @@ +{% docs avax_traces_call_data %} + +The raw JSON data for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_from.md b/models/doc_descriptions/traces/avax_traces_from.md new file mode 100644 index 0000000..5003bc6 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_from.md @@ -0,0 +1,5 @@ +{% docs avax_traces_from %} + +The sending address of this trace. This is not necessarily the from address of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_gas.md b/models/doc_descriptions/traces/avax_traces_gas.md new file mode 100644 index 0000000..c2fdda8 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_gas.md @@ -0,0 +1,5 @@ +{% docs avax_traces_gas %} + +The gas supplied for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_gas_used.md b/models/doc_descriptions/traces/avax_traces_gas_used.md new file mode 100644 index 0000000..17da65f --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_gas_used.md @@ -0,0 +1,5 @@ +{% docs avax_traces_gas_used %} + +The gas used for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_identifier.md b/models/doc_descriptions/traces/avax_traces_identifier.md new file mode 100644 index 0000000..d1b85d8 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_identifier.md @@ -0,0 +1,5 @@ +{% docs avax_traces_identifier %} + +This field represents the position and type of the trace within the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_input.md b/models/doc_descriptions/traces/avax_traces_input.md new file mode 100644 index 0000000..2b2d1ed --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_input.md @@ -0,0 +1,5 @@ +{% docs avax_traces_input %} + +The input data for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_output.md b/models/doc_descriptions/traces/avax_traces_output.md new file mode 100644 index 0000000..11743d4 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_output.md @@ -0,0 +1,5 @@ +{% docs avax_traces_output %} + +The output data for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_sub.md b/models/doc_descriptions/traces/avax_traces_sub.md new file mode 100644 index 0000000..570d7fd --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_sub.md @@ -0,0 +1,5 @@ +{% docs avax_traces_sub %} + +The amount of nested sub traces for this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_table_doc.md b/models/doc_descriptions/traces/avax_traces_table_doc.md new file mode 100644 index 0000000..40706d3 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_table_doc.md @@ -0,0 +1,5 @@ +{% docs avax_traces_table_doc %} + +This table contains flattened trace data for internal contract calls on the Avalanche Blockchain. Hex encoded fields can be decoded to integers by using `avalanche.public.udf_hex_to_int()`. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_to.md b/models/doc_descriptions/traces/avax_traces_to.md new file mode 100644 index 0000000..c054280 --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_to.md @@ -0,0 +1,5 @@ +{% docs avax_traces_to %} + +The receiving address of this trace. This is not necessarily the to address of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_tx_hash.md b/models/doc_descriptions/traces/avax_traces_tx_hash.md new file mode 100644 index 0000000..da61aee --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_tx_hash.md @@ -0,0 +1,5 @@ +{% docs avax_traces_tx_hash %} + +The transaction hash for the trace. Please note, this is not necessarily unique in this table as transactions frequently have multiple traces. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_type.md b/models/doc_descriptions/traces/avax_traces_type.md new file mode 100644 index 0000000..768428f --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_type.md @@ -0,0 +1,5 @@ +{% docs avax_traces_type %} + +The type of internal transaction. Common trace types are `CALL`, `DELEGATECALL`, and `STATICCALL`. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/traces/avax_traces_value.md b/models/doc_descriptions/traces/avax_traces_value.md new file mode 100644 index 0000000..eb1f9eb --- /dev/null +++ b/models/doc_descriptions/traces/avax_traces_value.md @@ -0,0 +1,5 @@ +{% docs avax_traces_value %} + +The amount of ETH transferred in this trace. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_cumulative_gas_used.md b/models/doc_descriptions/transactions/avax_cumulative_gas_used.md new file mode 100644 index 0000000..48fc568 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_cumulative_gas_used.md @@ -0,0 +1,5 @@ +{% docs avax_cumulative_gas_used %} + +The total amount of gas used when this transaction was executed in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_block_hash.md b/models/doc_descriptions/transactions/avax_tx_block_hash.md new file mode 100644 index 0000000..2e09696 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_block_hash.md @@ -0,0 +1,5 @@ +{% docs avax_tx_block_hash %} + +Block hash is a unique 66-character identifier that is generate when a block is produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_fee.md b/models/doc_descriptions/transactions/avax_tx_fee.md new file mode 100644 index 0000000..8b43e04 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_fee.md @@ -0,0 +1,5 @@ +{% docs avax_tx_fee %} + +Amount paid to validate the transaction in ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_gas_limit.md b/models/doc_descriptions/transactions/avax_tx_gas_limit.md new file mode 100644 index 0000000..2077f4b --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_gas_limit.md @@ -0,0 +1,5 @@ +{% docs avax_tx_gas_limit %} + +Maximum amount of gas allocated for the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_gas_price.md b/models/doc_descriptions/transactions/avax_tx_gas_price.md new file mode 100644 index 0000000..bc6cb4e --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_gas_price.md @@ -0,0 +1,5 @@ +{% docs avax_tx_gas_price %} + +Cost per unit of gas in Gwei. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_gas_used.md b/models/doc_descriptions/transactions/avax_tx_gas_used.md new file mode 100644 index 0000000..3a9b74c --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_gas_used.md @@ -0,0 +1,5 @@ +{% docs avax_tx_gas_used %} + +Gas used by transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_hash.md b/models/doc_descriptions/transactions/avax_tx_hash.md new file mode 100644 index 0000000..7cee633 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_hash.md @@ -0,0 +1,5 @@ +{% docs avax_tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_input_data.md b/models/doc_descriptions/transactions/avax_tx_input_data.md new file mode 100644 index 0000000..b501905 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_input_data.md @@ -0,0 +1,5 @@ +{% docs avax_tx_input_data %} + +This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_json.md b/models/doc_descriptions/transactions/avax_tx_json.md new file mode 100644 index 0000000..2776607 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_json.md @@ -0,0 +1,5 @@ +{% docs avax_tx_json %} + +This JSON column contains the transaction details, including event logs. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_nonce.md b/models/doc_descriptions/transactions/avax_tx_nonce.md new file mode 100644 index 0000000..c0eaeed --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_nonce.md @@ -0,0 +1,5 @@ +{% docs avax_tx_nonce %} + +The number of transactions sent from a given address. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_origin_sig.md b/models/doc_descriptions/transactions/avax_tx_origin_sig.md new file mode 100644 index 0000000..4d0ae50 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_origin_sig.md @@ -0,0 +1,5 @@ +{% docs avax_tx_origin_sig %} + +The function signature of the contract call. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_position.md b/models/doc_descriptions/transactions/avax_tx_position.md new file mode 100644 index 0000000..a84e1c1 --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_position.md @@ -0,0 +1,5 @@ +{% docs avax_tx_position %} + +The position of the transaction within the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_status.md b/models/doc_descriptions/transactions/avax_tx_status.md new file mode 100644 index 0000000..f3a041d --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_status.md @@ -0,0 +1,5 @@ +{% docs avax_tx_status %} + +Status of the transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_tx_table_doc.md b/models/doc_descriptions/transactions/avax_tx_table_doc.md new file mode 100644 index 0000000..2fbe5cf --- /dev/null +++ b/models/doc_descriptions/transactions/avax_tx_table_doc.md @@ -0,0 +1,5 @@ +{% docs avax_tx_table_doc %} + +This table contains transaction level data for the Avalanche Blockchain. Each transaction will have a unique transaction hash, along with transactions fees and a ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/) + +{% enddocs %} \ No newline at end of file diff --git a/models/doc_descriptions/transactions/avax_value.md b/models/doc_descriptions/transactions/avax_value.md new file mode 100644 index 0000000..fe5242c --- /dev/null +++ b/models/doc_descriptions/transactions/avax_value.md @@ -0,0 +1,5 @@ +{% docs avax_value %} + +The value transacted in ETH. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core__dim_labels.sql b/models/gold/core__dim_labels.sql new file mode 100644 index 0000000..6e2f420 --- /dev/null +++ b/models/gold/core__dim_labels.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + blockchain, + creator, + address, + address_name, + label_type, + label_subtype, + project_name +FROM + {{ source( + 'crosschain', + 'address_labels' + ) }} +WHERE + blockchain = 'avalanche' + AND address LIKE '0x%' diff --git a/models/gold/core__dim_labels.yml b/models/gold/core__dim_labels.yml new file mode 100644 index 0000000..0caf51c --- /dev/null +++ b/models/gold/core__dim_labels.yml @@ -0,0 +1,57 @@ +version: 2 +models: + - name: core__dim_labels + description: '{{ doc("avax_labels_table") }}' + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCKCHAIN + - CREATOR + - ADDRESS + columns: + - name: BLOCKCHAIN + description: '{{ doc("avax_label_blockchain") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['avalanche'] + - name: CREATOR + description: '{{ doc("avax_label_creator") }}' + tests: + - not_null + - name: ADDRESS + description: '{{ doc("avax_label_address") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ADDRESS_NAME + description: '{{ doc("avax_labels_table") }}' + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: LABEL_TYPE + description: '{{ doc("avax_label_type") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['flotsam', 'nft', 'defi', 'dex', 'cex', 'dapp', 'token', 'operator', 'layer2', 'chadmin', 'project'] + - name: LABEL_SUBTYPE + description: '{{ doc("avax_label_subtype") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: PROJECT_NAME + description: '{{ doc("avax_project_name") }}' + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + diff --git a/models/gold/core__fact_blocks.sql b/models/gold/core__fact_blocks.sql new file mode 100644 index 0000000..b36e456 --- /dev/null +++ b/models/gold/core__fact_blocks.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + network, + blockchain, + tx_count, + difficulty, + total_difficulty, + extra_data, + gas_limit, + gas_used, + HASH, + parent_hash, + receipts_root, + sha3_uncles, + SIZE, + uncle_blocks, + block_header_json +FROM + {{ ref('silver__blocks') }} diff --git a/models/gold/core__fact_blocks.yml b/models/gold/core__fact_blocks.yml new file mode 100644 index 0000000..8a709e2 --- /dev/null +++ b/models/gold/core__fact_blocks.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: core__fact_blocks + description: '{{ doc("avax_blocks_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("avax_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("avax_block_timestamp") }}' + - name: NETWORK + description: '{{ doc("avax_network") }}' + - name: BLOCKCHAIN + description: '{{ doc("avax_blockchain") }}' + - name: TX_COUNT + description: '{{ doc("avax_tx_count") }}' + - name: DIFFICULTY + description: '{{ doc("avax_difficulty") }}' + - name: TOTAL_DIFFICULTY + description: '{{ doc("avax_total_difficulty") }}' + - name: EXTRA_DATA + description: '{{ doc("avax_extra_data") }}' + - name: GAS_LIMIT + description: '{{ doc("avax_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("avax_gas_used") }}' + - name: HASH + description: '{{ doc("avax_blocks_hash") }}' + - name: PARENT_HASH + description: '{{ doc("avax_parent_hash") }}' + - name: RECEIPTS_ROOT + description: '{{ doc("avax_receipts_root") }}' + - name: SHA3_UNCLES + description: '{{ doc("avax_sha3_uncles") }}' + - name: SIZE + description: '{{ doc("avax_size") }}' + - name: UNCLE_BLOCKS + description: '{{ doc("avax_uncle_blocks") }}' + - name: BLOCK_HEADER_JSON + description: '{{ doc("avax_block_header_json") }}' \ No newline at end of file diff --git a/models/gold/core__fact_event_logs.sql b/models/gold/core__fact_event_logs.sql new file mode 100644 index 0000000..71af091 --- /dev/null +++ b/models/gold/core__fact_event_logs.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + event_index, + contract_address, + contract_name, + event_name, + event_inputs, + topics, + DATA, + event_removed, + tx_status, + _log_id +FROM + {{ ref('silver__logs') }} diff --git a/models/gold/core__fact_event_logs.yml b/models/gold/core__fact_event_logs.yml new file mode 100644 index 0000000..0ceeb45 --- /dev/null +++ b/models/gold/core__fact_event_logs.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: core__fact_event_logs + description: '{{ doc("avax_logs_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("avax_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("avax_block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("avax_logs_tx_hash") }}' + - name: EVENT_INDEX + description: '{{ doc("avax_event_index") }}' + - name: CONTRACT_ADDRESS + description: '{{ doc("avax_logs_contract_address") }}' + - name: CONTRACT_NAME + description: '{{ doc("avax_logs_contract_name") }}' + - name: EVENT_NAME + description: '{{ doc("avax_event_name") }}' + - name: EVENT_INPUTS + description: '{{ doc("avax_event_inputs") }}' + - name: TOPICS + description: '{{ doc("avax_topics") }}' + - name: DATA + description: '{{ doc("avax_logs_data") }}' + - name: EVENT_REMOVED + description: '{{ doc("avax_event_removed") }}' + - name: _LOG_ID + description: '{{ doc("avax_log_id_events") }}' + - name: TX_STATUS + description: '{{ doc("avax_tx_status") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("avax_origin_sig") }}' + - name: ORIGIN_FROM_ADDRESS + description: '{{ doc("avax_origin_from") }}' + - name: ORIGIN_TO_ADDRESS + description: '{{ doc("avax_origin_to") }}' diff --git a/models/gold/core__fact_traces.sql b/models/gold/core__fact_traces.sql new file mode 100644 index 0000000..1beb908 --- /dev/null +++ b/models/gold/core__fact_traces.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + DATA, + tx_status, + sub_traces +FROM + {{ ref('silver__traces') }} diff --git a/models/gold/core__fact_traces.yml b/models/gold/core__fact_traces.yml new file mode 100644 index 0000000..694c7a5 --- /dev/null +++ b/models/gold/core__fact_traces.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: core__fact_traces + description: '{{ doc("avax_traces_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("avax_traces_block_no") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("avax_traces_blocktime") }}' + - name: TX_HASH + description: '{{ doc("avax_traces_tx_hash") }}' + - name: FROM_ADDRESS + description: '{{ doc("avax_traces_from") }}' + - name: TO_ADDRESS + description: '{{ doc("avax_traces_to") }}' + - name: ETH_VALUE + description: '{{ doc("avax_traces_value") }}' + - name: GAS + description: '{{ doc("avax_traces_gas") }}' + - name: GAS_USED + description: '{{ doc("avax_traces_gas_used") }}' + - name: INPUT + description: '{{ doc("avax_traces_input") }}' + - name: OUTPUT + description: '{{ doc("avax_traces_output") }}' + - name: TYPE + description: '{{ doc("avax_traces_type") }}' + - name: IDENTIFIER + description: '{{ doc("avax_traces_identifier") }}' + - name: DATA + description: '{{ doc("avax_traces_call_data") }}' + - name: TX_STATUS + description: '{{ doc("avax_tx_status") }}' + - name: SUB_TRACES + description: '{{ doc("avax_traces_sub") }}' + + + + \ No newline at end of file diff --git a/models/gold/core__fact_transactions.sql b/models/gold/core__fact_transactions.sql new file mode 100644 index 0000000..2091172 --- /dev/null +++ b/models/gold/core__fact_transactions.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + block_number, + block_timestamp, + block_hash, + tx_hash, + nonce, + POSITION, + origin_function_signature, + from_address, + to_address, + eth_value, + tx_fee, + gas_price, + gas_limit, + gas_used, + cumulative_Gas_Used, + input_data, + status, + tx_json +FROM + {{ ref('silver__transactions') }} diff --git a/models/gold/core__fact_transactions.yml b/models/gold/core__fact_transactions.yml new file mode 100644 index 0000000..1810c1f --- /dev/null +++ b/models/gold/core__fact_transactions.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: core__fact_transactions + description: '{{ doc("avax_tx_table_doc") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("avax_block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("avax_block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("avax_tx_block_hash") }}' + - name: TX_HASH + description: '{{ doc("avax_tx_hash") }}' + - name: NONCE + description: '{{ doc("avax_tx_nonce") }}' + - name: POSITION + description: '{{ doc("avax_tx_position") }}' + - name: FROM_ADDRESS + description: '{{ doc("avax_from_address") }}' + - name: TO_ADDRESS + description: '{{ doc("avax_to_address") }}' + - name: ETH_VALUE + description: '{{ doc("avax_value") }}' + - name: TX_FEE + description: '{{ doc("avax_tx_fee") }}' + - name: GAS_PRICE + description: '{{ doc("avax_tx_gas_price") }}' + - name: GAS_LIMIT + description: '{{ doc("avax_tx_gas_limit") }}' + - name: GAS_USED + description: '{{ doc("avax_tx_gas_used") }}' + - name: CUMULATIVE_GAS_USED + description: '{{ doc("avax_cumulative_gas_used") }}' + - name: STATUS + description: '{{ doc("avax_tx_status") }}' + - name: TX_JSON + description: '{{ doc("avax_tx_json") }}' + - name: INPUT_DATA + description: '{{ doc("avax_tx_input_data") }}' + - name: ORIGIN_FUNCTION_SIGNATURE + description: '{{ doc("avax_tx_origin_sig") }}' \ No newline at end of file diff --git a/models/silver/silver__blocks.sql b/models/silver/silver__blocks.sql new file mode 100644 index 0000000..beab464 --- /dev/null +++ b/models/silver/silver__blocks.sql @@ -0,0 +1,77 @@ +{{ config( + materialized = 'incremental', + unique_key = "block_number", + cluster_by = ['_inserted_timestamp::DATE'] +) }} + +WITH base_tables AS ( + + SELECT + record_id, + offset_id, + block_id, + block_timestamp, + network, + chain_id, + tx_count, + header, + ingested_at, + _inserted_timestamp + FROM + {{ ref('bronze__blocks') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + block_id :: INTEGER AS block_number, + block_timestamp :: TIMESTAMP AS block_timestamp, + network :: STRING AS network, + chain_id :: STRING AS blockchain, + tx_count :: INTEGER AS tx_count, + udf_hex_to_int( + header :difficulty :: STRING + ) :: INTEGER AS difficulty, + udf_hex_to_int( + header :totalDifficulty :: STRING + ) :: INTEGER AS total_difficulty, + header: extraData :: STRING AS extra_data, + udf_hex_to_int( + header :gasLimit :: STRING + ) :: INTEGER AS gas_limit, + udf_hex_to_int( + header :gasUsed :: STRING + ) :: INTEGER AS gas_used, + header: "hash" :: STRING AS HASH, + header: parentHash :: STRING AS parent_hash, + header: miner :: STRING AS miner, + header: nonce :: STRING AS nonce, + header: receiptsRoot :: STRING AS receipts_root, + header: sha3Uncles :: STRING AS sha3_uncles, + udf_hex_to_int( + header: "size" :: STRING + ) :: INTEGER AS SIZE, + CASE + WHEN header: uncles [1] :: STRING IS NOT NULL THEN CONCAT( + header: uncles [0] :: STRING, + ', ', + header: uncles [1] :: STRING + ) + ELSE header: uncles [0] :: STRING + END AS uncle_blocks, + ingested_at :: TIMESTAMP AS ingested_at, + header :: OBJECT AS block_header_json, + _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp +FROM + base_tables qualify(ROW_NUMBER() over(PARTITION BY block_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/silver__blocks.yml b/models/silver/silver__blocks.yml new file mode 100644 index 0000000..b91c0ba --- /dev/null +++ b/models/silver/silver__blocks.yml @@ -0,0 +1,123 @@ +version: 2 +models: + - name: silver__blocks + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_NUMBER + # - sequence_gaps: + # column_name: BLOCK_NUMBER + # where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: NETWORK + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: BLOCKCHAIN + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TX_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_DIFFICULTY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EXTRA_DATA + tests: + - not_null + - name: GAS_LIMIT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: PARENT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: MINER + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: RECEIPTS_ROOT + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SHA3_UNCLES + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: SIZE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_HEADER_JSON + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - OBJECT + + diff --git a/models/silver/silver__logs.sql b/models/silver/silver__logs.sql new file mode 100644 index 0000000..0742e50 --- /dev/null +++ b/models/silver/silver__logs.sql @@ -0,0 +1,111 @@ +{{ config( + materialized = 'incremental', + unique_key = "_log_id", + cluster_by = ['_inserted_timestamp::DATE'] +) }} + +WITH base_txs AS ( + + SELECT + record_id, + tx_id, + tx_block_index, + offset_id, + block_id, + block_timestamp, + network, + chain_id, + tx, + ingested_at, + _inserted_timestamp + FROM + {{ ref('bronze__transactions') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} +), +logs_raw AS ( + SELECT + block_id, + block_timestamp, + tx_id AS tx_hash, + tx :receipt :logs AS full_logs, + ingested_at :: TIMESTAMP AS ingested_at, + _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp, + CASE + WHEN tx :receipt :status :: STRING = '0x1' THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + SUBSTR( + tx :input :: STRING, + 1, + 10 + ) AS origin_function_signature, + tx :from :: STRING AS origin_from_address, + tx :to :: STRING AS origin_to_address + FROM + base_txs +), +logs AS ( + SELECT + block_id, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + tx_status, + ingested_at, + _inserted_timestamp, + udf_hex_to_int( + VALUE :logIndex :: STRING + ) :: INTEGER AS event_index, + VALUE :address :: STRING AS contract_address, + VALUE :decoded :contractName :: STRING AS contract_name, + VALUE :decoded :eventName :: STRING AS event_name, + VALUE :decoded :inputs :: OBJECT AS event_inputs, + VALUE :topics AS topics, + VALUE :data :: STRING AS DATA, + VALUE :removed :: STRING AS event_removed + FROM + logs_raw, + LATERAL FLATTEN ( + input => full_logs + ) +) +SELECT + concat_ws( + '-', + tx_hash, + event_index + ) AS _log_id, + block_id AS block_number, + block_timestamp, + tx_hash, + origin_function_signature, + origin_from_address, + origin_to_address, + ingested_at, + _inserted_timestamp, + event_index, + contract_address, + contract_name, + event_name, + event_inputs, + topics, + DATA, + event_removed, + tx_status +FROM + logs qualify(ROW_NUMBER() over(PARTITION BY _log_id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/silver__logs.yml b/models/silver/silver__logs.yml new file mode 100644 index 0000000..7507428 --- /dev/null +++ b/models/silver/silver__logs.yml @@ -0,0 +1,81 @@ +version: 2 +models: + - name: silver__logs + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _LOG_ID + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + - TX_HASH + column_name: EVENT_INDEX + where: BLOCK_TIMESTAMP < CURRENT_DATE + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CONTRACT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: EVENT_NAME + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENT_INPUTS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - OBJECT + - name: TOPICS + tests: + - not_null + - name: DATA + tests: + - not_null + - name: EVENT_REMOVED + tests: + - not_null + - name: _LOG_ID + tests: + - not_null + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + - name: ORIGIN_FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: ORIGIN_TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ \ No newline at end of file diff --git a/models/silver/silver__traces.sql b/models/silver/silver__traces.sql new file mode 100644 index 0000000..f2f0df0 --- /dev/null +++ b/models/silver/silver__traces.sql @@ -0,0 +1,221 @@ +{{ config( + materialized = 'incremental', + unique_key = '_call_id', + cluster_by = ['_inserted_timestamp::DATE'] +) }} + +WITH new_blocks AS ( + + SELECT + block_id + FROM + {{ ref('bronze__blocks') }} + WHERE + tx_count > 0 + +{% if is_incremental() %} +AND block_id NOT IN ( + SELECT + DISTINCT block_number + FROM + {{ this }} +) +{% endif %} +ORDER BY + _inserted_timestamp DESC +LIMIT + 500000 +), traces_txs AS ( + SELECT + * + FROM + {{ ref('bronze__transactions') }} + WHERE + block_id IN ( + SELECT + block_id + FROM + new_blocks + ) qualify(ROW_NUMBER() over(PARTITION BY tx_id + ORDER BY + _inserted_timestamp DESC)) = 1 +), +base_table AS ( + SELECT + CASE + WHEN POSITION( + '.', + path :: STRING + ) > 0 THEN REPLACE( + REPLACE( + path :: STRING, + SUBSTR(path :: STRING, len(path :: STRING) - POSITION('.', REVERSE(path :: STRING)) + 1, POSITION('.', REVERSE(path :: STRING))), + '' + ), + '.', + '__' + ) + ELSE '__' + END AS id, + OBJECT_AGG( + DISTINCT key, + VALUE + ) AS DATA, + txs.tx_id AS tx_hash, + txs.block_id AS block_number, + txs.block_timestamp AS block_timestamp, + CASE + WHEN txs.tx :receipt :status :: STRING = '0x1' THEN 'SUCCESS' + ELSE 'FAIL' + END AS tx_status, + txs.ingested_at AS ingested_at, + txs._inserted_timestamp AS _inserted_timestamp + FROM + traces_txs txs, + TABLE( + FLATTEN( + input => PARSE_JSON( + txs.tx :traces + ), + recursive => TRUE + ) + ) f + WHERE + f.index IS NULL + AND f.key != 'calls' + GROUP BY + tx_hash, + id, + block_number, + block_timestamp, + ingested_at, + _inserted_timestamp, + tx_status +), +flattened_traces AS ( + SELECT + DATA :from :: STRING AS from_address, + udf_hex_to_int( + DATA :gas :: STRING + ) AS gas, + udf_hex_to_int( + DATA :gasUsed :: STRING + ) AS gas_used, + DATA :input :: STRING AS input, + DATA :output :: STRING AS output, + DATA :time :: STRING AS TIME, + DATA :to :: STRING AS to_address, + DATA :type :: STRING AS TYPE, + CASE + WHEN DATA :type :: STRING = 'CALL' THEN udf_hex_to_int( + DATA :value :: STRING + ) / pow( + 10, + 18 + ) + ELSE 0 + END AS eth_value, + CASE + WHEN id = '__' THEN CONCAT( + DATA :type :: STRING, + '_ORIGIN' + ) + ELSE CONCAT( + DATA :type :: STRING, + '_', + REPLACE( + REPLACE(REPLACE(REPLACE(id, 'calls', ''), '[', ''), ']', ''), + '__', + '_' + ) + ) + END AS identifier, + concat_ws( + '-', + tx_hash, + identifier + ) AS _call_id, + SPLIT( + identifier, + '_' + ) AS id_split, + ARRAY_SLICE(id_split, 1, ARRAY_SIZE(id_split)) AS levels, + ARRAY_TO_STRING( + levels, + '_' + ) AS LEVEL, + CASE + WHEN ARRAY_SIZE(levels) = 1 + AND levels [0] :: STRING = 'ORIGIN' THEN NULL + WHEN ARRAY_SIZE(levels) = 1 THEN 'ORIGIN' + ELSE ARRAY_TO_STRING(ARRAY_SLICE(levels, 0, ARRAY_SIZE(levels) -1), '_')END AS parent_level, + COUNT(parent_level) over ( + PARTITION BY tx_hash, + parent_level + ) AS sub_traces,* + FROM + base_table + ), + group_sub_traces AS ( + SELECT + tx_hash, + parent_level, + sub_traces + FROM + flattened_traces + GROUP BY + tx_hash, + parent_level, + sub_traces + ), + FINAL AS ( + SELECT + flattened_traces.tx_hash AS tx_hash, + flattened_traces.block_number AS block_number, + flattened_traces.block_timestamp AS block_timestamp, + flattened_traces.from_address AS from_address, + flattened_traces.to_address AS to_address, + flattened_traces.eth_value AS eth_value, + flattened_traces.gas AS gas, + flattened_traces.gas_used AS gas_used, + flattened_traces.input AS input, + flattened_traces.output AS output, + flattened_traces.type AS TYPE, + flattened_traces.identifier AS identifier, + flattened_traces._call_id AS _call_id, + flattened_traces.ingested_at AS ingested_at, + flattened_traces._inserted_timestamp AS _inserted_timestamp, + flattened_traces.data AS DATA, + flattened_traces.tx_status AS tx_status, + group_sub_traces.sub_traces AS sub_traces + FROM + flattened_traces + LEFT OUTER JOIN group_sub_traces + ON flattened_traces.tx_hash = group_sub_traces.tx_hash + AND flattened_traces.level = group_sub_traces.parent_level + ) + SELECT + tx_hash, + block_number, + block_timestamp, + from_address, + to_address, + eth_value, + gas, + gas_used, + input, + output, + TYPE, + identifier, + _call_id, + ingested_at, + _inserted_timestamp, + DATA, + tx_status, + sub_traces + FROM + FINAL + WHERE + identifier IS NOT NULL qualify(ROW_NUMBER() over(PARTITION BY _call_id + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/silver__traces.yml b/models/silver/silver__traces.yml new file mode 100644 index 0000000..2d2dff1 --- /dev/null +++ b/models/silver/silver__traces.yml @@ -0,0 +1,54 @@ +version: 2 +models: + - name: silver__traces + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _CALL_ID + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FROM_ADDRESS + tests: + - not_null: + where: TYPE <> 'SELFDESTRUCT' + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: IDENTIFIER + tests: + - not_null + - name: ETH_VALUE + tests: + - not_null + - name: GAS + tests: + - not_null + - name: GAS_USED + tests: + - not_null + + diff --git a/models/silver/silver__transactions.sql b/models/silver/silver__transactions.sql new file mode 100644 index 0000000..3a698e2 --- /dev/null +++ b/models/silver/silver__transactions.sql @@ -0,0 +1,110 @@ +{{ config( + materialized = 'incremental', + unique_key = "tx_hash", + cluster_by = ['ingested_at::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION" +) }} + +WITH base_table AS ( + + SELECT + block_timestamp, + block_id :: INTEGER AS block_number, + tx_id :: STRING AS tx_hash, + udf_hex_to_int( + tx :nonce :: STRING + ) :: INTEGER AS nonce, + tx_block_index :: INTEGER AS POSITION, + tx :from :: STRING AS from_address, + tx :to :: STRING AS to_address, + ( + udf_hex_to_int( + tx :value :: STRING + ) / pow( + 10, + 18 + ) + ) :: FLOAT AS eth_value, + tx :blockHash :: STRING AS block_hash, + ( + udf_hex_to_int( + tx :gasPrice :: STRING + ) / pow( + 10, + 9 + ) + ) :: FLOAT AS gas_price, + udf_hex_to_int( + tx :gas :: STRING + ) :: INTEGER AS gas_limit, + tx :input :: STRING AS DATA, + CASE + WHEN tx :receipt :status :: STRING = '0x1' THEN 'SUCCESS' + ELSE 'FAIL' + END AS status, + udf_hex_to_int( + tx :receipt :gasUsed :: STRING + ) :: INTEGER AS gas_used, + udf_hex_to_int( + tx :receipt :cumulativeGasUsed :: STRING + ) :: INTEGER AS cumulative_Gas_Used, + udf_hex_to_int( + tx :receipt :effectiveGasPrice :: STRING + ) :: INTEGER AS effective_Gas_Price, + ( + gas_price * gas_used + ) / pow( + 10, + 9 + ) AS tx_fee, + ingested_at :: TIMESTAMP AS ingested_at, + _inserted_timestamp :: TIMESTAMP AS _inserted_timestamp, + OBJECT_DELETE( + tx, + 'traces' + ) AS tx_json + FROM + {{ ref('bronze__transactions') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + block_timestamp, + block_number, + tx_hash, + nonce, + POSITION, + SUBSTR( + DATA, + 1, + 10 + ) AS origin_function_signature, + from_address, + to_address, + eth_value, + block_hash, + gas_price, + gas_limit, + DATA AS input_data, + status, + gas_used, + cumulative_Gas_Used, + effective_Gas_Price, + tx_fee, + ingested_at, + _inserted_timestamp, + tx_json +FROM + base_table qualify(ROW_NUMBER() over(PARTITION BY tx_hash +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml new file mode 100644 index 0000000..4fc84b1 --- /dev/null +++ b/models/silver/silver__transactions.yml @@ -0,0 +1,124 @@ +version: 2 +models: + - name: silver__transactions + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - sequence_gaps: + partition_by: + - BLOCK_NUMBER + column_name: POSITION + where: BLOCK_TIMESTAMP < CURRENT_DATE + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: NONCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: POSITION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: FROM_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: TO_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + where: TO_ADDRESS IS NOT NULL + - name: ETH_VALUE + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: GAS_PRICE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_LIMIT + tests: + - not_null + - name: INPUT_DATA + tests: + - not_null + - name: STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: ['SUCCESS', 'FAIL'] + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CUMULATIVE_GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_FEE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EFFECTIVE_GAS_PRICE + tests: + - not_null + - name: TX_JSON + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - OBJECT + - name: INGESTED_AT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: ORIGIN_FUNCTION_SIGNATURE + tests: + - not_null + + diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 0000000..48d0cff --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,15 @@ +version: 2 + +sources: + - name: prod + database: chainwalkers + schema: prod + tables: + - name: avalanche_blocks + - name: avalanche_txs + - name: crosschain + database: flipside_prod_db + schema: crosschain + tables: + - name: address_labels + \ No newline at end of file diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/optimism/silver__transactions__tx-gap.sql b/tests/optimism/silver__transactions__tx-gap.sql new file mode 100644 index 0000000..7f84b8f --- /dev/null +++ b/tests/optimism/silver__transactions__tx-gap.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('silver__blocks') }} +{{ tx_gaps(ref("silver__transactions")) }}