From 2e5ed627129a6a038bf0f43f7fe376a156d4a1f8 Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Fri, 28 Feb 2025 15:32:01 -0500 Subject: [PATCH] PR1 --- .github/workflows/dbt_docs_update.yml | 71 +++ .github/workflows/dbt_run_adhoc.yml | 67 ++ .github/workflows/dbt_run_dev_refresh.yml | 44 ++ .../workflows/dbt_run_incremental_core.yml | 45 ++ .gitignore | 20 + README.md | 74 +++ dbt_project.yml | 73 +++ docs/.gitkeep | 0 macros/.gitkeep | 0 macros/create_sps.sql | 7 + macros/create_udfs.sql | 2 + macros/custom_naming_macros.sql | 33 + macros/custom_query_tag.sql | 11 + macros/dbt/get_merge.sql | 30 + macros/dbt/get_tmp_relation_type.sql | 4 + macros/run_sp_create_prod_clone.sql | 10 + macros/sp_create_prod_clone.sql | 49 ++ macros/tags/add_database_or_schema_tags.sql | 6 + macros/tags/snowflake_tagging.sql | 127 ++++ macros/tests/compare_model_subset.sql | 29 + macros/tests/sequence_gaps.sql | 37 ++ maya mid.csv | 428 +++++++++++++ models/bronze/bronze__active_vault_events.sql | 17 + models/bronze/bronze__add_events.sql | 25 + models/bronze/bronze__block_log.sql | 18 + models/bronze/bronze__block_pool_depths.sql | 19 + models/bronze/bronze__bond_events.sql | 25 + models/bronze/bronze__cacao_price.sql | 16 + models/bronze/bronze__constants.sql | 16 + .../bronze__failed_deposit_messages.sql | 23 + models/bronze/bronze__fee_events.sql | 20 + models/bronze/bronze__gas_events.sql | 20 + .../bronze/bronze__inactive_vault_events.sql | 17 + .../bronze/bronze__mayaname_change_events.sql | 30 + models/bronze/bronze__new_node_events.sql | 17 + models/bronze/bronze__outbound_events.sql | 25 + .../bronze__pending_liquidity_events.sql | 25 + .../bronze__pool_balance_change_events.sql | 22 + models/bronze/bronze__pool_events.sql | 18 + models/bronze/bronze__refund_events.sql | 27 + models/bronze/bronze__reserve_events.sql | 25 + .../bronze/bronze__rewards_event_entries.sql | 19 + models/bronze/bronze__rewards_events.sql | 17 + models/bronze/bronze__send_messages.sql | 24 + .../bronze/bronze__set_ip_address_events.sql | 18 + models/bronze/bronze__set_mimir_events.sql | 18 + .../bronze/bronze__set_node_keys_events.sql | 20 + .../bronze/bronze__set_node_mimir_events.sql | 19 + models/bronze/bronze__set_version_events.sql | 18 + models/bronze/bronze__slash_events.sql | 19 + .../bronze/bronze__slash_liquidity_events.sql | 22 + models/bronze/bronze__slash_points_events.sql | 19 + models/bronze/bronze__stake_events.sql | 27 + ...bronze__streamling_swap_details_events.sql | 29 + models/bronze/bronze__swap_events.sql | 34 + models/bronze/bronze__transfer_events.sql | 20 + ...nze__update_node_account_status_events.sql | 19 + ...bronze__validator_request_leave_events.sql | 19 + models/bronze/bronze__withdraw_events.sql | 31 + models/silver/silver__active_vault_events.sql | 22 + models/silver/silver__active_vault_events.yml | 16 + models/silver/silver__add_events.sql | 26 + models/silver/silver__add_events.yml | 38 ++ models/silver/silver__block_log.sql | 43 ++ models/silver/silver__block_log.yml | 20 + models/silver/silver__block_pool_depths.sql | 20 + models/silver/silver__block_pool_depths.yml | 24 + models/silver/silver__block_rewards.sql | 160 +++++ models/silver/silver__block_rewards.yml | 30 + models/silver/silver__bond_events.sql | 26 + models/silver/silver__bond_events.yml | 19 + models/silver/silver__cacao_price.sql | 19 + models/silver/silver__cacao_price.yml | 10 + models/silver/silver__constants.sql | 9 + models/silver/silver__daily_earnings.sql | 109 ++++ models/silver/silver__daily_earnings.yml | 45 ++ models/silver/silver__daily_pool_stats.sql | 177 ++++++ models/silver/silver__daily_pool_stats.yml | 126 ++++ models/silver/silver__daily_tvl.sql | 72 +++ models/silver/silver__daily_tvl.yml | 32 + .../silver__failed_deposit_messages.sql | 26 + .../silver__failed_deposit_messages.yml | 29 + models/silver/silver__fee_events.sql | 21 + models/silver/silver__fee_events.yml | 19 + models/silver/silver__gas_events.sql | 21 + models/silver/silver__gas_events.yml | 25 + .../silver/silver__inactive_vault_events.sql | 18 + models/silver/silver__liquidity_actions.sql | 180 ++++++ models/silver/silver__liquidity_actions.yml | 33 + .../silver/silver__mayaname_change_events.sql | 31 + models/silver/silver__new_node_events.sql | 18 + models/silver/silver__outbound_events.sql | 26 + models/silver/silver__outbound_events.yml | 40 ++ .../silver__pending_liquidity_events.sql | 26 + .../silver__pending_liquidity_events.yml | 28 + .../silver__pool_balance_change_events.sql | 23 + .../silver__pool_balance_change_events.yml | 26 + models/silver/silver__pool_block_balances.sql | 45 ++ models/silver/silver__pool_block_balances.yml | 40 ++ models/silver/silver__pool_block_fees.sql | 200 ++++++ models/silver/silver__pool_block_fees.yml | 33 + .../silver/silver__pool_block_statistics.sql | 586 ++++++++++++++++++ .../silver/silver__pool_block_statistics.yml | 125 ++++ models/silver/silver__pool_events.sql | 19 + models/silver/silver__prices.sql | 88 +++ models/silver/silver__prices.yml | 33 + models/silver/silver__refund_events.sql | 28 + models/silver/silver__refund_events.yml | 48 ++ models/silver/silver__reserve_events.sql | 26 + .../silver/silver__rewards_event_entries.sql | 20 + .../silver/silver__rewards_event_entries.yml | 19 + models/silver/silver__rewards_events.sql | 18 + models/silver/silver__rewards_events.yml | 14 + models/silver/silver__send_messages.sql | 25 + models/silver/silver__send_messages.yml | 30 + .../silver/silver__set_ip_address_events.sql | 19 + models/silver/silver__set_mimir_events.sql | 19 + .../silver/silver__set_node_keys_events.sql | 21 + .../silver/silver__set_node_mimir_events.sql | 20 + models/silver/silver__set_version_events.sql | 19 + models/silver/silver__slash_events.sql | 20 + .../silver/silver__slash_liquidity_events.sql | 23 + models/silver/silver__slash_points_events.sql | 20 + models/silver/silver__stake_events.sql | 28 + models/silver/silver__stake_events.yml | 34 + ...silver__streamling_swap_details_events.sql | 30 + ...silver__streamling_swap_details_events.yml | 49 ++ models/silver/silver__swap_events.sql | 35 ++ models/silver/silver__swap_events.yml | 72 +++ models/silver/silver__swaps.sql | 165 +++++ models/silver/silver__swaps.yml | 80 +++ models/silver/silver__total_block_rewards.sql | 137 ++++ models/silver/silver__total_block_rewards.yml | 28 + models/silver/silver__total_value_locked.sql | 122 ++++ models/silver/silver__total_value_locked.yml | 23 + models/silver/silver__transfer_events.sql | 21 + models/silver/silver__transfer_events.yml | 19 + models/silver/silver__transfers.sql | 54 ++ models/silver/silver__transfers.yml | 31 + ...ver__update_node_account_status_events.sql | 20 + ...ver__update_node_account_status_events.yml | 24 + ...silver__validator_request_leave_events.sql | 20 + ...silver__validator_request_leave_events.yml | 27 + models/silver/silver__withdraw_events.sql | 34 + models/silver/silver__withdraw_events.yml | 64 ++ models/sources.yml | 57 ++ package-lock.yml | 16 + packages.yml | 11 + profiles.yml | 29 + requirements.txt | 3 + scripts/update_bronze_models.py | 0 151 files changed, 6388 insertions(+) create mode 100644 .github/workflows/dbt_docs_update.yml create mode 100644 .github/workflows/dbt_run_adhoc.yml create mode 100644 .github/workflows/dbt_run_dev_refresh.yml create mode 100644 .github/workflows/dbt_run_incremental_core.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 dbt_project.yml create mode 100644 docs/.gitkeep create mode 100644 macros/.gitkeep create mode 100644 macros/create_sps.sql create mode 100644 macros/create_udfs.sql create mode 100644 macros/custom_naming_macros.sql create mode 100644 macros/custom_query_tag.sql create mode 100644 macros/dbt/get_merge.sql create mode 100644 macros/dbt/get_tmp_relation_type.sql create mode 100644 macros/run_sp_create_prod_clone.sql create mode 100644 macros/sp_create_prod_clone.sql create mode 100644 macros/tags/add_database_or_schema_tags.sql create mode 100644 macros/tags/snowflake_tagging.sql create mode 100644 macros/tests/compare_model_subset.sql create mode 100644 macros/tests/sequence_gaps.sql create mode 100644 maya mid.csv create mode 100644 models/bronze/bronze__active_vault_events.sql create mode 100644 models/bronze/bronze__add_events.sql create mode 100644 models/bronze/bronze__block_log.sql create mode 100644 models/bronze/bronze__block_pool_depths.sql create mode 100644 models/bronze/bronze__bond_events.sql create mode 100644 models/bronze/bronze__cacao_price.sql create mode 100644 models/bronze/bronze__constants.sql create mode 100644 models/bronze/bronze__failed_deposit_messages.sql create mode 100644 models/bronze/bronze__fee_events.sql create mode 100644 models/bronze/bronze__gas_events.sql create mode 100644 models/bronze/bronze__inactive_vault_events.sql create mode 100644 models/bronze/bronze__mayaname_change_events.sql create mode 100644 models/bronze/bronze__new_node_events.sql create mode 100644 models/bronze/bronze__outbound_events.sql create mode 100644 models/bronze/bronze__pending_liquidity_events.sql create mode 100644 models/bronze/bronze__pool_balance_change_events.sql create mode 100644 models/bronze/bronze__pool_events.sql create mode 100644 models/bronze/bronze__refund_events.sql create mode 100644 models/bronze/bronze__reserve_events.sql create mode 100644 models/bronze/bronze__rewards_event_entries.sql create mode 100644 models/bronze/bronze__rewards_events.sql create mode 100644 models/bronze/bronze__send_messages.sql create mode 100644 models/bronze/bronze__set_ip_address_events.sql create mode 100644 models/bronze/bronze__set_mimir_events.sql create mode 100644 models/bronze/bronze__set_node_keys_events.sql create mode 100644 models/bronze/bronze__set_node_mimir_events.sql create mode 100644 models/bronze/bronze__set_version_events.sql create mode 100644 models/bronze/bronze__slash_events.sql create mode 100644 models/bronze/bronze__slash_liquidity_events.sql create mode 100644 models/bronze/bronze__slash_points_events.sql create mode 100644 models/bronze/bronze__stake_events.sql create mode 100644 models/bronze/bronze__streamling_swap_details_events.sql create mode 100644 models/bronze/bronze__swap_events.sql create mode 100644 models/bronze/bronze__transfer_events.sql create mode 100644 models/bronze/bronze__update_node_account_status_events.sql create mode 100644 models/bronze/bronze__validator_request_leave_events.sql create mode 100644 models/bronze/bronze__withdraw_events.sql create mode 100644 models/silver/silver__active_vault_events.sql create mode 100644 models/silver/silver__active_vault_events.yml create mode 100644 models/silver/silver__add_events.sql create mode 100644 models/silver/silver__add_events.yml create mode 100644 models/silver/silver__block_log.sql create mode 100644 models/silver/silver__block_log.yml create mode 100644 models/silver/silver__block_pool_depths.sql create mode 100644 models/silver/silver__block_pool_depths.yml create mode 100644 models/silver/silver__block_rewards.sql create mode 100644 models/silver/silver__block_rewards.yml create mode 100644 models/silver/silver__bond_events.sql create mode 100644 models/silver/silver__bond_events.yml create mode 100644 models/silver/silver__cacao_price.sql create mode 100644 models/silver/silver__cacao_price.yml create mode 100644 models/silver/silver__constants.sql create mode 100644 models/silver/silver__daily_earnings.sql create mode 100644 models/silver/silver__daily_earnings.yml create mode 100644 models/silver/silver__daily_pool_stats.sql create mode 100644 models/silver/silver__daily_pool_stats.yml create mode 100644 models/silver/silver__daily_tvl.sql create mode 100644 models/silver/silver__daily_tvl.yml create mode 100644 models/silver/silver__failed_deposit_messages.sql create mode 100644 models/silver/silver__failed_deposit_messages.yml create mode 100644 models/silver/silver__fee_events.sql create mode 100644 models/silver/silver__fee_events.yml create mode 100644 models/silver/silver__gas_events.sql create mode 100644 models/silver/silver__gas_events.yml create mode 100644 models/silver/silver__inactive_vault_events.sql create mode 100644 models/silver/silver__liquidity_actions.sql create mode 100644 models/silver/silver__liquidity_actions.yml create mode 100644 models/silver/silver__mayaname_change_events.sql create mode 100644 models/silver/silver__new_node_events.sql create mode 100644 models/silver/silver__outbound_events.sql create mode 100644 models/silver/silver__outbound_events.yml create mode 100644 models/silver/silver__pending_liquidity_events.sql create mode 100644 models/silver/silver__pending_liquidity_events.yml create mode 100644 models/silver/silver__pool_balance_change_events.sql create mode 100644 models/silver/silver__pool_balance_change_events.yml create mode 100644 models/silver/silver__pool_block_balances.sql create mode 100644 models/silver/silver__pool_block_balances.yml create mode 100644 models/silver/silver__pool_block_fees.sql create mode 100644 models/silver/silver__pool_block_fees.yml create mode 100644 models/silver/silver__pool_block_statistics.sql create mode 100644 models/silver/silver__pool_block_statistics.yml create mode 100644 models/silver/silver__pool_events.sql create mode 100644 models/silver/silver__prices.sql create mode 100644 models/silver/silver__prices.yml create mode 100644 models/silver/silver__refund_events.sql create mode 100644 models/silver/silver__refund_events.yml create mode 100644 models/silver/silver__reserve_events.sql create mode 100644 models/silver/silver__rewards_event_entries.sql create mode 100644 models/silver/silver__rewards_event_entries.yml create mode 100644 models/silver/silver__rewards_events.sql create mode 100644 models/silver/silver__rewards_events.yml create mode 100644 models/silver/silver__send_messages.sql create mode 100644 models/silver/silver__send_messages.yml create mode 100644 models/silver/silver__set_ip_address_events.sql create mode 100644 models/silver/silver__set_mimir_events.sql create mode 100644 models/silver/silver__set_node_keys_events.sql create mode 100644 models/silver/silver__set_node_mimir_events.sql create mode 100644 models/silver/silver__set_version_events.sql create mode 100644 models/silver/silver__slash_events.sql create mode 100644 models/silver/silver__slash_liquidity_events.sql create mode 100644 models/silver/silver__slash_points_events.sql create mode 100644 models/silver/silver__stake_events.sql create mode 100644 models/silver/silver__stake_events.yml create mode 100644 models/silver/silver__streamling_swap_details_events.sql create mode 100644 models/silver/silver__streamling_swap_details_events.yml create mode 100644 models/silver/silver__swap_events.sql create mode 100644 models/silver/silver__swap_events.yml create mode 100644 models/silver/silver__swaps.sql create mode 100644 models/silver/silver__swaps.yml create mode 100644 models/silver/silver__total_block_rewards.sql create mode 100644 models/silver/silver__total_block_rewards.yml create mode 100644 models/silver/silver__total_value_locked.sql create mode 100644 models/silver/silver__total_value_locked.yml create mode 100644 models/silver/silver__transfer_events.sql create mode 100644 models/silver/silver__transfer_events.yml create mode 100644 models/silver/silver__transfers.sql create mode 100644 models/silver/silver__transfers.yml create mode 100644 models/silver/silver__update_node_account_status_events.sql create mode 100644 models/silver/silver__update_node_account_status_events.yml create mode 100644 models/silver/silver__validator_request_leave_events.sql create mode 100644 models/silver/silver__validator_request_leave_events.yml create mode 100644 models/silver/silver__withdraw_events.sql create mode 100644 models/silver/silver__withdraw_events.yml create mode 100644 models/sources.yml create mode 100644 package-lock.yml create mode 100644 packages.yml create mode 100644 profiles.yml create mode 100644 requirements.txt create mode 100644 scripts/update_bronze_models.py diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml new file mode 100644 index 0000000..fc20ba8 --- /dev/null +++ b/.github/workflows/dbt_docs_update.yml @@ -0,0 +1,71 @@ +name: docs_update + +on: + push: + branches: + - "main" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: checkout docs branch + run: | + git checkout -B docs origin/main + - name: generate dbt docs + run: | + dbt ls -t prod + dbt docs generate --no-compile -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..435a87d --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,67 @@ +name: dbt_run_adhoc +run-name: dbt_run_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..32db163 --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,44 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + workflow_dispatch: +# schedule: +# - cron: '27 8 * * *' + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_incremental_core.yml b/.github/workflows/dbt_run_incremental_core.yml new file mode 100644 index 0000000..b278b82 --- /dev/null +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -0,0 +1,45 @@ +name: dbt_run_incremental_core +run-name: dbt_run_incremental_core + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -s "maya_models,tag:scheduled_core" + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2769f0f --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +target/ +dbt_modules/ +# newer versions of dbt use this directory instead of dbt_modules for test dependencies +dbt_packages/ +logs/ + +.venv/ +.python-version +dbt-env/ +venv/ + +# Visual Studio Code files +*/.vscode +*.code-workspace +.history/ +**/.DS_Store +.vscode/ +.env +.DS_Store +.user.yml \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..00ae875 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +## Profile Set Up + +#### Use the following within profiles.yml +---- + +```yml +maya: + target: dev + outputs: + dev: + type: snowflake + account: + role: + user: + password: + region: + database: maya_DEV + warehouse: + schema: silver + threads: 4 + client_session_keep_alive: False + query_tag: +``` + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +## Applying Model Tags + +### Database / Schema level tags + +Database and schema tags are applied via the `add_database_or_schema_tags` macro. These tags are inherited by their downstream objects. To add/modify tags call the appropriate tag set function within the macro. + +``` +{{ set_database_tag_value('SOME_DATABASE_TAG_KEY','SOME_DATABASE_TAG_VALUE') }} +{{ set_schema_tag_value('SOME_SCHEMA_TAG_KEY','SOME_SCHEMA_TAG_VALUE') }} +``` + +### Model tags + +To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT. + +``` +{{ config( + ..., + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'SOME_PURPOSE' + } + } + }, + ... +) }} +``` + +By default, model tags are not pushed to snowflake on each load. You can push a tag update for a model by specifying the `UPDATE_SNOWFLAKE_TAGS` project variable during a run. + +``` +dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.sql +``` + +### Querying for existing tags on a model in snowflake + +``` +select * +from table(maya.information_schema.tag_references('maya.core.fact_blocks', 'table')); +``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..1ff63ba --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,73 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "maya_models" +version: "1.0.0" +config-version: 2 + +require-dbt-version: ">=1.8.0" + +# This setting configures which "profile" dbt uses for this project. +profile: "maya" + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + - "dbt_packages" + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. +models: + +copy_grants: true + +persist_docs: + relation: true + columns: true + +on_schema_change: "append_new_columns" + livequery_models: + deploy: + core: + materialized: ephemeral + + +data_tests: + maya_models: # replace with the name of the chain + +store_failures: true # all tests + +on-run-start: + - '{{create_sps()}}' + - '{{create_udfs()}}' + + +dispatch: + - macro_namespace: dbt + search_order: + - maya-models + - dbt_snowflake_query_tags + - dbt + +query-comment: + comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' + append: true # Snowflake removes prefixed comments. + +vars: + "dbt_date:time_zone": GMT + OBSERV_FULL_TEST: False + START_GHA_TASKS: False + UPDATE_SNOWFLAKE_TAGS: True + UPDATE_UDFS_AND_SPS: True + diff --git a/docs/.gitkeep b/docs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/create_sps.sql b/macros/create_sps.sql new file mode 100644 index 0000000..3bdff2e --- /dev/null +++ b/macros/create_sps.sql @@ -0,0 +1,7 @@ +{% macro create_sps() %} + {% if target.database == 'MAYA' %} + CREATE schema IF NOT EXISTS _internal; +{{ sp_create_prod_clone('_internal') }}; + {% endif %} +{% endmacro %} +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql new file mode 100644 index 0000000..56fe0ff --- /dev/null +++ b/macros/create_udfs.sql @@ -0,0 +1,2 @@ +{% macro create_udfs() %} +{% endmacro %} diff --git a/macros/custom_naming_macros.sql b/macros/custom_naming_macros.sql new file mode 100644 index 0000000..53fe8c7 --- /dev/null +++ b/macros/custom_naming_macros.sql @@ -0,0 +1,33 @@ +{% macro generate_schema_name( + custom_schema_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name [0] | trim }} +{%- endmacro %} + +{% macro generate_alias_name( + custom_alias_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {% if split_name | length < 2 %} + {{ split_name [0] | trim }} + {% else %} + {{ split_name [1] | trim }} + {% endif %} +{%- endmacro %} + +{% macro generate_tmp_view_name(model_name) -%} + {% set node_name = model_name.name %} + {% set split_name = node_name.split('__') %} + {{ target.database ~ '.' ~ split_name[0] ~ '.' ~ split_name [1] ~ '__dbt_tmp' | trim }} +{%- endmacro %} + +{% macro generate_view_name(model_name) -%} + {% set node_name = model_name.name %} + {% set split_name = node_name.split('__') %} + {{ target.database ~ '.' ~ split_name[0] ~ '.' ~ split_name [1] | trim }} +{%- endmacro %} \ No newline at end of file diff --git a/macros/custom_query_tag.sql b/macros/custom_query_tag.sql new file mode 100644 index 0000000..809e1bb --- /dev/null +++ b/macros/custom_query_tag.sql @@ -0,0 +1,11 @@ +{% macro set_query_tag() -%} + {% set new_json = {"repo":project_name, "object":this.table, "profile":target.profile_name, "env":target.name, "existing_tag":get_current_query_tag() } %} +{% set new_query_tag = tojson(new_json) | as_text %} + {% if new_query_tag %} + {% set original_query_tag = get_current_query_tag() %} + {{ log("Setting query_tag to '" ~ new_query_tag ~ "'. Will reset to '" ~ original_query_tag ~ "' after materialization.") }} + {% do run_query("alter session set query_tag = '{}'".format(new_query_tag)) %} + {{ return(original_query_tag)}} + {% endif %} + {{ return(none)}} +{% endmacro %} \ No newline at end of file diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql new file mode 100644 index 0000000..7563d88 --- /dev/null +++ b/macros/dbt/get_merge.sql @@ -0,0 +1,30 @@ +-- incremental_strategy="merge" +{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {{ return(merge_sql) }} +{% endmacro %} + +-- incremental_strategy="delete+insert" +{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {% set predicate_override = "" %} + -- get the min value of column + {% if incremental_predicates[0] == "min_value_predicate" %} + {% set min_column_name = incremental_predicates[1] %} + {% set query %} + select min({{ min_column_name }}) from {{ source }} + {% endset %} + {% set min_block = run_query(query).columns[0][0] %} + + {% if min_block is not none %} + {% set predicate_override %} + round({{ target }}.{{ min_column_name }},-5) >= round({{ min_block }},-5) + {% endset %} + {% else %} + {% set predicate_override = "1=1" %} + {% endif %} + {% endif %} + {% set predicates = [predicate_override] + incremental_predicates[2:] if predicate_override else incremental_predicates %} + -- standard delete+insert from here + {% set merge_sql = dbt.get_delete_insert_merge_sql(target, source, unique_key, dest_columns, predicates) %} + {{ return(merge_sql) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/dbt/get_tmp_relation_type.sql b/macros/dbt/get_tmp_relation_type.sql new file mode 100644 index 0000000..e7c2d77 --- /dev/null +++ b/macros/dbt/get_tmp_relation_type.sql @@ -0,0 +1,4 @@ +{% macro dbt_snowflake_get_tmp_relation_type(strategy, unique_key, language) %} + -- always table + {{ return('table') }} +{% endmacro %} \ No newline at end of file diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql new file mode 100644 index 0000000..dff3c11 --- /dev/null +++ b/macros/run_sp_create_prod_clone.sql @@ -0,0 +1,10 @@ +{% macro run_sp_create_prod_clone() %} + {% set clone_query %} + call maya._internal.create_prod_clone( + 'maya', + 'maya_dev', + 'internal_dev' + ); +{% endset %} + {% do run_query(clone_query) %} +{% endmacro %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql new file mode 100644 index 0000000..b29c4d7 --- /dev/null +++ b/macros/sp_create_prod_clone.sql @@ -0,0 +1,49 @@ +{% macro sp_create_prod_clone(target_schema) -%} + +create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string) +returns boolean +language javascript +execute as caller +as +$$ + snowflake.execute({sqlText: `BEGIN TRANSACTION;`}); + try { + snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); + snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}) + + var existing_tags = snowflake.execute({sqlText: `SHOW TAGS IN DATABASE ${DESTINATION_DB_NAME};`}); + while (existing_tags.next()) { + var schema = existing_tags.getColumnValue(4); + var tag_name = existing_tags.getColumnValue(2) + snowflake.execute({sqlText: `GRANT OWNERSHIP ON TAG ${DESTINATION_DB_NAME}.${schema}.${tag_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + } + + snowflake.execute({sqlText: `COMMIT;`}); + } catch (err) { + snowflake.execute({sqlText: `ROLLBACK;`}); + throw(err); + } + + return true +$$ + +{%- endmacro %} \ No newline at end of file diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql new file mode 100644 index 0000000..fcff3d7 --- /dev/null +++ b/macros/tags/add_database_or_schema_tags.sql @@ -0,0 +1,6 @@ +{% macro add_database_or_schema_tags() %} + {{ set_database_tag_value( + 'BLOCKCHAIN_NAME', + 'MAYA' + ) }} +{% endmacro %} diff --git a/macros/tags/snowflake_tagging.sql b/macros/tags/snowflake_tagging.sql new file mode 100644 index 0000000..bc25e69 --- /dev/null +++ b/macros/tags/snowflake_tagging.sql @@ -0,0 +1,127 @@ +{% macro apply_meta_as_tags(results) %} + {% if var("UPDATE_SNOWFLAKE_TAGS") %} + {{ log('apply_meta_as_tags', info=False) }} + {{ log(results, info=False) }} + {% if execute %} + + {%- set tags_by_schema = {} -%} + {% for res in results -%} + {% if res.node.meta.database_tags %} + + {%- set model_database = res.node.database -%} + {%- set model_schema = res.node.schema -%} + {%- set model_schema_full = model_database+'.'+model_schema -%} + {%- set model_alias = res.node.alias -%} + + {% if model_schema_full not in tags_by_schema.keys() %} + {{ log('need to fetch tags for schema '+model_schema_full, info=False) }} + {%- call statement('main', fetch_result=True) -%} + show tags in {{model_database}}.{{model_schema}} + {%- endcall -%} + {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%} + {{ log('Added tags to cache', info=False) }} + {% else %} + {{ log('already have tag info for schema', info=False) }} + {% endif %} + + {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%} + {{ log('current_tags_in_schema:', info=False) }} + {{ log(current_tags_in_schema, info=False) }} + {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }} + + {% set line -%} + node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }}) + node full: {{ res.node}} + meta: {{ res.node.meta}} + materialized: {{ res.node.config.materialized }} + {%- endset %} + {{ log(line, info=False) }} + + {%- call statement('main', fetch_result=True) -%} + select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table')) + {%- endcall -%} + {%- set existing_tags_for_table = load_result('main')['data'] -%} + {{ log('Existing tags for table:', info=False) }} + {{ log(existing_tags_for_table, info=False) }} + + {{ log('--', info=False) }} + {% for table_tag in res.node.meta.database_tags.table %} + + {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }} + {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %} + + {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}} + {% endfor %} + {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +{% endmacro %} + + +{% macro create_tag_if_missing(all_tag_names,table_tag) %} + {% if table_tag not in all_tag_names %} + {{ log('Creating missing tag '+table_tag, info=False) }} + {%- call statement('main', fetch_result=True) -%} + create tag if not exists silver.{{table_tag}} + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% else %} + {{ log('Tag already exists: '+table_tag, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }} + {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for table:', info=False) }} + {{ log(existing_tag_for_table[0], info=False) }} + {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }} + {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for column:', info=False) }} + {{ log(existing_tag_for_column[0], info=False) }} + {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_database_tag_value(tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} + +{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} \ No newline at end of file diff --git a/macros/tests/compare_model_subset.sql b/macros/tests/compare_model_subset.sql new file mode 100644 index 0000000..18aa624 --- /dev/null +++ b/macros/tests/compare_model_subset.sql @@ -0,0 +1,29 @@ +{% test compare_model_subset(model, compare_model, compare_columns, model_condition) %} + +{% set compare_cols_csv = compare_columns | join(', ') %} + +with a as ( + select {{compare_cols_csv}} from {{ model }} + {{ model_condition }} +), +b as ( + select {{compare_cols_csv}} from {{ compare_model }} +), +a_minus_b as ( + select * from a + except + select * from b +), +b_minus_a as ( + select * from b + except + select * from a +), +unioned as ( + select 'in_actual_not_in_expected' as which_diff, a_minus_b.* from a_minus_b + union all + select 'in_expected_not_in_actual' as which_diff, b_minus_a.* from b_minus_a +) +select * from unioned + +{% endtest %} \ No newline at end of file diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql new file mode 100644 index 0000000..84a9aa9 --- /dev/null +++ b/macros/tests/sequence_gaps.sql @@ -0,0 +1,37 @@ +{% macro sequence_gaps( + table, + partition_by, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + {{ table }} + WHERE + block_timestamp::date <= current_date - 1 + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/maya mid.csv b/maya mid.csv new file mode 100644 index 0000000..c338a6f --- /dev/null +++ b/maya mid.csv @@ -0,0 +1,428 @@ +TABLE_NAME,COLUMN_NAME +MIDGARD_BLOCK_LOG,__HEVO__DATABASE_NAME +MIDGARD_STAKE_EVENTS,CACAO_ADDR +MIDGARD_FEE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_BLOCK_POOL_DEPTHS,BLOCK_TIMESTAMP +MIDGARD_REFUND_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SLASH_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_CACAO_PRICE,CACAO_PRICE_E8 +MIDGARD_ACTIVE_VAULT_EVENTS,EVENT_ID +MIDGARD_SLASH_EVENTS,ASSET_E8 +MIDGARD_REFUND_EVENTS,ASSET +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SLASH_POINTS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,FROM_ADDR +MIDGARD_OUTBOUND_EVENTS,__HEVO__LOADED_AT +MIDGARD_REWARDS_EVENT_ENTRIES,SAVER_E8 +MIDGARD_BOND_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,LAST_HEIGHT +MIDGARD_OUTBOUND_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SET_MIMIR_EVENTS,__HEVO__INGESTED_AT +MIDGARD_FAILED_DEPOSIT_MESSAGES,__HEVO__SCHEMA_NAME +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,FAILED_SWAP_REASONS +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,EVENT_ID +MIDGARD_PENDING_LIQUIDITY_EVENTS,ASSET_E8 +MIDGARD_CONSTANTS,__HEVO__INGESTED_AT +MIDGARD_SET_NODE_KEYS_EVENTS,EVENT_ID +MIDGARD_SET_IP_ADDRESS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SLASH_POINTS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_OUTBOUND_EVENTS,IN_TX +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,EVENT_ID +MIDGARD_SWAP_EVENTS,EVENT_ID +MIDGARD_SET_NODE_MIMIR_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SET_NODE_KEYS_EVENTS,NODE_ADDR +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_WITHDRAW_EVENTS,FROM_ADDR +MIDGARD_SET_IP_ADDRESS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_SET_VERSION_EVENTS,VERSION +MIDGARD_REFUND_EVENTS,TO_ADDR +MIDGARD_STAKE_EVENTS,__HEVO__LOADED_AT +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,EVENT_ID +MIDGARD_BLOCK_POOL_DEPTHS,POOL +MIDGARD_WITHDRAW_EVENTS,_EMIT_ASSET_IN_CACAO_E8 +MIDGARD_TRANSFER_EVENTS,BLOCK_TIMESTAMP +MIDGARD_CACAO_PRICE,__HEVO__SCHEMA_NAME +MIDGARD_CACAO_PRICE,__HEVO__INGESTED_AT +MIDGARD_SET_MIMIR_EVENTS,__HEVO__LOADED_AT +MIDGARD_RESERVE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_CONSTANTS,__HEVO__SCHEMA_NAME +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,TX_ID +MIDGARD_INACTIVE_VAULT_EVENTS,ADD_ASGARD_ADDR +MIDGARD_CACAO_PRICE,BLOCK_TIMESTAMP +MIDGARD_NEW_NODE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_BLOCK_LOG,__HEVO__INGESTED_AT +MIDGARD_BOND_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,COUNT +MIDGARD_BOND_EVENTS,E8 +MIDGARD_RESERVE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SET_NODE_KEYS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_GAS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SWAP_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_MAYANAME_CHANGE_EVENTS,CHAIN +MIDGARD_ADD_EVENTS,CACAO_E8 +MIDGARD_ACTIVE_VAULT_EVENTS,BLOCK_TIMESTAMP +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SWAP_EVENTS,TO_E8 +MIDGARD_MAYANAME_CHANGE_EVENTS,AFFILIATE_BPS +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_CONSTANTS,VALUE +MIDGARD_MAYANAME_CHANGE_EVENTS,NAME +MIDGARD_BLOCK_LOG,__HEVO__LOADED_AT +MIDGARD_BLOCK_POOL_DEPTHS,__HEVO__LOADED_AT +MIDGARD_REWARDS_EVENT_ENTRIES,__HEVO__DATABASE_NAME +MIDGARD_ADD_EVENTS,ASSET_E8 +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,QUANTITY +MIDGARD_MAYANAME_CHANGE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_STAKE_EVENTS,POOL +MIDGARD_MAYANAME_CHANGE_EVENTS,OWNER +MIDGARD_REWARDS_EVENT_ENTRIES,__HEVO__SCHEMA_NAME +MIDGARD_SLASH_EVENTS,BLOCK_TIMESTAMP +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_WITHDRAW_EVENTS,POOL +MIDGARD_STAKE_EVENTS,ASSET_TX +MIDGARD_SWAP_EVENTS,LIQ_FEE_IN_CACAO_E8 +MIDGARD_BOND_EVENTS,MEMO +MIDGARD_SEND_MESSAGES,RAW_LOG +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,IN_ASSET +MIDGARD_STAKE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_BOND_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SET_NODE_KEYS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SLASH_LIQUIDITY_EVENTS,__HEVO__INGESTED_AT +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SET_VERSION_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,REASON +MIDGARD_REWARDS_EVENTS,EVENT_ID +MIDGARD_FEE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_REFUND_EVENTS,ASSET_E8 +MIDGARD_POOL_EVENTS,__HEVO__LOADED_AT +MIDGARD_REFUND_EVENTS,__HEVO__INGESTED_AT +MIDGARD_BLOCK_POOL_DEPTHS,ASSET_E8 +MIDGARD_SWAP_EVENTS,TO_E8_MIN +MIDGARD_SEND_MESSAGES,FROM_ADDR +MIDGARD_STAKE_EVENTS,STAKE_UNITS +MIDGARD_ADD_EVENTS,__HEVO__LOADED_AT +MIDGARD_OUTBOUND_EVENTS,MEMO +MIDGARD_SET_IP_ADDRESS_EVENTS,__HEVO__LOADED_AT +MIDGARD_SET_MIMIR_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SET_NODE_MIMIR_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_NEW_NODE_EVENTS,__HEVO__LOADED_AT +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,ASSET_AMT +MIDGARD_GAS_EVENTS,TX_COUNT +MIDGARD_PENDING_LIQUIDITY_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SWAP_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SET_IP_ADDRESS_EVENTS,NODE_ADDR +MIDGARD_OUTBOUND_EVENTS,TO_ADDR +MIDGARD_SLASH_POINTS_EVENTS,SLASH_POINTS +MIDGARD_ADD_EVENTS,EVENT_ID +MIDGARD_SEND_MESSAGES,TX_ID +MIDGARD_SEND_MESSAGES,MEMO +MIDGARD_FEE_EVENTS,__HEVO__LOADED_AT +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,OUT_ASSET +MIDGARD_SWAP_EVENTS,TO_ASSET +MIDGARD_NEW_NODE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_RESERVE_EVENTS,EVENT_ID +MIDGARD_WITHDRAW_EVENTS,ASYMMETRY +MIDGARD_REFUND_EVENTS,ASSET_2ND +MIDGARD_OUTBOUND_EVENTS,INTERNAL +MIDGARD_INACTIVE_VAULT_EVENTS,BLOCK_TIMESTAMP +MIDGARD_FAILED_DEPOSIT_MESSAGES,ASSET +MIDGARD_SET_IP_ADDRESS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_REFUND_EVENTS,ASSET_2ND_E8 +MIDGARD_SLASH_LIQUIDITY_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_REWARDS_EVENT_ENTRIES,__HEVO__LOADED_AT +MIDGARD_BLOCK_LOG,HEIGHT +MIDGARD_REWARDS_EVENT_ENTRIES,__HEVO__INGESTED_AT +MIDGARD_CACAO_PRICE,__HEVO__DATABASE_NAME +MIDGARD_BLOCK_POOL_DEPTHS,__HEVO__INGESTED_AT +MIDGARD_NEW_NODE_EVENTS,EVENT_ID +MIDGARD_REFUND_EVENTS,CHAIN +MIDGARD_SWAP_EVENTS,POOL +MIDGARD_OUTBOUND_EVENTS,EVENT_ID +MIDGARD_PENDING_LIQUIDITY_EVENTS,POOL +MIDGARD_STAKE_EVENTS,ASSET_E8 +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,NODE_ADDR +MIDGARD_MAYANAME_CHANGE_EVENTS,REGISTRATION_FEE_E8 +MIDGARD_SWAP_EVENTS,_STREAMING +MIDGARD_SLASH_LIQUIDITY_EVENTS,LP_ADDRESS +MIDGARD_WITHDRAW_EVENTS,IMP_LOSS_PROTECTION_E8 +MIDGARD_SWAP_EVENTS,FROM_ADDR +MIDGARD_SET_MIMIR_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SEND_MESSAGES,__HEVO__INGESTED_AT +MIDGARD_POOL_EVENTS,BLOCK_TIMESTAMP +MIDGARD_ACTIVE_VAULT_EVENTS,ADD_ASGARD_ADDR +MIDGARD_CACAO_PRICE,__HEVO__LOADED_AT +MIDGARD_GAS_EVENTS,EVENT_ID +MIDGARD_MAYANAME_CHANGE_EVENTS,SENDER +MIDGARD_SLASH_LIQUIDITY_EVENTS,EVENT_ID +MIDGARD_POOL_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SLASH_EVENTS,ASSET +MIDGARD_REWARDS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_PENDING_LIQUIDITY_EVENTS,ASSET_ADDR +MIDGARD_BOND_EVENTS,TX +MIDGARD_WITHDRAW_EVENTS,EMIT_CACAO_E8 +MIDGARD_ADD_EVENTS,FROM_ADDR +MIDGARD_MAYANAME_CHANGE_EVENTS,MEMO +MIDGARD_SEND_MESSAGES,__HEVO__SCHEMA_NAME +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_REFUND_EVENTS,TX +MIDGARD_STAKE_EVENTS,CACAO_E8 +MIDGARD_REFUND_EVENTS,__HEVO__LOADED_AT +MIDGARD_INACTIVE_VAULT_EVENTS,__HEVO__LOADED_AT +MIDGARD_ADD_EVENTS,TO_ADDR +MIDGARD_FAILED_DEPOSIT_MESSAGES,CODE +MIDGARD_CONSTANTS,KEY +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,__HEVO__LOADED_AT +MIDGARD_WITHDRAW_EVENTS,BASIS_POINTS +MIDGARD_OUTBOUND_EVENTS,TX +MIDGARD_MAYANAME_CHANGE_EVENTS,EXPIRE +MIDGARD_RESERVE_EVENTS,E8 +MIDGARD_INACTIVE_VAULT_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_OUTBOUND_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_POOL_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SLASH_LIQUIDITY_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_GAS_EVENTS,ASSET_E8 +MIDGARD_PENDING_LIQUIDITY_EVENTS,CACAO_TX +MIDGARD_ADD_EVENTS,BLOCK_TIMESTAMP +MIDGARD_WITHDRAW_EVENTS,MEMO +MIDGARD_RESERVE_EVENTS,ASSET +MIDGARD_BLOCK_LOG,__HEVO__SCHEMA_NAME +MIDGARD_FAILED_DEPOSIT_MESSAGES,TX_ID +MIDGARD_SWAP_EVENTS,LIQ_FEE_E8 +MIDGARD_MAYANAME_CHANGE_EVENTS,PREFERRED_ASSET +MIDGARD_ADD_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SET_IP_ADDRESS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_INACTIVE_VAULT_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_ADD_EVENTS,CHAIN +MIDGARD_SWAP_EVENTS,STREAMING_COUNT +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_POOL_EVENTS,ASSET +MIDGARD_SET_VERSION_EVENTS,BLOCK_TIMESTAMP +MIDGARD_RESERVE_EVENTS,CHAIN +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,CACAO_ADD +MIDGARD_REWARDS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_PENDING_LIQUIDITY_EVENTS,__HEVO__LOADED_AT +MIDGARD_TRANSFER_EVENTS,AMOUNT_E8 +MIDGARD_SWAP_EVENTS,__HEVO__INGESTED_AT +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_MAYANAME_CHANGE_EVENTS,FUND_AMOUNT_E8 +MIDGARD_FAILED_DEPOSIT_MESSAGES,__HEVO__DATABASE_NAME +MIDGARD_REWARDS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SET_NODE_MIMIR_EVENTS,KEY +MIDGARD_BOND_EVENTS,CHAIN +MIDGARD_SLASH_LIQUIDITY_EVENTS,BOND_ADDRESS +MIDGARD_ACTIVE_VAULT_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_FEE_EVENTS,ASSET_E8 +MIDGARD_TRANSFER_EVENTS,__HEVO__LOADED_AT +MIDGARD_REWARDS_EVENT_ENTRIES,CACAO_E8 +MIDGARD_BLOCK_POOL_DEPTHS,CACAO_E8 +MIDGARD_INACTIVE_VAULT_EVENTS,__HEVO__INGESTED_AT +MIDGARD_GAS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_OUTBOUND_EVENTS,ASSET_E8 +MIDGARD_REWARDS_EVENT_ENTRIES,POOL +MIDGARD_SET_NODE_KEYS_EVENTS,VALIDATOR_CONSENSUS +MIDGARD_SWAP_EVENTS,FROM_ASSET +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,NODE_ADDR +MIDGARD_BLOCK_POOL_DEPTHS,SYNTH_E8 +MIDGARD_WITHDRAW_EVENTS,TO_ADDR +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,DEPOSIT_E8 +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,FAILED_SWAPS +MIDGARD_MAYANAME_CHANGE_EVENTS,ADDRESS +MIDGARD_FAILED_DEPOSIT_MESSAGES,REASON +MIDGARD_BOND_EVENTS,BOND_TYPE +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,INTERVAL +MIDGARD_MAYANAME_CHANGE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_GAS_EVENTS,__HEVO__LOADED_AT +MIDGARD_SLASH_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SWAP_EVENTS,TO_ADDR +MIDGARD_BOND_EVENTS,__HEVO__LOADED_AT +MIDGARD_MAYANAME_CHANGE_EVENTS,EVENT_ID +MIDGARD_ADD_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SET_NODE_KEYS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_MAYANAME_CHANGE_EVENTS,HEIGHT +MIDGARD_SWAP_EVENTS,BLOCK_TIMESTAMP +MIDGARD_ACTIVE_VAULT_EVENTS,__HEVO__LOADED_AT +MIDGARD_STAKE_EVENTS,EVENT_ID +MIDGARD_RESERVE_EVENTS,FROM_ADDR +MIDGARD_TRANSFER_EVENTS,EVENT_ID +MIDGARD_REWARDS_EVENT_ENTRIES,EVENT_ID +MIDGARD_MAYANAME_CHANGE_EVENTS,SUB_AFFILIATES +MIDGARD_ADD_EVENTS,POOL +MIDGARD_SET_IP_ADDRESS_EVENTS,IP_ADDR +MIDGARD_SET_NODE_KEYS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_FAILED_DEPOSIT_MESSAGES,AMOUNT_E8 +MIDGARD_ADD_EVENTS,ASSET +MIDGARD_SWAP_EVENTS,_DIRECTION +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,IN_E8 +MIDGARD_SET_NODE_MIMIR_EVENTS,__HEVO__LOADED_AT +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,ASSET +MIDGARD_WITHDRAW_EVENTS,EVENT_ID +MIDGARD_SWAP_EVENTS,TX +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,__HEVO__LOADED_AT +MIDGARD_STAKE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_FEE_EVENTS,POOL_DEDUCT +MIDGARD_GAS_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_GAS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_PENDING_LIQUIDITY_EVENTS,EVENT_ID +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,CURRENT +MIDGARD_SET_NODE_MIMIR_EVENTS,ADDRESS +MIDGARD_SET_VERSION_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SLASH_LIQUIDITY_EVENTS,ASSET_E8_LOSS +MIDGARD_OUTBOUND_EVENTS,ASSET +MIDGARD_SLASH_EVENTS,__HEVO__INGESTED_AT +MIDGARD_ADD_EVENTS,TX +MIDGARD_REWARDS_EVENT_ENTRIES,BLOCK_TIMESTAMP +MIDGARD_SET_VERSION_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_RESERVE_EVENTS,ASSET_E8 +MIDGARD_NEW_NODE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_GAS_EVENTS,CACAO_E8 +MIDGARD_SET_IP_ADDRESS_EVENTS,EVENT_ID +MIDGARD_SLASH_POINTS_EVENTS,REASON +MIDGARD_SWAP_EVENTS,STREAMING_QUANTITY +MIDGARD_PENDING_LIQUIDITY_EVENTS,ASSET_TX +MIDGARD_FAILED_DEPOSIT_MESSAGES,__HEVO__LOADED_AT +MIDGARD_BLOCK_LOG,HASH +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,OUT_E8 +MIDGARD_SLASH_POINTS_EVENTS,NODE_ADDRESS +MIDGARD_SLASH_LIQUIDITY_EVENTS,BLOCK_TIMESTAMP +MIDGARD_NEW_NODE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_BLOCK_LOG,AGG_STATE +MIDGARD_BOND_EVENTS,FROM_ADDR +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_STAKE_EVENTS,ASSET_CHAIN +MIDGARD_FEE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_FAILED_DEPOSIT_MESSAGES,__HEVO__INGESTED_AT +MIDGARD_SET_NODE_MIMIR_EVENTS,EVENT_ID +MIDGARD_SET_MIMIR_EVENTS,KEY +MIDGARD_POOL_EVENTS,STATUS +MIDGARD_SET_MIMIR_EVENTS,BLOCK_TIMESTAMP +MIDGARD_RESERVE_EVENTS,__HEVO__LOADED_AT +MIDGARD_MAYANAME_CHANGE_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_TRANSFER_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_FEE_EVENTS,ASSET +MIDGARD_NEW_NODE_EVENTS,NODE_ADDR +MIDGARD_POOL_EVENTS,EVENT_ID +MIDGARD_SLASH_POINTS_EVENTS,__HEVO__LOADED_AT +MIDGARD_SET_MIMIR_EVENTS,EVENT_ID +MIDGARD_CONSTANTS,__HEVO__DATABASE_NAME +MIDGARD_PENDING_LIQUIDITY_EVENTS,PENDING_TYPE +MIDGARD_STAKE_EVENTS,MEMO +MIDGARD_SET_NODE_KEYS_EVENTS,SECP256K1 +MIDGARD_SLASH_EVENTS,POOL +MIDGARD_ADD_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_ADD_EVENTS,MEMO +MIDGARD_SEND_MESSAGES,__HEVO__DATABASE_NAME +MIDGARD_ACTIVE_VAULT_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SLASH_EVENTS,EVENT_ID +MIDGARD_ACTIVE_VAULT_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_FAILED_DEPOSIT_MESSAGES,EVENT_ID +MIDGARD_SWAP_EVENTS,SWAP_SLIP_BP +MIDGARD_SWAP_EVENTS,FROM_E8 +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,__HEVO__LOADED_AT +MIDGARD_SET_NODE_KEYS_EVENTS,__HEVO__LOADED_AT +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,ASSET_ADD +MIDGARD_PENDING_LIQUIDITY_EVENTS,ASSET_CHAIN +MIDGARD_WITHDRAW_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_SLASH_POINTS_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_WITHDRAW_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SLASH_LIQUIDITY_EVENTS,CACAO_E10_LOSS +MIDGARD_REWARDS_EVENTS,BLOCK_TIMESTAMP +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,FORMER +MIDGARD_FEE_EVENTS,TX +MIDGARD_WITHDRAW_EVENTS,BLOCK_TIMESTAMP +MIDGARD_WITHDRAW_EVENTS,STAKE_UNITS +MIDGARD_BLOCK_LOG,TIMESTAMP +MIDGARD_OUTBOUND_EVENTS,FROM_ADDR +MIDGARD_STAKE_EVENTS,_ASSET_IN_CACAO_E8 +MIDGARD_WITHDRAW_EVENTS,TX +MIDGARD_STAKE_EVENTS,CACAO_TX +MIDGARD_WITHDRAW_EVENTS,__HEVO__INGESTED_AT +MIDGARD_TRANSFER_EVENTS,FROM_ADDR +MIDGARD_SEND_MESSAGES,__HEVO__LOADED_AT +MIDGARD_SLASH_POINTS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_WITHDRAW_EVENTS,ASSET +MIDGARD_SET_MIMIR_EVENTS,VALUE +MIDGARD_MAYANAME_CHANGE_EVENTS,__HEVO__LOADED_AT +MIDGARD_SLASH_LIQUIDITY_EVENTS,__HEVO__LOADED_AT +MIDGARD_REWARDS_EVENTS,__HEVO__LOADED_AT +MIDGARD_FEE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_MAYANAME_CHANGE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_PENDING_LIQUIDITY_EVENTS,CACAO_E8 +MIDGARD_SET_NODE_KEYS_EVENTS,ED25519 +MIDGARD_BOND_EVENTS,ASSET_E8 +MIDGARD_SET_NODE_MIMIR_EVENTS,BLOCK_TIMESTAMP +MIDGARD_SWAP_EVENTS,CHAIN +MIDGARD_TRANSFER_EVENTS,__HEVO__INGESTED_AT +MIDGARD_GAS_EVENTS,ASSET +MIDGARD_SET_NODE_MIMIR_EVENTS,__HEVO__INGESTED_AT +MIDGARD_REWARDS_EVENTS,BOND_E8 +MIDGARD_SEND_MESSAGES,AMOUNT_E8 +MIDGARD_REFUND_EVENTS,FROM_ADDR +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,DEPOSIT_ASSET +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,__HEVO__LOADED_AT +MIDGARD_FAILED_DEPOSIT_MESSAGES,MEMO +MIDGARD_SEND_MESSAGES,BLOCK_TIMESTAMP +MIDGARD_RESERVE_EVENTS,BLOCK_TIMESTAMP +MIDGARD_OUTBOUND_EVENTS,BLOCK_TIMESTAMP +MIDGARD_RESERVE_EVENTS,TO_ADDR +MIDGARD_BOND_EVENTS,TO_ADDR +MIDGARD_REFUND_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_RESERVE_EVENTS,ADDR +MIDGARD_SET_VERSION_EVENTS,__HEVO__LOADED_AT +MIDGARD_PENDING_LIQUIDITY_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_UPDATE_NODE_ACCOUNT_STATUS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_WITHDRAW_EVENTS,ASSET_E8 +MIDGARD_SEND_MESSAGES,ASSET +MIDGARD_SET_NODE_MIMIR_EVENTS,VALUE +MIDGARD_REFUND_EVENTS,REASON +MIDGARD_BOND_EVENTS,EVENT_ID +MIDGARD_SLASH_EVENTS,__HEVO__LOADED_AT +MIDGARD_STAKE_EVENTS,ASSET_ADDR +MIDGARD_POOL_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_FAILED_DEPOSIT_MESSAGES,FROM_ADDR +MIDGARD_BOND_EVENTS,ASSET +MIDGARD_WITHDRAW_EVENTS,__HEVO__LOADED_AT +MIDGARD_CONSTANTS,__HEVO__LOADED_AT +MIDGARD_PENDING_LIQUIDITY_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_SWAP_EVENTS,MEMO +MIDGARD_PENDING_LIQUIDITY_EVENTS,CACAO_ADDR +MIDGARD_SEND_MESSAGES,TO_ADDR +MIDGARD_SET_VERSION_EVENTS,EVENT_ID +MIDGARD_REFUND_EVENTS,BLOCK_TIMESTAMP +MIDGARD_FAILED_DEPOSIT_MESSAGES,BLOCK_TIMESTAMP +MIDGARD_OUTBOUND_EVENTS,CHAIN +MIDGARD_INACTIVE_VAULT_EVENTS,EVENT_ID +MIDGARD_OUTBOUND_EVENTS,__HEVO__SCHEMA_NAME +MIDGARD_POOL_BALANCE_CHANGE_EVENTS,CACAO_AMT +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,__HEVO__INGESTED_AT +MIDGARD_SWAP_EVENTS,__HEVO__LOADED_AT +MIDGARD_RESERVE_EVENTS,MEMO +MIDGARD_STAKE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_STAKE_EVENTS,__HEVO__INGESTED_AT +MIDGARD_WITHDRAW_EVENTS,CHAIN +MIDGARD_RESERVE_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_BLOCK_POOL_DEPTHS,__HEVO__SCHEMA_NAME +MIDGARD_TRANSFER_EVENTS,__HEVO__DATABASE_NAME +MIDGARD_MAYANAME_CHANGE_EVENTS,TX_ID +MIDGARD_PENDING_LIQUIDITY_EVENTS,BLOCK_TIMESTAMP +MIDGARD_SEND_MESSAGES,CODE +MIDGARD_SET_VERSION_EVENTS,NODE_ADDR +MIDGARD_SLASH_LIQUIDITY_EVENTS,ASSET +MIDGARD_BLOCK_POOL_DEPTHS,__HEVO__DATABASE_NAME +MIDGARD_SEND_MESSAGES,EVENT_ID +MIDGARD_FEE_EVENTS,EVENT_ID +MIDGARD_TRANSFER_EVENTS,TO_ADDR +MIDGARD_REFUND_EVENTS,MEMO +MIDGARD_VALIDATOR_REQUEST_LEAVE_EVENTS,TX +MIDGARD_REFUND_EVENTS,EVENT_ID +MIDGARD_TRANSFER_EVENTS,ASSET +MIDGARD_RESERVE_EVENTS,TX +MIDGARD_WITHDRAW_EVENTS,EMIT_ASSET_E8 +MIDGARD_BOND_EVENTS,BLOCK_TIMESTAMP +MIDGARD_SLASH_LIQUIDITY_EVENTS,LP_UNITS +MIDGARD_SLASH_POINTS_EVENTS,EVENT_ID +MIDGARD_REFUND_EVENTS,CODE +MIDGARD_STREAMING_SWAP_DETAILS_EVENTS,EVENT_ID diff --git a/models/bronze/bronze__active_vault_events.sql b/models/bronze/bronze__active_vault_events.sql new file mode 100644 index 0000000..48fb269 --- /dev/null +++ b/models/bronze/bronze__active_vault_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_active_vault_events' + ) }} diff --git a/models/bronze/bronze__add_events.sql b/models/bronze/bronze__add_events.sql new file mode 100644 index 0000000..181a54d --- /dev/null +++ b/models/bronze/bronze__add_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + cacao_e8, + pool, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_add_events' + ) }} diff --git a/models/bronze/bronze__block_log.sql b/models/bronze/bronze__block_log.sql new file mode 100644 index 0000000..96c7c19 --- /dev/null +++ b/models/bronze/bronze__block_log.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + HASH, + height, + TIMESTAMP, + agg_state, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'maya_midgard', + 'midgard_block_log' + ) }} diff --git a/models/bronze/bronze__block_pool_depths.sql b/models/bronze/bronze__block_pool_depths.sql new file mode 100644 index 0000000..7a7f97e --- /dev/null +++ b/models/bronze/bronze__block_pool_depths.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset_e8, + cacao_e8, + synth_e8, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'maya_midgard', + 'midgard_block_pool_depths' + ) }} diff --git a/models/bronze/bronze__bond_events.sql b/models/bronze/bronze__bond_events.sql new file mode 100644 index 0000000..9cfa583 --- /dev/null +++ b/models/bronze/bronze__bond_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + bond_type, + e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'maya_midgard', + 'midgard_bond_events' + ) }} diff --git a/models/bronze/bronze__cacao_price.sql b/models/bronze/bronze__cacao_price.sql new file mode 100644 index 0000000..ac2c068 --- /dev/null +++ b/models/bronze/bronze__cacao_price.sql @@ -0,0 +1,16 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + cacao_price_e8, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_cacao_price' + ) }} diff --git a/models/bronze/bronze__constants.sql b/models/bronze/bronze__constants.sql new file mode 100644 index 0000000..c7715dd --- /dev/null +++ b/models/bronze/bronze__constants.sql @@ -0,0 +1,16 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + key, + VALUE, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_constants' + ) }} diff --git a/models/bronze/bronze__failed_deposit_messages.sql b/models/bronze/bronze__failed_deposit_messages.sql new file mode 100644 index 0000000..c03acb5 --- /dev/null +++ b/models/bronze/bronze__failed_deposit_messages.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + code, + memo, + asset, + amount_e8, + from_addr, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_failed_deposit_messages' + ) }} diff --git a/models/bronze/bronze__fee_events.sql b/models/bronze/bronze__fee_events.sql new file mode 100644 index 0000000..fa2a5a4 --- /dev/null +++ b/models/bronze/bronze__fee_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + asset, + asset_e8, + pool_deduct, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_fee_events' + ) }} diff --git a/models/bronze/bronze__gas_events.sql b/models/bronze/bronze__gas_events.sql new file mode 100644 index 0000000..8e12d1c --- /dev/null +++ b/models/bronze/bronze__gas_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + asset_e8, + cacao_e8, + tx_count, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_gas_events' + ) }} diff --git a/models/bronze/bronze__inactive_vault_events.sql b/models/bronze/bronze__inactive_vault_events.sql new file mode 100644 index 0000000..5984eec --- /dev/null +++ b/models/bronze/bronze__inactive_vault_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_inactive_vault_events' + ) }} diff --git a/models/bronze/bronze__mayaname_change_events.sql b/models/bronze/bronze__mayaname_change_events.sql new file mode 100644 index 0000000..28abe83 --- /dev/null +++ b/models/bronze/bronze__mayaname_change_events.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + NAME, + chain, + address, + registration_fee_e8, + fund_amount_e8, + height, + expire, + owner, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_mayaname_change_events' + ) }} diff --git a/models/bronze/bronze__new_node_events.sql b/models/bronze/bronze__new_node_events.sql new file mode 100644 index 0000000..2b87cef --- /dev/null +++ b/models/bronze/bronze__new_node_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_new_node_events' + ) }} diff --git a/models/bronze/bronze__outbound_events.sql b/models/bronze/bronze__outbound_events.sql new file mode 100644 index 0000000..124e2ff --- /dev/null +++ b/models/bronze/bronze__outbound_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + in_tx, + internal, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_outbound_events' + ) }} diff --git a/models/bronze/bronze__pending_liquidity_events.sql b/models/bronze/bronze__pending_liquidity_events.sql new file mode 100644 index 0000000..b3aaa74 --- /dev/null +++ b/models/bronze/bronze__pending_liquidity_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset_tx, + asset_chain, + asset_addr, + asset_e8, + cacao_tx, + cacao_addr, + cacao_e8, + pending_type, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_pending_liquidity_events' + ) }} diff --git a/models/bronze/bronze__pool_balance_change_events.sql b/models/bronze/bronze__pool_balance_change_events.sql new file mode 100644 index 0000000..6114c16 --- /dev/null +++ b/models/bronze/bronze__pool_balance_change_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + cacao_amt, + cacao_add, + asset_amt, + asset_add, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_pool_balance_change_events' + ) }} diff --git a/models/bronze/bronze__pool_events.sql b/models/bronze/bronze__pool_events.sql new file mode 100644 index 0000000..ef98d7e --- /dev/null +++ b/models/bronze/bronze__pool_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + status, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_pool_events' + ) }} diff --git a/models/bronze/bronze__refund_events.sql b/models/bronze/bronze__refund_events.sql new file mode 100644 index 0000000..c0226eb --- /dev/null +++ b/models/bronze/bronze__refund_events.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_refund_events' + ) }} diff --git a/models/bronze/bronze__reserve_events.sql b/models/bronze/bronze__reserve_events.sql new file mode 100644 index 0000000..7aef888 --- /dev/null +++ b/models/bronze/bronze__reserve_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + addr, + e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_reserve_events' + ) }} diff --git a/models/bronze/bronze__rewards_event_entries.sql b/models/bronze/bronze__rewards_event_entries.sql new file mode 100644 index 0000000..7ca83b6 --- /dev/null +++ b/models/bronze/bronze__rewards_event_entries.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + cacao_e8, + saver_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_rewards_event_entries' + ) }} diff --git a/models/bronze/bronze__rewards_events.sql b/models/bronze/bronze__rewards_events.sql new file mode 100644 index 0000000..6b35002 --- /dev/null +++ b/models/bronze/bronze__rewards_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_rewards_events' + ) }} diff --git a/models/bronze/bronze__send_messages.sql b/models/bronze/bronze__send_messages.sql new file mode 100644 index 0000000..424aabb --- /dev/null +++ b/models/bronze/bronze__send_messages.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + amount_e8, + asset, + from_addr, + to_addr, + memo, + tx_id, + raw_log, + code, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_send_messages' + ) }} diff --git a/models/bronze/bronze__set_ip_address_events.sql b/models/bronze/bronze__set_ip_address_events.sql new file mode 100644 index 0000000..26c33ba --- /dev/null +++ b/models/bronze/bronze__set_ip_address_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + ip_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_ip_address_events' + ) }} diff --git a/models/bronze/bronze__set_mimir_events.sql b/models/bronze/bronze__set_mimir_events.sql new file mode 100644 index 0000000..de9592f --- /dev/null +++ b/models/bronze/bronze__set_mimir_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + key, + VALUE, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_mimir_events' + ) }} diff --git a/models/bronze/bronze__set_node_keys_events.sql b/models/bronze/bronze__set_node_keys_events.sql new file mode 100644 index 0000000..f9d3373 --- /dev/null +++ b/models/bronze/bronze__set_node_keys_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + secp256k1, + ed25519, + validator_consensus, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_node_keys_events' + ) }} diff --git a/models/bronze/bronze__set_node_mimir_events.sql b/models/bronze/bronze__set_node_mimir_events.sql new file mode 100644 index 0000000..4043ac7 --- /dev/null +++ b/models/bronze/bronze__set_node_mimir_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + address, + key, + VALUE, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_node_mimir_events' + ) }} diff --git a/models/bronze/bronze__set_version_events.sql b/models/bronze/bronze__set_version_events.sql new file mode 100644 index 0000000..75957a3 --- /dev/null +++ b/models/bronze/bronze__set_version_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + version, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_version_events' + ) }} diff --git a/models/bronze/bronze__slash_events.sql b/models/bronze/bronze__slash_events.sql new file mode 100644 index 0000000..2a84579 --- /dev/null +++ b/models/bronze/bronze__slash_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset, + asset_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_slash_events' + ) }} diff --git a/models/bronze/bronze__slash_liquidity_events.sql b/models/bronze/bronze__slash_liquidity_events.sql new file mode 100644 index 0000000..e3c12df --- /dev/null +++ b/models/bronze/bronze__slash_liquidity_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_slash_liquidity_events' + ) }} diff --git a/models/bronze/bronze__slash_points_events.sql b/models/bronze/bronze__slash_points_events.sql new file mode 100644 index 0000000..f8667e3 --- /dev/null +++ b/models/bronze/bronze__slash_points_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_address, + slash_points, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_slash_points_events' + ) }} diff --git a/models/bronze/bronze__stake_events.sql b/models/bronze/bronze__stake_events.sql new file mode 100644 index 0000000..e193afc --- /dev/null +++ b/models/bronze/bronze__stake_events.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset_tx, + asset_chain, + asset_addr, + asset_e8, + stake_units, + cacao_tx, + cacao_addr, + cacao_e8, + _ASSET_IN_CACAO_E8, + memo, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_stake_events' + ) }} diff --git a/models/bronze/bronze__streamling_swap_details_events.sql b/models/bronze/bronze__streamling_swap_details_events.sql new file mode 100644 index 0000000..ae44289 --- /dev/null +++ b/models/bronze/bronze__streamling_swap_details_events.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swap_reasons, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_streaming_swap_details_events' + ) }} diff --git a/models/bronze/bronze__swap_events.sql b/models/bronze/bronze__swap_events.sql new file mode 100644 index 0000000..ac43e78 --- /dev/null +++ b/models/bronze/bronze__swap_events.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + _STREAMING, + streaming_count, + streaming_quantity, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_swap_events' + ) }} diff --git a/models/bronze/bronze__transfer_events.sql b/models/bronze/bronze__transfer_events.sql new file mode 100644 index 0000000..ea7ab0c --- /dev/null +++ b/models/bronze/bronze__transfer_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + from_addr, + to_addr, + asset, + amount_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_transfer_events' + ) }} diff --git a/models/bronze/bronze__update_node_account_status_events.sql b/models/bronze/bronze__update_node_account_status_events.sql new file mode 100644 index 0000000..abf763f --- /dev/null +++ b/models/bronze/bronze__update_node_account_status_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + former, + "CURRENT", + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_update_node_account_status_events' + ) }} diff --git a/models/bronze/bronze__validator_request_leave_events.sql b/models/bronze/bronze__validator_request_leave_events.sql new file mode 100644 index 0000000..7648cd1 --- /dev/null +++ b/models/bronze/bronze__validator_request_leave_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + from_addr, + node_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_validator_request_leave_events' + ) }} diff --git a/models/bronze/bronze__withdraw_events.sql b/models/bronze/bronze__withdraw_events.sql new file mode 100644 index 0000000..71f6142 --- /dev/null +++ b/models/bronze/bronze__withdraw_events.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + emit_asset_e8, + emit_cacao_e8, + memo, + pool, + stake_units, + basis_points, + asymmetry, + imp_loss_protection_e8, + _EMIT_ASSET_IN_CACAO_E8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_withdraw_events' + ) }} diff --git a/models/silver/silver__active_vault_events.sql b/models/silver/silver__active_vault_events.sql new file mode 100644 index 0000000..9f1af26 --- /dev/null +++ b/models/silver/silver__active_vault_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__active_vault_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__active_vault_events.yml b/models/silver/silver__active_vault_events.yml new file mode 100644 index 0000000..d9edbdc --- /dev/null +++ b/models/silver/silver__active_vault_events.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: silver__active_vault_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - BLOCK_TIMESTAMP + - ADD_ASGARD_ADDR + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ADD_ASGARD_ADDR + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__add_events.sql b/models/silver/silver__add_events.sql new file mode 100644 index 0000000..21109c1 --- /dev/null +++ b/models/silver/silver__add_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + cacao_e8, + pool AS pool_name, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__add_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__add_events.yml b/models/silver/silver__add_events.yml new file mode 100644 index 0000000..2244613 --- /dev/null +++ b/models/silver/silver__add_events.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: silver__add_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - TX_ID + - BLOCKCHAIN + - POOL_NAME + - FROM_ADDRESS + - TO_ADDRESS + - MEMO + columns: + - name: TX_ID + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: cacao_E8 + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__block_log.sql b/models/silver/silver__block_log.sql new file mode 100644 index 0000000..8e4d7e6 --- /dev/null +++ b/models/silver/silver__block_log.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = 'view' +) }} + +WITH base AS ( + + SELECT + height, + TO_TIMESTAMP( + TIMESTAMP / 1000000000 + ) AS block_timestamp, + TIMESTAMP, + HASH, + agg_state, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP + FROM + {{ ref('bronze__block_log') }} + qualify(ROW_NUMBER() over(PARTITION BY height + ORDER BY + __HEVO__LOADED_AT DESC)) = 1 +) +SELECT + height, + block_timestamp, + block_timestamp :: DATE AS block_date, + HOUR(block_timestamp) AS block_hour, + week(block_timestamp) AS block_week, + MONTH(block_timestamp) AS block_month, + quarter(block_timestamp) AS block_quarter, + YEAR(block_timestamp) AS block_year, + dayofmonth(block_timestamp) AS block_DAYOFMONTH, + dayofweek(block_timestamp) AS block_DAYOFWEEK, + dayofyear(block_timestamp) AS block_DAYOFYEAR, + TIMESTAMP, + HASH, + agg_state, + _INSERTED_TIMESTAMP +FROM + base diff --git a/models/silver/silver__block_log.yml b/models/silver/silver__block_log.yml new file mode 100644 index 0000000..18eb923 --- /dev/null +++ b/models/silver/silver__block_log.yml @@ -0,0 +1,20 @@ +version: 2 +models: + - name: silver__block_log + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HEIGHT + - TIMESTAMP + - HASH + - AGG_STATE + columns: + - name: TIMESTAMP + data_tests: + - not_null + - name: HEIGHT + data_tests: + - not_null + - name: HASH + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__block_pool_depths.sql b/models/silver/silver__block_pool_depths.sql new file mode 100644 index 0000000..1bdb920 --- /dev/null +++ b/models/silver/silver__block_pool_depths.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset_e8, + cacao_e8, + synth_e8, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__block_pool_depths') }} + qualify(ROW_NUMBER() over(PARTITION BY pool, block_timestamp +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__block_pool_depths.yml b/models/silver/silver__block_pool_depths.yml new file mode 100644 index 0000000..d02771d --- /dev/null +++ b/models/silver/silver__block_pool_depths.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__block_pool_depths + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + - POOL_NAME + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: SYNTH_E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__block_rewards.sql b/models/silver/silver__block_rewards.sql new file mode 100644 index 0000000..74e59d5 --- /dev/null +++ b/models/silver/silver__block_rewards.sql @@ -0,0 +1,160 @@ +{{ config( + materialized = 'table', + cluster_by = ['day'] +) }} + +WITH all_block_id AS ( + + SELECT + block_timestamp, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} + GROUP BY + block_timestamp +), +avg_nodes_tbl AS ( + SELECT + block_timestamp, + SUM( + CASE + WHEN current_status = 'Active' THEN 1 + WHEN former_status = 'Active' THEN -1 + ELSE 0 + END + ) AS delta + FROM + {{ ref('silver__update_node_account_status_events') }} + GROUP BY + block_timestamp +), +all_block_with_nodes AS ( + SELECT + all_block_id.block_timestamp, + delta, + SUM(delta) over ( + ORDER BY + all_block_id.block_timestamp ASC + ) AS avg_nodes, + _inserted_timestamp AS _inserted_timestamp + FROM + all_block_id + LEFT JOIN avg_nodes_tbl + ON all_block_id.block_timestamp = avg_nodes_tbl.block_timestamp +), +all_block_with_nodes_date AS ( + SELECT + b.block_timestamp :: DATE AS DAY, + AVG(avg_nodes) AS avg_nodes, + MAX( + A._inserted_timestamp + ) AS _inserted_timestamp + FROM + all_block_with_nodes A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY +), +liquidity_fee_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + COALESCE(SUM(liq_fee_in_cacao_e8), 0) AS liquidity_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + 1 +), +bond_earnings_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + SUM(bond_e8) AS bond_earnings + FROM + {{ ref('silver__rewards_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY +), +total_pool_rewards_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + SUM(cacao_e8) AS total_pool_rewards + FROM + {{ ref('silver__rewards_event_entries') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY +) +SELECT + all_block_with_nodes_date.day, + COALESCE((liquidity_fee_tbl.liquidity_fee / power(10, 8)), 0) AS liquidity_fee, + ( + ( + COALESCE( + total_pool_rewards_tbl.total_pool_rewards, + 0 + ) + COALESCE( + bond_earnings_tbl.bond_earnings, + 0 + ) + ) + ) / power( + 10, + 8 + ) AS block_rewards, + ( + ( + COALESCE( + total_pool_rewards_tbl.total_pool_rewards, + 0 + ) + COALESCE( + liquidity_fee_tbl.liquidity_fee, + 0 + ) + COALESCE( + bond_earnings_tbl.bond_earnings, + 0 + ) + ) + ) / power( + 10, + 8 + ) AS earnings, + COALESCE((bond_earnings_tbl.bond_earnings / power(10, 8)), 0) AS bonding_earnings, + ( + ( + COALESCE( + total_pool_rewards_tbl.total_pool_rewards, + 0 + ) + COALESCE( + liquidity_fee_tbl.liquidity_fee, + 0 + ) + ) + ) / power( + 10, + 8 + ) AS liquidity_earnings, + all_block_with_nodes_date.avg_nodes + 2 AS avg_node_count, + all_block_with_nodes_date._inserted_timestamp +FROM + all_block_with_nodes_date + LEFT JOIN liquidity_fee_tbl + ON all_block_with_nodes_date.day = liquidity_fee_tbl.day + LEFT JOIN total_pool_rewards_tbl + ON all_block_with_nodes_date.day = total_pool_rewards_tbl.day + LEFT JOIN bond_earnings_tbl + ON all_block_with_nodes_date.day = bond_earnings_tbl.day diff --git a/models/silver/silver__block_rewards.yml b/models/silver/silver__block_rewards.yml new file mode 100644 index 0000000..b7e8b50 --- /dev/null +++ b/models/silver/silver__block_rewards.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: silver__block_rewards + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: LIQUIDITY_FEE + data_tests: + - not_null + - name: BLOCK_REWARDS + data_tests: + - not_null + - name: EARNINGS + data_tests: + - not_null + - name: BONDING_EARNINGS + data_tests: + - not_null + - name: LIQUIDITY_EARNINGS + data_tests: + - not_null + - name: AVG_NODE_COUNT \ No newline at end of file diff --git a/models/silver/silver__bond_events.sql b/models/silver/silver__bond_events.sql new file mode 100644 index 0000000..40b09a2 --- /dev/null +++ b/models/silver/silver__bond_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + bond_type, + e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__bond_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__bond_events.yml b/models/silver/silver__bond_events.yml new file mode 100644 index 0000000..f086b3d --- /dev/null +++ b/models/silver/silver__bond_events.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__bond_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: TX_ID + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: BOND_TYPE + data_tests: + - not_null + - name: E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__cacao_price.sql b/models/silver/silver__cacao_price.sql new file mode 100644 index 0000000..26f9160 --- /dev/null +++ b/models/silver/silver__cacao_price.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + cacao_price_e8, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref( + 'bronze__cacao_price' + ) }} + qualify(ROW_NUMBER() over (PARTITION BY block_timestamp +ORDER BY + cacao_price_e8 DESC) = 1) diff --git a/models/silver/silver__cacao_price.yml b/models/silver/silver__cacao_price.yml new file mode 100644 index 0000000..c966256 --- /dev/null +++ b/models/silver/silver__cacao_price.yml @@ -0,0 +1,10 @@ +version: 2 +models: + - name: silver__cacao_price + columns: + - name: CACAO_PRICE_E8 + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__constants.sql b/models/silver/silver__constants.sql new file mode 100644 index 0000000..0558482 --- /dev/null +++ b/models/silver/silver__constants.sql @@ -0,0 +1,9 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + C.key, + C.value +FROM + {{ ref('bronze__constants') }} C diff --git a/models/silver/silver__daily_earnings.sql b/models/silver/silver__daily_earnings.sql new file mode 100644 index 0000000..dd4dcdb --- /dev/null +++ b/models/silver/silver__daily_earnings.sql @@ -0,0 +1,109 @@ +{{ config( + materialized = 'incremental', + unique_key = 'day', + incremental_strategy = 'merge', + cluster_by = ['day'] +) }} + +WITH max_daily_block AS ( + + SELECT + MAX( + block_id + ) AS block_id, + DATE_TRUNC( + 'day', + block_timestamp + ) AS DAY + FROM + {{ ref('silver__prices') }} A + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY +), +daily_cacao_price AS ( + SELECT + p.block_id, + DAY, + AVG(cacao_usd) AS cacao_usd + FROM + {{ ref('silver__prices') }} + p + JOIN max_daily_block mdb + ON p.block_id = mdb.block_id + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + p.block_id +) +SELECT + br.day, + COALESCE( + liquidity_fee, + 0 + ) AS liquidity_fees, + COALESCE( + liquidity_fee * cacao_usd, + 0 + ) AS liquidity_fees_usd, + block_rewards AS block_rewards, + block_rewards * cacao_usd AS block_rewards_usd, + COALESCE( + earnings, + 0 + ) AS total_earnings, + COALESCE( + earnings * cacao_usd, + 0 + ) AS total_earnings_usd, + bonding_earnings AS earnings_to_nodes, + bonding_earnings * cacao_usd AS earnings_to_nodes_usd, + COALESCE( + liquidity_earnings, + 0 + ) AS earnings_to_pools, + COALESCE( + liquidity_earnings * cacao_usd, + 0 + ) AS earnings_to_pools_usd, + avg_node_count, + br._inserted_timestamp +FROM + {{ ref('silver__block_rewards') }} + br + JOIN daily_cacao_price drp + ON br.day = drp.day + +{% if is_incremental() %} +WHERE + br.day >= ( + SELECT + MAX( + DAY + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__daily_earnings.yml b/models/silver/silver__daily_earnings.yml new file mode 100644 index 0000000..c5cd598 --- /dev/null +++ b/models/silver/silver__daily_earnings.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: silver__daily_earnings + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: LIQUIDITY_FEES + data_tests: + - not_null + - name: LIQUIDITY_FEES_USD + data_tests: + - not_null + - name: BLOCK_REWARDS + data_tests: + - not_null + - name: BLOCK_REWARDS_USD + data_tests: + - not_null + - name: TOTAL_EARNINGS + data_tests: + - not_null + - name: TOTAL_EARNINGS_USD + data_tests: + - not_null + - name: EARNINGS_TO_NODES + data_tests: + - not_null + - name: EARNINGS_TO_NODES_USD + data_tests: + - not_null + - name: EARNINGS_TO_POOLS + data_tests: + - not_null + - name: EARNINGS_TO_POOLS_USD + data_tests: + - not_null + - name: AVG_NODE_COUNT diff --git a/models/silver/silver__daily_pool_stats.sql b/models/silver/silver__daily_pool_stats.sql new file mode 100644 index 0000000..a00303e --- /dev/null +++ b/models/silver/silver__daily_pool_stats.sql @@ -0,0 +1,177 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['day'], + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'] +) }} + +WITH daily_cacao_price AS ( + + SELECT + pool_name, + block_timestamp :: DATE AS DAY, + AVG(cacao_usd) AS cacao_usd, + AVG(asset_usd) AS asset_usd + FROM + {{ ref('silver__prices') }} + p + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + pool_name, + DAY +), +pool_fees AS ( + SELECT + pbf.day, + pbf.pool_name, + rewards AS system_rewards, + rewards * cacao_usd AS system_rewards_usd, + asset_liquidity_fees, + asset_liquidity_fees * asset_usd AS asset_liquidity_fees_usd, + cacao_liquidity_fees, + cacao_liquidity_fees * cacao_usd AS cacao_liquidity_fees_usd + FROM + {{ ref('silver__pool_block_fees') }} + pbf + JOIN daily_cacao_price drp + ON pbf.day = drp.day + AND pbf.pool_name = drp.pool_name + +{% if is_incremental() %} +WHERE + pbf.day >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + pbs.day, + pbs.asset AS pool_name, + COALESCE( + system_rewards, + 0 + ) AS system_rewards, + COALESCE( + system_rewards_usd, + 0 + ) AS system_rewards_usd, + COALESCE(asset_depth / pow(10, 8), 0) AS asset_liquidity, + COALESCE( + asset_price, + 0 + ) AS asset_price, + COALESCE( + asset_price_usd, + 0 + ) AS asset_price_usd, + COALESCE(cacao_depth / pow(10, 8), 0) AS cacao_liquidity, + COALESCE(asset_price_usd / NULLIF(cacao_usd, 0), 0) AS cacao_price, + COALESCE( + cacao_usd, + 0 + ) AS cacao_price_usd, + COALESCE( + add_liquidity_count, + 0 + ) AS add_liquidity_count, + COALESCE(add_asset_liquidity_volume / pow(10, 8), 0) AS add_asset_liquidity, + COALESCE(add_asset_liquidity_volume / pow(10, 8) * asset_usd, 0) AS add_asset_liquidity_usd, + COALESCE(add_cacao_liquidity_volume / pow(10, 8), 0) AS add_cacao_liquidity, + COALESCE(add_cacao_liquidity_volume / pow(10, 8) * cacao_usd, 0) AS add_cacao_liquidity_usd, + COALESCE( + withdraw_count, + 0 + ) AS withdraw_count, + COALESCE(withdraw_asset_volume / pow(10, 8), 0) AS withdraw_asset_liquidity, + COALESCE(withdraw_asset_volume / pow(10, 8) * asset_usd, 0) AS withdraw_asset_liquidity_usd, + COALESCE(withdraw_cacao_volume / pow(10, 8), 0) AS withdraw_cacao_liquidity, + COALESCE(withdraw_cacao_volume / pow(10, 8) * cacao_usd, 0) AS withdraw_cacao_liquidity_usd, + COALESCE(impermanent_loss_protection_paid / pow(10, 8), 0) AS il_protection_paid, + COALESCE(impermanent_loss_protection_paid / pow(10, 8) * cacao_usd, 0) AS il_protection_paid_usd, + COALESCE( + average_slip, + 0 + ) AS average_slip, + COALESCE( + to_asset_average_slip, + 0 + ) AS to_asset_average_slip, + COALESCE( + to_cacao_average_slip, + 0 + ) AS to_cacao_average_slip, + COALESCE( + swap_count, + 0 + ) AS swap_count, + COALESCE( + to_asset_count, + 0 + ) AS to_asset_swap_count, + COALESCE( + to_cacao_count, + 0 + ) AS to_cacao_swap_count, + COALESCE(swap_volume / pow(10, 8), 0) AS swap_volume_cacao, + COALESCE(swap_volume / pow(10, 8) * cacao_usd, 0) AS swap_volume_cacao_usd, + COALESCE(to_asset_volume / pow(10, 8), 0) AS to_asset_swap_volume, + COALESCE(to_cacao_volume / pow(10, 8), 0) AS to_cacao_swap_volume, + COALESCE(totalfees / pow(10, 8), 0) AS total_swap_fees_cacao, + COALESCE(totalfees / pow(10, 8) * cacao_usd, 0) AS total_swap_fees_usd, + COALESCE(to_asset_fees / pow(10, 8), 0) AS total_asset_swap_fees, + COALESCE(to_cacao_fees / pow(10, 8), 0) AS total_asset_cacao_fees, + COALESCE( + unique_member_count, + 0 + ) AS unique_member_count, + COALESCE( + unique_swapper_count, + 0 + ) AS unique_swapper_count, + COALESCE( + units, + 0 + ) AS liquidity_units, + concat_ws( + '-', + pbs.day, + pbs.asset + ) AS _unique_key +FROM + {{ ref('silver__pool_block_statistics') }} + pbs + LEFT JOIN daily_cacao_price drp + ON pbs.day = drp.day + AND pbs.asset = drp.pool_name + LEFT JOIN pool_fees pf + ON pbs.day = pf.day + AND pbs.asset = pf.pool_name + +{% if is_incremental() %} +WHERE + pbs.day >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__daily_pool_stats.yml b/models/silver/silver__daily_pool_stats.yml new file mode 100644 index 0000000..e9f8e66 --- /dev/null +++ b/models/silver/silver__daily_pool_stats.yml @@ -0,0 +1,126 @@ +version: 2 +models: + - name: silver__daily_pool_stats + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + - POOL_NAME + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: POOL_NAME + data_tests: + - not_null + - name: SYSTEM_REWARDS + data_tests: + - not_null + - name: SYSTEM_REWARDS_USD + data_tests: + - not_null + - name: ASSET_LIQUIDITY + data_tests: + - not_null + - name: ASSET_PRICE + data_tests: + - not_null + - name: ASSET_PRICE_USD + data_tests: + - not_null + - name: cacao_LIQUIDITY + data_tests: + - not_null + - name: cacao_PRICE + data_tests: + - not_null + - name: cacao_PRICE_USD + data_tests: + - not_null + - name: ADD_LIQUIDITY_COUNT + data_tests: + - not_null + - name: ADD_ASSET_LIQUIDITY + data_tests: + - not_null + - name: ADD_ASSET_LIQUIDITY_USD + data_tests: + - not_null + - name: ADD_cacao_LIQUIDITY_USD + data_tests: + - not_null + - name: WITHDRAW_COUNT + data_tests: + - not_null + - name: WITHDRAW_ASSET_LIQUIDITY + data_tests: + - not_null + - name: WITHDRAW_ASSET_LIQUIDITY_USD + data_tests: + - not_null + - name: WITHDRAW_cacao_LIQUIDITY + data_tests: + - not_null + - name: WITHDRAW_cacao_LIQUIDITY_USD + data_tests: + - not_null + - name: IL_PROTECTION_PAID + data_tests: + - not_null + - name: IL_PROTECTION_PAID_USD + data_tests: + - not_null + - name: AVERAGE_SLIP + data_tests: + - not_null + - name: TO_ASSET_AVERAGE_SLIP + data_tests: + - not_null + - name: TO_cacao_AVERAGE_SLIP + data_tests: + - not_null + - name: SWAP_COUNT + data_tests: + - not_null + - name: TO_ASSET_SWAP_COUNT + data_tests: + - not_null + - name: TO_cacao_SWAP_COUNT + data_tests: + - not_null + - name: SWAP_VOLUME_cacao + data_tests: + - not_null + - name: SWAP_VOLUME_cacao_USD + data_tests: + - not_null + - name: TO_ASSET_SWAP_VOLUME + data_tests: + - not_null + - name: TO_cacao_SWAP_VOLUME + data_tests: + - not_null + - name: TOTAL_SWAP_FEES_cacao + data_tests: + - not_null + - name: TOTAL_SWAP_FEES_USD + data_tests: + - not_null + - name: TOTAL_ASSET_SWAP_FEES + data_tests: + - not_null + - name: TOTAL_ASSET_cacao_FEES + data_tests: + - not_null + - name: UNIQUE_MEMBER_COUNT + data_tests: + - not_null + - name: UNIQUE_SWAPPER_COUNT + data_tests: + - not_null + - name: LIQUIDITY_UNITS + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__daily_tvl.sql b/models/silver/silver__daily_tvl.sql new file mode 100644 index 0000000..d23f724 --- /dev/null +++ b/models/silver/silver__daily_tvl.sql @@ -0,0 +1,72 @@ +{{ config( + materialized = 'incremental', + unique_key = "day", + incremental_strategy = 'merge', + cluster_by = ['day'], + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'] +) }} + +WITH max_daily_block AS ( + + SELECT + MAX(block_id) AS block_id, + DATE_TRUNC( + 'day', + block_timestamp + ) AS DAY + FROM + {{ ref('silver__prices') }} + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY +), +daily_cacao_price AS ( + SELECT + p.block_id, + DAY, + AVG(cacao_usd) AS cacao_usd + FROM + {{ ref('silver__prices') }} + p + JOIN max_daily_block mdb + ON p.block_id = mdb.block_id + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + p.block_id +) +SELECT + br.day, + total_value_pooled AS total_value_pooled, + total_value_pooled * cacao_usd AS total_value_pooled_usd, + total_value_bonded AS total_value_bonded, + total_value_bonded * cacao_usd AS total_value_bonded_usd, + total_value_locked AS total_value_locked, + total_value_locked * cacao_usd AS total_value_locked_usd +FROM + {{ ref('silver__total_value_locked') }} + br + JOIN daily_cacao_price drp + ON br.day = drp.day diff --git a/models/silver/silver__daily_tvl.yml b/models/silver/silver__daily_tvl.yml new file mode 100644 index 0000000..f8b22fc --- /dev/null +++ b/models/silver/silver__daily_tvl.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: silver__daily_tvl + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: TOTAL_VALUE_POOLED + data_tests: + - not_null + - name: TOTAL_VALUE_POOLED_USD + data_tests: + - not_null + - name: TOTAL_VALUE_BONDED + data_tests: + - not_null + - name: TOTAL_VALUE_BONDED_USD + data_tests: + - not_null + - name: TOTAL_VALUE_LOCKED + data_tests: + - not_null + - name: TOTAL_VALUE_LOCKED_USD + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__failed_deposit_messages.sql b/models/silver/silver__failed_deposit_messages.sql new file mode 100644 index 0000000..55caaef --- /dev/null +++ b/models/silver/silver__failed_deposit_messages.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + code, + memo, + asset, + amount_e8, + from_addr AS from_address, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref( + 'bronze__failed_deposit_messages' + ) }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__failed_deposit_messages.yml b/models/silver/silver__failed_deposit_messages.yml new file mode 100644 index 0000000..577d132 --- /dev/null +++ b/models/silver/silver__failed_deposit_messages.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: silver__failed_deposit_messages + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + columns: + - name: AMOUNT_E8 + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: MEMO + - name: CODE + - name: REASON + - name: TX_ID + data_tests: + - not_null + - name: EVENT_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__fee_events.sql b/models/silver/silver__fee_events.sql new file mode 100644 index 0000000..55af07f --- /dev/null +++ b/models/silver/silver__fee_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + asset, + asset_e8, + pool_deduct, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__fee_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__fee_events.yml b/models/silver/silver__fee_events.yml new file mode 100644 index 0000000..fab3b90 --- /dev/null +++ b/models/silver/silver__fee_events.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__fee_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: TX_ID + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: POOL_DEDUCT + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null diff --git a/models/silver/silver__gas_events.sql b/models/silver/silver__gas_events.sql new file mode 100644 index 0000000..3869018 --- /dev/null +++ b/models/silver/silver__gas_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + asset_e8, + cacao_e8, + tx_count, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__gas_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__gas_events.yml b/models/silver/silver__gas_events.yml new file mode 100644 index 0000000..f6900ee --- /dev/null +++ b/models/silver/silver__gas_events.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__gas_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - ASSET + - BLOCK_TIMESTAMP + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: TX_COUNT + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__inactive_vault_events.sql b/models/silver/silver__inactive_vault_events.sql new file mode 100644 index 0000000..803107b --- /dev/null +++ b/models/silver/silver__inactive_vault_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr AS add_asgard_address, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__inactive_vault_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__liquidity_actions.sql b/models/silver/silver__liquidity_actions.sql new file mode 100644 index 0000000..b589b3f --- /dev/null +++ b/models/silver/silver__liquidity_actions.sql @@ -0,0 +1,180 @@ +{{ config( + materialized = 'incremental', + unique_key = '_unique_key', + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH stakes AS ( + + SELECT + block_timestamp, + cacao_tx_id, + pool_name, + cacao_address, + cacao_e8, + asset_e8, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__stake_events') }} +), +unstakes AS ( + SELECT + block_timestamp, + tx_id, + pool_name, + from_address, + to_address, + emit_cacao_e8, + emit_asset_e8, + stake_units, + imp_loss_protection_e8, + asymmetry, + basis_points, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__withdraw_events') }} +) +SELECT + b.block_timestamp, + b.height AS block_id, + cacao_tx_id AS tx_id, + 'add_liquidity' AS lp_action, + se.pool_name, + cacao_address AS from_address, + NULL AS to_address, + COALESCE((cacao_e8 / pow(10, 8)), 0) AS cacao_amount, + COALESCE((cacao_e8 / pow(10, 8) * cacao_usd), 0) AS cacao_amount_usd, + COALESCE((asset_e8 / pow(10, 8)), 0) AS asset_amount, + COALESCE((asset_e8 / pow(10, 8) * asset_usd), 0) AS asset_amount_usd, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + NULL AS il_protection, + NULL AS il_protection_usd, + NULL AS unstake_asymmetry, + NULL AS unstake_basis_points, + concat_ws( + '-', + event_id, + se.block_timestamp, + COALESCE( + tx_id, + '' + ), + lp_action, + se.pool_name, + COALESCE( + from_address, + '' + ), + COALESCE( + to_address, + '' + ), + COALESCE( + asset_tx_id, + '' + ) + ) AS _unique_key, + se._inserted_timestamp +FROM + stakes se + JOIN {{ ref('silver__block_log') }} + b + ON se.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND se.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +UNION +SELECT + b.block_timestamp, + b.height AS block_id, + tx_id, + 'remove_liquidity' AS lp_action, + ue.pool_name, + from_address, + to_address, + COALESCE(emit_cacao_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(emit_cacao_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + COALESCE(emit_asset_e8 / pow(10, 8), 0) AS asset_amount, + COALESCE(emit_asset_e8 / pow(10, 8) * asset_usd, 0) AS asset_amount_usd, + stake_units, + NULL AS asset_tx_id, + NULL AS asset_address, + NULL AS asset_blockchain, + imp_loss_protection_e8 / pow( + 10, + 8 + ) AS il_protection, + imp_loss_protection_e8 / pow( + 10, + 8 + ) * cacao_usd AS il_protection_usd, + asymmetry AS unstake_asymmetry, + basis_points AS unstake_basis_points, + concat_ws( + '-', + event_id, + ue.block_timestamp, + COALESCE( + tx_id, + '' + ), + lp_action, + ue.pool_name, + COALESCE( + from_address, + '' + ), + COALESCE( + to_address, + '' + ), + COALESCE( + asset_tx_id, + '' + ) + ) AS _unique_key, + ue._inserted_timestamp +FROM + unstakes ue + JOIN {{ ref('silver__block_log') }} + b + ON ue.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND ue.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__liquidity_actions.yml b/models/silver/silver__liquidity_actions.yml new file mode 100644 index 0000000..9afcaa8 --- /dev/null +++ b/models/silver/silver__liquidity_actions.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: silver__liquidity_actions + + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + data_tests: + - not_null + - name: LP_ACTION + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= SYSDATE() - interval '2 day' AND BLOCK_TIMESTAMP >= '2021-04-13' + - name: ASSET_AMOUNT + data_tests: + - not_null + - name: STAKE_UNITS + data_tests: + - not_null diff --git a/models/silver/silver__mayaname_change_events.sql b/models/silver/silver__mayaname_change_events.sql new file mode 100644 index 0000000..32f3d41 --- /dev/null +++ b/models/silver/silver__mayaname_change_events.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + owner, + chain, + address, + expire, + NAME, + fund_amount_e8, + registration_fee_e8, + event_id, + height, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__mayaname_change_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__new_node_events.sql b/models/silver/silver__new_node_events.sql new file mode 100644 index 0000000..0fab353 --- /dev/null +++ b/models/silver/silver__new_node_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__new_node_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__outbound_events.sql b/models/silver/silver__outbound_events.sql new file mode 100644 index 0000000..bcd71d2 --- /dev/null +++ b/models/silver/silver__outbound_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + in_tx, + internal, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__outbound_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, in_tx +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__outbound_events.yml b/models/silver/silver__outbound_events.yml new file mode 100644 index 0000000..b2ed12b --- /dev/null +++ b/models/silver/silver__outbound_events.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: silver__outbound_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - ASSET + - MEMO + - IN_TX + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: IN_TX + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null diff --git a/models/silver/silver__pending_liquidity_events.sql b/models/silver/silver__pending_liquidity_events.sql new file mode 100644 index 0000000..98626d8 --- /dev/null +++ b/models/silver/silver__pending_liquidity_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset_tx AS asset_tx_id, + asset_chain AS asset_blockchain, + asset_addr AS asset_address, + asset_e8, + cacao_tx AS cacao_tx_ID, + cacao_addr AS cacao_address, + cacao_e8, + pending_type, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__pending_liquidity_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, asset_addr, cacao_addr +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__pending_liquidity_events.yml b/models/silver/silver__pending_liquidity_events.yml new file mode 100644 index 0000000..1d31e85 --- /dev/null +++ b/models/silver/silver__pending_liquidity_events.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__pending_liquidity_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - ASSET_ADDRESS + - CACAO_ADDRESS + columns: + - name: POOL_NAME + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: CACAO_ADDRESS + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: PENDING_TYPE + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__pool_balance_change_events.sql b/models/silver/silver__pool_balance_change_events.sql new file mode 100644 index 0000000..e7383f1 --- /dev/null +++ b/models/silver/silver__pool_balance_change_events.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + cacao_amt AS cacao_amount, + cacao_add, + asset_amt AS asset_amount, + asset_add, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__pool_balance_change_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, asset +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__pool_balance_change_events.yml b/models/silver/silver__pool_balance_change_events.yml new file mode 100644 index 0000000..cde412c --- /dev/null +++ b/models/silver/silver__pool_balance_change_events.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__pool_balance_change_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: CACAO_AMOUNT + data_tests: + - not_null + - name: CACAO_ADD + data_tests: + - not_null + - name: ASSET_AMOUNT + data_tests: + - not_null + - name: ASSET_ADD + data_tests: + - not_null + - name: REASON + data_tests: + - not_null + diff --git a/models/silver/silver__pool_block_balances.sql b/models/silver/silver__pool_block_balances.sql new file mode 100644 index 0000000..a65f181 --- /dev/null +++ b/models/silver/silver__pool_block_balances.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + unique_key = '_unique_key', + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + b.block_timestamp, + b.height AS block_id, + bpd.pool_name, + COALESCE(cacao_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(cacao_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + COALESCE(asset_e8 / pow(10, 8), 0) AS asset_amount, + COALESCE(asset_e8 / pow(10, 8) * asset_usd, 0) AS asset_amount_usd, + COALESCE(synth_e8 / pow(10, 8), 0) AS synth_amount, + COALESCE(synth_e8 / pow(10, 8) * asset_usd, 0) AS synth_amount_usd, + concat_ws( + '-', + bpd.block_timestamp, + bpd.pool_name + ) AS _unique_key, + bpd._inserted_timestamp +FROM + {{ ref('silver__block_pool_depths') }} + bpd + JOIN {{ ref('silver__block_log') }} + b + ON bpd.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND bpd.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__pool_block_balances.yml b/models/silver/silver__pool_block_balances.yml new file mode 100644 index 0000000..6ed6998 --- /dev/null +++ b/models/silver/silver__pool_block_balances.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: silver__pool_block_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - POOL_NAME + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= current_date -2 AND BLOCK_TIMESTAMP >= '2021-04-13' + - name: ASSET_AMOUNT + data_tests: + - not_null + - name: ASSET_AMOUNT_USD + data_tests: + - not_null + - name: SYNTH_AMOUNT + data_tests: + - not_null + - name: SYNTH_AMOUNT_USD + data_tests: + - not_null diff --git a/models/silver/silver__pool_block_fees.sql b/models/silver/silver__pool_block_fees.sql new file mode 100644 index 0000000..1ed1692 --- /dev/null +++ b/models/silver/silver__pool_block_fees.sql @@ -0,0 +1,200 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.DAY >= (select min(DAY) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH all_block_id AS ( + + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + MAX( + A._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + pool_name +), +total_pool_rewards_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + SUM(cacao_e8) AS rewards + FROM + {{ ref('silver__rewards_event_entries') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + pool_name +), +total_liquidity_fees_CACAO_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + SUM(liq_fee_in_CACAO_e8) AS total_liquidity_fees_CACAO + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + pool_name +), +liquidity_fees_asset_tbl AS ( + SELECT + DATE(block_timestamp) AS DAY, + pool_name, + SUM(asset_fee) AS assetLiquidityFees + FROM + ( + SELECT + b.block_timestamp, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +GROUP BY + DAY, + pool_name +), +liquidity_fees_cacao_tbl AS ( + SELECT + DATE(block_timestamp) AS DAY, + pool_name, + SUM(asset_fee) AS cacaoLiquidityFees + FROM + ( + SELECT + b.block_timestamp, + pool_name, + CASE + WHEN to_asset <> 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +GROUP BY + DAY, + pool_name +) +SELECT + A.day, + A.pool_name, + COALESCE((rewards / power(10, 8)), 0) AS rewards, + COALESCE((total_liquidity_fees_cacao / power(10, 8)), 0) AS total_liquidity_fees_cacao, + COALESCE((assetLiquidityFees / power(10, 8)), 0) AS asset_liquidity_fees, + COALESCE((cacaoLiquidityFees / power(10, 8)), 0) AS cacao_liquidity_fees, + ( + (COALESCE(total_liquidity_fees_cacao, 0) + COALESCE(rewards, 0)) / power( + 10, + 8 + ) + ) AS earnings, + concat_ws( + '-', + A.day, + A.pool_name + ) AS _unique_key, + _inserted_timestamp +FROM + all_block_id A + LEFT JOIN total_pool_rewards_tbl b + ON A.day = b.day + AND A.pool_name = b.pool_name + LEFT JOIN total_liquidity_fees_cacao_tbl C + ON A.day = C.day + AND A.pool_name = C.pool_name + LEFT JOIN liquidity_fees_asset_tbl d + ON A.day = d.day + AND A.pool_name = d.pool_name + LEFT JOIN liquidity_fees_cacao_tbl e + ON A.day = e.day + AND A.pool_name = e.pool_name diff --git a/models/silver/silver__pool_block_fees.yml b/models/silver/silver__pool_block_fees.yml new file mode 100644 index 0000000..be00203 --- /dev/null +++ b/models/silver/silver__pool_block_fees.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: silver__pool_block_fees + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + - POOL_NAME + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: POOL_NAME + data_tests: + - not_null + - name: REWARDS + data_tests: + - not_null + - name: TOTAL_LIQUIDITY_FEES_cacao + data_tests: + - not_null + - name: ASSET_LIQUIDITY_FEES + data_tests: + - not_null + - name: cacao_LIQUIDITY_FEES + data_tests: + - not_null + - name: EARNINGS + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__pool_block_statistics.sql b/models/silver/silver__pool_block_statistics.sql new file mode 100644 index 0000000..9516a30 --- /dev/null +++ b/models/silver/silver__pool_block_statistics.sql @@ -0,0 +1,586 @@ +{{ config( + materialized = 'table', + cluster_by = ['day'] +) }} + +WITH pool_depth AS ( + + SELECT + DAY, + pool_name, + cacao_e8 AS cacao_depth, + asset_e8 AS asset_depth, + synth_e8 AS synth_depth, + cacao_e8 / NULLIF( + asset_e8, + 0 + ) AS asset_price + FROM + ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + b.height AS block_id, + pool_name, + cacao_e8, + synth_e8, + asset_e8, + MAX( + b.height + ) over (PARTITION BY pool_name, DATE(b.block_timestamp)) AS max_block_id + FROM + {{ ref("silver__block_pool_depths") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + asset_e8 > 0 + ) + WHERE + block_id = max_block_id +), +pool_status AS ( + SELECT + DAY, + pool_name, + status + FROM + ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + asset AS pool_name, + status, + ROW_NUMBER() over (PARTITION BY pool_name, DATE(b.block_timestamp) + ORDER BY + b.block_timestamp DESC, status) AS rn + FROM + {{ ref("silver__pool_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + WHERE + rn = 1 +), +add_liquidity_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + COUNT(*) AS add_liquidity_count, + SUM(cacao_e8) AS add_cacao_liquidity_volume, + SUM(asset_e8) AS add_asset_liquidity_volume, + SUM(stake_units) AS added_stake + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY, + pool_name +), +withdraw_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + COUNT(*) AS withdraw_count, + SUM(emit_cacao_e8) AS withdraw_cacao_volume, + SUM(emit_asset_e8) AS withdraw_asset_volume, + SUM(stake_units) AS withdrawn_stake, + SUM(imp_loss_protection_e8) AS impermanent_loss_protection_paid + FROM + {{ ref("silver__withdraw_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY, + pool_name +), +swap_total_tbl AS ( + SELECT + DAY, + pool_name, + SUM(volume) AS swap_volume + FROM + ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN to_e8 + ELSE from_e8 + END AS volume + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + GROUP BY + DAY, + pool_name +), +swap_to_asset_tbl AS ( + SELECT + DAY, + pool_name, + SUM(liq_fee_in_cacao_e8) AS to_asset_fees, + SUM(from_e8) AS to_asset_volume, + COUNT(*) AS to_asset_count, + AVG(swap_slip_bp) AS to_asset_average_slip + FROM( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 'to_cacao' + ELSE 'to_asset' + END AS to_tune_asset, + liq_fee_in_cacao_e8, + to_e8, + from_e8, + swap_slip_bp, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + GROUP BY + to_tune_asset, + pool_name, + DAY + HAVING + to_tune_asset = 'to_asset' +), +swap_to_cacao_tbl AS ( + SELECT + DAY, + pool_name, + SUM(liq_fee_in_cacao_e8) AS to_cacao_fees, + SUM(to_e8) AS to_cacao_volume, + COUNT(*) AS to_cacao_count, + AVG(swap_slip_bp) AS to_cacao_average_slip + FROM( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 'to_cacao' + ELSE 'to_asset' + END AS to_tune_asset, + liq_fee_in_cacao_e8, + to_e8, + from_e8, + swap_slip_bp, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + GROUP BY + to_tune_asset, + pool_name, + DAY + HAVING + to_tune_asset = 'to_cacao' +), +average_slip_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + AVG(swap_slip_bp) AS average_slip + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + pool_name, + DAY +), +unique_swapper_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + COUNT( + DISTINCT from_address + ) AS unique_swapper_count + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + pool_name, + DAY +), +stake_amount AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + SUM(stake_units) AS units + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + pool_name, + DAY +), +unstake_umc AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + from_address AS address, + pool_name, + SUM(stake_units) AS unstake_liquidity_units + FROM + {{ ref("silver__withdraw_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + from_address, + pool_name, + DAY +), +stake_umc AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + cacao_address AS address, + pool_name, + SUM(stake_units) AS liquidity_units + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + cacao_address IS NOT NULL + GROUP BY + cacao_address, + pool_name, + DAY + UNION ALL + SELECT + DATE( + b.block_timestamp + ) AS DAY, + asset_address AS address, + pool_name, + SUM(stake_units) AS liquidity_units + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + asset_address IS NOT NULL + AND cacao_address IS NULL + GROUP BY + asset_address, + pool_name, + DAY +), +unique_member_count AS ( + SELECT + DAY, + pool_name, + COUNT( + DISTINCT address + ) AS unique_member_count + FROM + ( + SELECT + stake_umc.day, + stake_umc.pool_name, + stake_umc.address, + stake_umc.liquidity_units, + CASE + WHEN unstake_umc.unstake_liquidity_units IS NOT NULL THEN unstake_umc.unstake_liquidity_units + ELSE 0 + END AS unstake_liquidity_units + FROM + stake_umc + LEFT JOIN unstake_umc + ON stake_umc.address = unstake_umc.address + AND stake_umc.pool_name = unstake_umc.pool_name + ) + WHERE + liquidity_units - unstake_liquidity_units > 0 + GROUP BY + pool_name, + DAY +), +asset_price_usd_tbl AS ( + SELECT + DAY, + pool_name, + asset_usd AS asset_price_usd + FROM + ( + SELECT + DATE(block_timestamp) AS DAY, + block_id, + MAX(block_id) over (PARTITION BY pool_name, DATE(block_timestamp)) AS max_block_id, + pool_name, + asset_usd + FROM + {{ ref("silver__prices") }} + ) + WHERE + block_id = max_block_id +), +joined AS ( + SELECT + pool_depth.day AS DAY, + COALESCE( + add_asset_liquidity_volume, + 0 + ) AS add_asset_liquidity_volume, + COALESCE( + add_liquidity_count, + 0 + ) AS add_liquidity_count, + COALESCE( + ( + add_asset_liquidity_volume + add_cacao_liquidity_volume + ), + 0 + ) AS add_liquidity_volume, + COALESCE( + add_cacao_liquidity_volume, + 0 + ) AS add_cacao_liquidity_volume, + pool_depth.pool_name AS asset, + asset_depth, + COALESCE( + asset_price, + 0 + ) AS asset_price, + COALESCE( + asset_price_usd, + 0 + ) AS asset_price_usd, + COALESCE( + average_slip, + 0 + ) AS average_slip, + COALESCE( + impermanent_loss_protection_paid, + 0 + ) AS impermanent_loss_protection_paid, + COALESCE( + cacao_depth, + 0 + ) AS cacao_depth, + COALESCE( + synth_depth, + 0 + ) AS synth_depth, + COALESCE( + status, + 'no status' + ) AS status, + COALESCE((to_cacao_count + to_asset_count), 0) AS swap_count, + COALESCE( + swap_volume, + 0 + ) AS swap_volume, + COALESCE( + to_asset_average_slip, + 0 + ) AS to_asset_average_slip, + COALESCE( + to_asset_count, + 0 + ) AS to_asset_count, + COALESCE( + to_asset_fees, + 0 + ) AS to_asset_fees, + COALESCE( + to_asset_volume, + 0 + ) AS to_asset_volume, + COALESCE( + to_cacao_average_slip, + 0 + ) AS to_cacao_average_slip, + COALESCE( + to_cacao_count, + 0 + ) AS to_cacao_count, + COALESCE( + to_cacao_fees, + 0 + ) AS to_cacao_fees, + COALESCE( + to_cacao_volume, + 0 + ) AS to_cacao_volume, + COALESCE((to_cacao_fees + to_asset_fees), 0) AS totalFees, + COALESCE( + unique_member_count, + 0 + ) AS unique_member_count, + COALESCE( + unique_swapper_count, + 0 + ) AS unique_swapper_count, + COALESCE( + units, + 0 + ) AS units, + COALESCE( + withdraw_asset_volume, + 0 + ) AS withdraw_asset_volume, + COALESCE( + withdraw_count, + 0 + ) AS withdraw_count, + COALESCE( + withdraw_cacao_volume, + 0 + ) AS withdraw_cacao_volume, + COALESCE((withdraw_cacao_volume + withdraw_asset_volume), 0) AS withdraw_volume, + SUM(COALESCE(added_stake, 0) - COALESCE(withdrawn_stake, 0)) over ( + PARTITION BY pool_depth.pool_name + ORDER BY + pool_depth.day ASC + ) AS total_stake, + asset_depth * COALESCE( + cacao_depth, + 0 + ) AS depth_product, + total_stake * synth_depth / ((asset_depth * 2) - synth_depth) AS synth_units, + CASE + WHEN total_stake = 0 THEN 0 + WHEN depth_product < 0 THEN 0 + ELSE SQRT(depth_product) / ( + total_stake + synth_units + ) + END AS liquidity_unit_value_index + FROM + pool_depth + LEFT JOIN pool_status + ON pool_depth.pool_name = pool_status.pool_name + AND pool_depth.day = pool_status.day + LEFT JOIN add_liquidity_tbl + ON pool_depth.pool_name = add_liquidity_tbl.pool_name + AND pool_depth.day = add_liquidity_tbl.day + LEFT JOIN withdraw_tbl + ON pool_depth.pool_name = withdraw_tbl.pool_name + AND pool_depth.day = withdraw_tbl.day + LEFT JOIN swap_total_tbl + ON pool_depth.pool_name = swap_total_tbl.pool_name + AND pool_depth.day = swap_total_tbl.day + LEFT JOIN swap_to_asset_tbl + ON pool_depth.pool_name = swap_to_asset_tbl.pool_name + AND pool_depth.day = swap_to_asset_tbl.day + LEFT JOIN swap_to_cacao_tbl + ON pool_depth.pool_name = swap_to_cacao_tbl.pool_name + AND pool_depth.day = swap_to_cacao_tbl.day + LEFT JOIN unique_swapper_tbl + ON pool_depth.pool_name = unique_swapper_tbl.pool_name + AND pool_depth.day = unique_swapper_tbl.day + LEFT JOIN stake_amount + ON pool_depth.pool_name = stake_amount.pool_name + AND pool_depth.day = stake_amount.day + LEFT JOIN average_slip_tbl + ON pool_depth.pool_name = average_slip_tbl.pool_name + AND pool_depth.day = average_slip_tbl.day + LEFT JOIN unique_member_count + ON pool_depth.pool_name = unique_member_count.pool_name + AND pool_depth.day = unique_member_count.day + LEFT JOIN asset_price_usd_tbl + ON pool_depth.pool_name = asset_price_usd_tbl.pool_name + AND pool_depth.day = asset_price_usd_tbl.day +) +SELECT + DISTINCT DAY, + add_asset_liquidity_volume, + add_liquidity_count, + add_liquidity_volume, + add_cacao_liquidity_volume, + asset, + asset_depth, + asset_price, + asset_price_usd, + average_slip, + impermanent_loss_protection_paid, + cacao_depth, + status, + swap_count, + swap_volume, + to_asset_average_slip, + to_asset_count, + to_asset_fees, + to_asset_volume, + to_cacao_average_slip, + to_cacao_count, + to_cacao_fees, + to_cacao_volume, + totalFees, + unique_member_count, + unique_swapper_count, + units, + withdraw_asset_volume, + withdraw_count, + withdraw_cacao_volume, + withdraw_volume, + total_stake, + depth_product, + synth_units, + total_stake + synth_units AS pool_units, + liquidity_unit_value_index, + LAG( + liquidity_unit_value_index, + 1 + ) over ( + PARTITION BY asset + ORDER BY + DAY ASC + ) AS prev_liquidity_unit_value_index, + concat_ws( + '-', + DAY, + asset + ) AS _unique_key +FROM + joined diff --git a/models/silver/silver__pool_block_statistics.yml b/models/silver/silver__pool_block_statistics.yml new file mode 100644 index 0000000..45668e7 --- /dev/null +++ b/models/silver/silver__pool_block_statistics.yml @@ -0,0 +1,125 @@ +version: 2 +models: + - name: silver__pool_block_statistics + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + - ASSET + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: ADD_ASSET_LIQUIDITY_VOLUME + data_tests: + - not_null + - name: ADD_LIQUIDITY_COUNT + data_tests: + - not_null + - name: ADD_LIQUIDITY_VOLUME + data_tests: + - not_null + - name: ADD_cacao_LIQUIDITY_VOLUME + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_DEPTH + data_tests: + - not_null + - name: ASSET_PRICE + data_tests: + - not_null + - name: ASSET_PRICE_USD + data_tests: + - not_null + - name: AVERAGE_SLIP + data_tests: + - not_null + - name: IMPERMANENT_LOSS_PROTECTION_PAID + data_tests: + - not_null + - name: cacao_DEPTH + data_tests: + - not_null + - name: STATUS + data_tests: + - not_null + - name: SWAP_COUNT + data_tests: + - not_null + - name: SWAP_VOLUME + data_tests: + - not_null + - name: TO_ASSET_AVERAGE_SLIP + data_tests: + - not_null + - name: TO_ASSET_COUNT + data_tests: + - not_null + - name: TO_ASSET_FEES + data_tests: + - not_null + - name: TO_ASSET_VOLUME + data_tests: + - not_null + - name: TO_cacao_AVERAGE_SLIP + data_tests: + - not_null + - name: TO_cacao_COUNT + data_tests: + - not_null + - name: TO_cacao_FEES + data_tests: + - not_null + - name: TO_cacao_VOLUME + data_tests: + - not_null + - name: TOTALFEES + data_tests: + - not_null + - name: UNIQUE_MEMBER_COUNT + data_tests: + - not_null + - name: UNIQUE_SWAPPER_COUNT + data_tests: + - not_null + - name: UNITS + data_tests: + - not_null + - name: WITHDRAW_ASSET_VOLUME + data_tests: + - not_null + - name: WITHDRAW_COUNT + data_tests: + - not_null + - name: WITHDRAW_cacao_VOLUME + data_tests: + - not_null + - name: WITHDRAW_VOLUME + data_tests: + - not_null + - name: TOTAL_STAKE + data_tests: + - not_null + - name: DEPTH_PRODUCT + data_tests: + - not_null + - name: SYNTH_UNITS + data_tests: + - not_null + - name: POOL_UNITS + data_tests: + - not_null + - name: LIQUIDITY_UNIT_VALUE_INDEX + data_tests: + - not_null + - name: PREV_LIQUIDITY_UNIT_VALUE_INDEX + data_tests: + - not_null: + where: day <= '2021-04-10' + diff --git a/models/silver/silver__pool_events.sql b/models/silver/silver__pool_events.sql new file mode 100644 index 0000000..81c93b5 --- /dev/null +++ b/models/silver/silver__pool_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + status, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__pool_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__prices.sql b/models/silver/silver__prices.sql new file mode 100644 index 0000000..ee0e7a2 --- /dev/null +++ b/models/silver/silver__prices.sql @@ -0,0 +1,88 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} +-- block level prices by pool +-- step 1 what is the USD pool with the highest balance (aka deepest pool) +WITH blocks AS ( + + SELECT + height AS block_id, + b.block_timestamp, + pool_name, + cacao_e8, + asset_e8 + FROM + {{ ref('silver__block_pool_depths') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +), +price AS ( + SELECT + height AS block_id, + b.block_timestamp, + cacao_price_e8 AS cacao_usd + FROM + {{ ref('silver__cacao_price') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +) -- step 3 calculate the prices of assets by pool, in terms of tokens per tokens +-- and in USD for both tokens +SELECT + DISTINCT b.block_id, + b.block_timestamp, + COALESCE( + cacao_e8 / asset_e8, + 0 + ) AS price_cacao_asset, + COALESCE( + asset_e8 / cacao_e8, + 0 + ) AS price_asset_cacao, + COALESCE(cacao_usd * (cacao_e8 / asset_e8), 0) AS asset_usd, + COALESCE( + cacao_usd, + 0 + ) AS cacao_usd, + pool_name, + concat_ws( + '-', + b.block_id :: STRING, + pool_name :: STRING + ) AS _unique_key +FROM + blocks b + JOIN price ru + ON b.block_id = ru.block_id +WHERE + cacao_e8 > 0 + AND asset_e8 > 0 diff --git a/models/silver/silver__prices.yml b/models/silver/silver__prices.yml new file mode 100644 index 0000000..206b7da --- /dev/null +++ b/models/silver/silver__prices.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: silver__prices + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - POOL_NAME + columns: + - name: BLOCK_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: PRICE_cacao_ASSET + data_tests: + - not_null + - name: PRICE_ASSET_cacao + data_tests: + - not_null + - name: ASSET_USD + data_tests: + - not_null + - name: cacao_USD + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null diff --git a/models/silver/silver__refund_events.sql b/models/silver/silver__refund_events.sql new file mode 100644 index 0000000..dff481a --- /dev/null +++ b/models/silver/silver__refund_events.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__refund_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__refund_events.yml b/models/silver/silver__refund_events.yml new file mode 100644 index 0000000..e454676 --- /dev/null +++ b/models/silver/silver__refund_events.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: silver__refund_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - ASSET + - ASSET_2ND + - MEMO + - CODE + - REASON + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: ASSET_2ND_E8 + data_tests: + - not_null + - name: CODE + data_tests: + - not_null + - name: REASON + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__reserve_events.sql b/models/silver/silver__reserve_events.sql new file mode 100644 index 0000000..e946a1b --- /dev/null +++ b/models/silver/silver__reserve_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + addr AS address, + e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__reserve_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__rewards_event_entries.sql b/models/silver/silver__rewards_event_entries.sql new file mode 100644 index 0000000..b677d24 --- /dev/null +++ b/models/silver/silver__rewards_event_entries.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + cacao_e8, + saver_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__rewards_event_entries') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, pool +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__rewards_event_entries.yml b/models/silver/silver__rewards_event_entries.yml new file mode 100644 index 0000000..f61e874 --- /dev/null +++ b/models/silver/silver__rewards_event_entries.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__rewards_event_entries + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + - POOL_NAME + - EVENT_ID + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__rewards_events.sql b/models/silver/silver__rewards_events.sql new file mode 100644 index 0000000..bb8666e --- /dev/null +++ b/models/silver/silver__rewards_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__rewards_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__rewards_events.yml b/models/silver/silver__rewards_events.yml new file mode 100644 index 0000000..c5ae267 --- /dev/null +++ b/models/silver/silver__rewards_events.yml @@ -0,0 +1,14 @@ +version: 2 +models: + - name: silver__rewards_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: BOND_E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__send_messages.sql b/models/silver/silver__send_messages.sql new file mode 100644 index 0000000..5b683ce --- /dev/null +++ b/models/silver/silver__send_messages.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + amount_e8, + asset, + from_addr AS from_address, + to_addr AS to_address, + memo, + tx_id, + raw_log, + code, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__send_messages') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__send_messages.yml b/models/silver/silver__send_messages.yml new file mode 100644 index 0000000..7b39b40 --- /dev/null +++ b/models/silver/silver__send_messages.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: silver__send_messages + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + columns: + - name: AMOUNT_E8 + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: MEMO + - name: TX_ID + data_tests: + - not_null + - name: EVENT_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__set_ip_address_events.sql b/models/silver/silver__set_ip_address_events.sql new file mode 100644 index 0000000..541df67 --- /dev/null +++ b/models/silver/silver__set_ip_address_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + ip_addr, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_ip_address_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__set_mimir_events.sql b/models/silver/silver__set_mimir_events.sql new file mode 100644 index 0000000..7565b4c --- /dev/null +++ b/models/silver/silver__set_mimir_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + key, + VALUE, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_mimir_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__set_node_keys_events.sql b/models/silver/silver__set_node_keys_events.sql new file mode 100644 index 0000000..e3ac62a --- /dev/null +++ b/models/silver/silver__set_node_keys_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + secp256k1, + ed25519, + validator_consensus, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_node_keys_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__set_node_mimir_events.sql b/models/silver/silver__set_node_mimir_events.sql new file mode 100644 index 0000000..4f59ae7 --- /dev/null +++ b/models/silver/silver__set_node_mimir_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + address, + key, + VALUE, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_node_mimir_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__set_version_events.sql b/models/silver/silver__set_version_events.sql new file mode 100644 index 0000000..a539760 --- /dev/null +++ b/models/silver/silver__set_version_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + version, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_version_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__slash_events.sql b/models/silver/silver__slash_events.sql new file mode 100644 index 0000000..c754bc2 --- /dev/null +++ b/models/silver/silver__slash_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset, + asset_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__slash_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, pool, asset +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__slash_liquidity_events.sql b/models/silver/silver__slash_liquidity_events.sql new file mode 100644 index 0000000..525270b --- /dev/null +++ b/models/silver/silver__slash_liquidity_events.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__slash_liquidity_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, bond_address, lp_address, asset +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__slash_points_events.sql b/models/silver/silver__slash_points_events.sql new file mode 100644 index 0000000..b535559 --- /dev/null +++ b/models/silver/silver__slash_points_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_address, + slash_points, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__slash_points_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__stake_events.sql b/models/silver/silver__stake_events.sql new file mode 100644 index 0000000..a905289 --- /dev/null +++ b/models/silver/silver__stake_events.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset_tx AS asset_tx_id, + asset_chain AS asset_blockchain, + asset_addr AS asset_address, + asset_e8, + stake_units, + cacao_tx AS cacao_tx_id, + cacao_addr AS cacao_address, + cacao_e8, + _ASSET_IN_CACAO_E8, + memo, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__stake_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, cacao_addr, asset_addr +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__stake_events.yml b/models/silver/silver__stake_events.yml new file mode 100644 index 0000000..d07d169 --- /dev/null +++ b/models/silver/silver__stake_events.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: silver__stake_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - POOL_NAME + - ASSET_TX_ID + - ASSET_BLOCKCHAIN + - ASSET_ADDRESS + - STAKE_UNITS + - CACAO_TX_ID + - CACAO_ADDRESS + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: POOL_NAME + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: STAKE_UNITS + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: _ASSET_IN_CACAO_E8 + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__streamling_swap_details_events.sql b/models/silver/silver__streamling_swap_details_events.sql new file mode 100644 index 0000000..7e6304e --- /dev/null +++ b/models/silver/silver__streamling_swap_details_events.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swap_reasons, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__streamling_swap_details_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__streamling_swap_details_events.yml b/models/silver/silver__streamling_swap_details_events.yml new file mode 100644 index 0000000..931a665 --- /dev/null +++ b/models/silver/silver__streamling_swap_details_events.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: silver__streamling_swap_details_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: INTERVAL + data_tests: + - not_null + - name: quantity + data_tests: + - not_null + - name: COUNT + data_tests: + - not_null + - name: last_height + data_tests: + - not_null + - name: deposit_asset + data_tests: + - not_null + - name: deposit_e8 + data_tests: + - not_null + - name: in_asset + data_tests: + - not_null + - name: in_e8 + data_tests: + - not_null + - name: out_asset + data_tests: + - not_null + - name: out_e8 + data_tests: + - not_null + - name: failed_swaps + - name: failed_swaps_reasons + - name: event_id + data_tests: + - not_null + - name: block_timestamp + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__swap_events.sql b/models/silver/silver__swap_events.sql new file mode 100644 index 0000000..9d7a2ef --- /dev/null +++ b/models/silver/silver__swap_events.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool AS pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + _STREAMING, + streaming_count, + streaming_quantity, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__swap_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__swap_events.yml b/models/silver/silver__swap_events.yml new file mode 100644 index 0000000..9c84f89 --- /dev/null +++ b/models/silver/silver__swap_events.yml @@ -0,0 +1,72 @@ +version: 2 +models: + - name: silver__swap_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - FROM_ASSET + - FROM_E8 + - TO_ASSET + - TO_E8 + - MEMO + - POOL_NAME + - TO_E8_MIN + - SWAP_SLIP_BP + - LIQ_FEE_E8 + - LIQ_FEE_IN_CACAO_E8 + - _DIRECTION + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: FROM_ASSET + data_tests: + - not_null + - name: FROM_E8 + data_tests: + - not_null + - name: TO_ASSET + data_tests: + - not_null + - name: TO_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: TO_E8_MIN + data_tests: + - not_null + - name: SWAP_SLIP_BP + data_tests: + - not_null + - name: LIQ_FEE_E8 + data_tests: + - not_null + - name: LIQ_FEE_IN_CACAO_E8 + data_tests: + - not_null + - name: _DIRECTION + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__swaps.sql b/models/silver/silver__swaps.sql new file mode 100644 index 0000000..4fc4804 --- /dev/null +++ b/models/silver/silver__swaps.sql @@ -0,0 +1,165 @@ +{{ config( + materialized = 'incremental', + unique_key = '_unique_key', + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH swaps AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + event_id, + streaming_count, + streaming_quantity, + b.block_timestamp, + b.height AS block_id, + A._INSERTED_TIMESTAMP, + COUNT(1) over ( + PARTITION BY tx_id + ) AS n_tx, + RANK() over ( + PARTITION BY tx_id + ORDER BY + liq_fee_e8 ASC + ) AS rank_liq_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + se.block_timestamp, + se.block_id, + tx_id, + blockchain, + se.pool_name, + from_address, + CASE + WHEN n_tx > 1 + AND rank_liq_fee = 1 + AND len(SPLIT(memo, ':') [4] :: STRING) = 43 THEN SPLIT( + memo, + ':' + ) [4] :: STRING + WHEN n_tx > 1 + AND LOWER(LEFT(memo, 1)) IN ( + 's', + '=' + ) + AND len(COALESCE(SPLIT(memo, ':') [2] :: STRING, '')) = 0 THEN from_address + ELSE SPLIT( + memo, + ':' + ) [2] :: STRING + END AS native_to_address, + to_address AS to_pool_address, + CASE + WHEN COALESCE(SPLIT(memo, ':') [4], '') = '' THEN NULL + WHEN CONTAINS(SPLIT(memo, ':') [4], '/') THEN SPLIT(SPLIT(memo, ':') [4], '/') [0] + ELSE SPLIT( + memo, + ':' + ) [4] :: STRING + END AS affiliate_address, + TRY_CAST( + CASE + WHEN COALESCE(SPLIT(memo, ':') [5], '') = '' THEN NULL + WHEN CONTAINS(SPLIT(memo, ':') [5], '/') THEN SPLIT(SPLIT(memo, ':') [5], '/') [0] + ELSE SPLIT( + memo, + ':' + ) [5] + END :: STRING AS INT + ) AS affiliate_fee_basis_points, + SPLIT(COALESCE(SPLIT(SPLIT(memo, '|') [0], ':') [4], ''), '/') AS affiliate_addresses_array, + ARRAY_AGG(TRY_CAST(TRIM(f.value) AS INTEGER)) within GROUP ( + ORDER BY + f.index + ) AS affiliate_fee_basis_points_array, + from_asset, + to_asset, + COALESCE(from_e8 / pow(10, 8), 0) AS from_amount, + COALESCE(to_e8 / pow(10, 8), 0) AS to_amount, + COALESCE(to_e8_min / pow(10, 8), 0) AS min_to_amount, + CASE + WHEN from_asset = 'MAYA.CACAO' THEN COALESCE(from_e8 * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(from_e8 * asset_usd / pow(10, 8), 0) + END AS from_amount_usd, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(to_e8 * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(to_e8 * asset_usd / pow(10, 8), 0) + END AS to_amount_usd, + cacao_usd, + asset_usd, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(to_e8_min * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(to_e8_min * asset_usd / pow(10, 8), 0) + END AS to_amount_min_usd, + swap_slip_bp, + COALESCE(liq_fee_in_cacao_e8 / pow(10, 8), 0) AS liq_fee_cacao, + COALESCE(liq_fee_in_cacao_e8 / pow(10, 8) * cacao_usd, 0) AS liq_fee_cacao_usd, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(liq_fee_e8 / pow(10, 8), 0) + ELSE COALESCE(liq_fee_e8 / pow(10, 8), 0) + END AS liq_fee_asset, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(liq_fee_e8 * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(liq_fee_e8 * asset_usd / pow(10, 8), 0) + END AS liq_fee_asset_usd, + streaming_count, + streaming_quantity, + concat_ws( + '-', + tx_id, + se.block_id, + to_asset, + from_asset, + COALESCE( + native_to_address, + '' + ), + from_address, + se.pool_name, + to_pool_address, + event_id + ) AS _unique_key, + _INSERTED_TIMESTAMP +FROM + swaps se + LEFT JOIN {{ ref('silver__prices') }} + p + ON se.block_id = p.block_id + AND se.pool_name = p.pool_name, + LATERAL FLATTEN( + input => SPLIT(COALESCE(SPLIT(memo, ':') [5], ''), '/') + ) f +GROUP BY + ALL diff --git a/models/silver/silver__swaps.yml b/models/silver/silver__swaps.yml new file mode 100644 index 0000000..4221eda --- /dev/null +++ b/models/silver/silver__swaps.yml @@ -0,0 +1,80 @@ +version: 2 +models: + - name: silver__swaps + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + - POOL_NAME + - FROM_ADDRESS + - TO_POOL_ADDRESS + - FROM_ASSET + - TO_ASSET + - NATIVE_TO_ADDRESS + - FROM_AMOUNT + - TO_AMOUNT + columns: + - name: BLOCK_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP::DATE + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_POOL_ADDRESS + data_tests: + - not_null + - name: FROM_ASSET + data_tests: + - not_null + - name: TO_ASSET + data_tests: + - not_null + - name: FROM_AMOUNT + data_tests: + - not_null + - name: TO_AMOUNT + data_tests: + - not_null + - name: MIN_TO_AMOUNT + data_tests: + - not_null + - name: FROM_AMOUNT_USD + data_tests: + - not_null + - name: TO_AMOUNT_USD + data_tests: + - not_null + - name: TO_AMOUNT_MIN_USD + data_tests: + - not_null + - name: SWAP_SLIP_BP + data_tests: + - not_null + - name: LIQ_FEE_cacao + data_tests: + - not_null + - name: LIQ_FEE_cacao_USD + data_tests: + - not_null + - name: LIQ_FEE_ASSET + data_tests: + - not_null + - name: LIQ_FEE_ASSET_USD + data_tests: + - not_null diff --git a/models/silver/silver__total_block_rewards.sql b/models/silver/silver__total_block_rewards.sql new file mode 100644 index 0000000..b2e7aa9 --- /dev/null +++ b/models/silver/silver__total_block_rewards.sql @@ -0,0 +1,137 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH block_prices AS ( + + SELECT + AVG(cacao_usd) AS cacao_usd, + block_id + FROM + {{ ref('silver__prices') }} + GROUP BY + block_id +), +fin AS ( + SELECT + b.block_timestamp, + b.height AS block_id, + ree.pool_name AS reward_entity, + COALESCE(cacao_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(cacao_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + concat_ws( + '-', + b.height, + reward_entity + ) AS _unique_key, + ree._inserted_timestamp + FROM + {{ ref('silver__rewards_event_entries') }} + ree + JOIN {{ ref('silver__block_log') }} + b + ON ree.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND ree.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + ( + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR concat_ws( + '-', + b.height, + reward_entity + ) IN ( + SELECT + _unique_key + FROM + {{ this }} + WHERE + cacao_amount_USD IS NULL + ) + ) +{% endif %} +UNION +SELECT + b.block_timestamp, + b.height AS block_id, + 'bond_holders' AS reward_entity, + bond_e8 / pow( + 10, + 8 + ) AS cacao_amount, + bond_e8 / pow( + 10, + 8 + ) * cacao_usd AS cacao_amount_usd, + concat_ws( + '-', + b.height, + reward_entity + ) AS _unique_key, + re._inserted_timestamp +FROM + {{ ref('silver__rewards_events') }} + re + JOIN {{ ref('silver__block_log') }} + b + ON re.block_timestamp = b.timestamp + LEFT JOIN block_prices p + ON b.height = p.block_id + +{% if is_incremental() %} +WHERE + ( + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR concat_ws( + '-', + b.height, + reward_entity + ) IN ( + SELECT + _unique_key + FROM + {{ this }} + WHERE + cacao_amount_USD IS NULL + ) + ) +{% endif %} +) +SELECT + block_timestamp, + block_id, + reward_entity, + SUM( + cacao_amount + ) AS cacao_amount, + SUM(cacao_amount_usd) AS cacao_amount_usd, + _unique_key, + MAX(_inserted_timestamp) AS _inserted_timestamp +FROM + fin +GROUP BY + block_timestamp, + block_id, + reward_entity, + _unique_key diff --git a/models/silver/silver__total_block_rewards.yml b/models/silver/silver__total_block_rewards.yml new file mode 100644 index 0000000..f80b0a5 --- /dev/null +++ b/models/silver/silver__total_block_rewards.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__total_block_rewards + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - REWARD_ENTITY + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + data_tests: + - not_null + - name: REWARD_ENTITY + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= CURRENT_DATE - 2 AND BLOCK_TIMESTAMP >= '2021-04-13' diff --git a/models/silver/silver__total_value_locked.sql b/models/silver/silver__total_value_locked.sql new file mode 100644 index 0000000..b1f6030 --- /dev/null +++ b/models/silver/silver__total_value_locked.sql @@ -0,0 +1,122 @@ +{{ config( + materialized = 'table', + cluster_by = ['day'] +) }} + +WITH bond_type_day AS ( + + SELECT + DATE( + b.block_timestamp + ) AS DAY, + bond_type, + (SUM(e8) / pow(10, 8)) AS cacao_amount, + MAX( + A._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ ref('silver__bond_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY, + bond_type +), +bond_type_day_direction AS ( + SELECT + DAY, + bond_type, + CASE + WHEN bond_type IN ( + 'bond_returned', + 'bond_cost' + ) THEN -1 + ELSE 1 + END AS direction, + cacao_amount, + cacao_amount * direction AS abs_CACAO_amount, + _inserted_timestamp + FROM + bond_type_day +), +total_value_bonded_tbl AS ( + SELECT + DAY, + SUM(abs_CACAO_amount) AS total_value_bonded, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + bond_type_day_direction + GROUP BY + DAY +), +total_pool_depth AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + b.height AS block_id, + pool_name, + cacao_e8, + asset_e8, + MAX(height) over (PARTITION BY pool_name, DATE(b.block_timestamp)) AS max_block_id, + A._inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + LOWER(pool_name) NOT LIKE 'thor.%' +), +total_pool_depth_max AS ( + SELECT + DAY, + cacao_e8 AS cacao_depth, + asset_e8 AS asset_depth, + _inserted_timestamp + FROM + total_pool_depth + WHERE + block_id = max_block_id +), +total_value_pooled_tbl AS ( + SELECT + DAY, + SUM(cacao_depth) * 2 / power( + 10, + 8 + ) AS total_value_pooled, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + total_pool_depth_max + GROUP BY + DAY +) +SELECT + COALESCE( + total_value_bonded_tbl.day, + total_value_pooled_tbl.day + ) AS DAY, + COALESCE( + total_value_pooled, + 0 + ) AS total_value_pooled, + COALESCE(SUM(total_value_bonded) over ( +ORDER BY + COALESCE(total_value_bonded_tbl.day, total_value_pooled_tbl.day) ASC), 0) AS total_value_bonded, + COALESCE( + total_value_pooled, + 0 + ) + SUM(COALESCE(total_value_bonded, 0)) over ( + ORDER BY + COALESCE( + total_value_bonded_tbl.day, + total_value_pooled_tbl.day + ) ASC + ) AS total_value_locked, + total_value_bonded_tbl._inserted_timestamp +FROM + total_value_bonded_tbl full + JOIN total_value_pooled_tbl + ON total_value_bonded_tbl.day = total_value_pooled_tbl.day diff --git a/models/silver/silver__total_value_locked.yml b/models/silver/silver__total_value_locked.yml new file mode 100644 index 0000000..7b44679 --- /dev/null +++ b/models/silver/silver__total_value_locked.yml @@ -0,0 +1,23 @@ +version: 2 +models: + - name: silver__total_value_locked + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: TOTAL_VALUE_POOLED + data_tests: + - not_null + - name: TOTAL_VALUE_BONDED + data_tests: + - not_null + - name: TOTAL_VALUE_LOCKED + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__transfer_events.sql b/models/silver/silver__transfer_events.sql new file mode 100644 index 0000000..e22b103 --- /dev/null +++ b/models/silver/silver__transfer_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + from_addr AS from_address, + to_addr AS to_address, + asset, + amount_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__transfer_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, from_addr, to_addr, asset, amount_e8 +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__transfer_events.yml b/models/silver/silver__transfer_events.yml new file mode 100644 index 0000000..0197937 --- /dev/null +++ b/models/silver/silver__transfer_events.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__transfer_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: AMOUNT_E8 + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql new file mode 100644 index 0000000..5fb8a73 --- /dev/null +++ b/models/silver/silver__transfers.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH block_prices AS ( + + SELECT + AVG(cacao_usd) AS cacao_usd, + block_id + FROM + {{ ref('silver__prices') }} + GROUP BY + block_id +) +SELECT + b.block_timestamp, + b.height AS block_id, + from_address, + to_address, + asset, + COALESCE(amount_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(amount_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + event_id, + concat_ws( + b.height :: STRING, + from_address :: STRING, + to_address :: STRING, + asset :: STRING, + event_id :: STRING + ) _unique_key, + se._inserted_timestamp +FROM + {{ ref('silver__transfer_events') }} + se + JOIN {{ ref('silver__block_log') }} + b + ON se.block_timestamp = b.timestamp + LEFT JOIN block_prices p + ON b.height = p.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__transfers.yml b/models/silver/silver__transfers.yml new file mode 100644 index 0000000..f3e7b72 --- /dev/null +++ b/models/silver/silver__transfers.yml @@ -0,0 +1,31 @@ +version: 2 +models: + - name: silver__transfers + + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: BLOCK_ID + data_tests: + - not_null + - name: ASSET + - name: FROM_ADDRESS + data_tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: maya1[0-9a-zA-Z]{39} + where: FROM_ADDRESS <> 'MidgardBalanceCorrectionAddress' + - name: TO_ADDRESS + data_tests: + - not_null + - dbt_expectations.expect_column_values_to_match_regex: + regex: maya1[0-9a-zA-Z]{39} + where: TO_ADDRESS <> 'MidgardBalanceCorrectionAddress' + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= SYSDATE() - interval '2 day' diff --git a/models/silver/silver__update_node_account_status_events.sql b/models/silver/silver__update_node_account_status_events.sql new file mode 100644 index 0000000..1865abc --- /dev/null +++ b/models/silver/silver__update_node_account_status_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + "CURRENT" AS current_status, + former AS former_status, + block_timestamp, + event_id, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__update_node_account_status_events') }} + e qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__update_node_account_status_events.yml b/models/silver/silver__update_node_account_status_events.yml new file mode 100644 index 0000000..73289b4 --- /dev/null +++ b/models/silver/silver__update_node_account_status_events.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__update_node_account_status_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NODE_ADDRESS + - CURRENT_STATUS + - FORMER_STATUS + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: NODE_ADDRESS + data_tests: + - not_null + - name: CURRENT_STATUS + data_tests: + - not_null + - name: FORMER_STATUS + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__validator_request_leave_events.sql b/models/silver/silver__validator_request_leave_events.sql new file mode 100644 index 0000000..6bd53ff --- /dev/null +++ b/models/silver/silver__validator_request_leave_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + from_addr AS from_address, + node_addr AS node_address, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__validator_request_leave_events') }} + e qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__validator_request_leave_events.yml b/models/silver/silver__validator_request_leave_events.yml new file mode 100644 index 0000000..9a3ff53 --- /dev/null +++ b/models/silver/silver__validator_request_leave_events.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: silver__validator_request_leave_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - TX_ID + - BLOCK_TIMESTAMP + - FROM_ADDRESS + - NODE_ADDRESS + columns: + - name: EVENT_ID + data_tests: + - not_null + - name: TX_ID + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: NODE_ADDRESS + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__withdraw_events.sql b/models/silver/silver__withdraw_events.sql new file mode 100644 index 0000000..9284337 --- /dev/null +++ b/models/silver/silver__withdraw_events.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + emit_asset_e8, + emit_cacao_e8, + memo, + pool AS pool_name, + stake_units, + basis_points, + asymmetry, + imp_loss_protection_e8, + _EMIT_ASSET_IN_CACAO_E8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref( + 'bronze__withdraw_events' + ) }} + e qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__withdraw_events.yml b/models/silver/silver__withdraw_events.yml new file mode 100644 index 0000000..35a8a9f --- /dev/null +++ b/models/silver/silver__withdraw_events.yml @@ -0,0 +1,64 @@ +version: 2 +models: + - name: silver__withdraw_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - ASSET + - MEMO + - POOL_NAME + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: EMIT_ASSET_E8 + data_tests: + - not_null + - name: EMIT_cacao_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: STAKE_UNITS + data_tests: + - not_null + - name: BASIS_POINTS + data_tests: + - not_null + - name: ASYMMETRY + data_tests: + - not_null + - name: IMP_LOSS_PROTECTION_E8 + data_tests: + - not_null + - name: _EMIT_ASSET_IN_cacao_E8 + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 0000000..5c1ce7f --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,57 @@ +version: 2 + +sources: + - name: maya_midgard + database: HEVO + schema: MAYA_MIDGARD + tables: + - name: midgard_active_vault_events + - name: midgard_add_events + - name: midgard_block_log + - name: midgard_block_pool_depths + - name: midgard_bond_events + - name: midgard_cacao_price + - name: midgard_constants + - name: midgard_failed_deposit_messages + - name: midgard_fee_events + - name: midgard_gas_events + - name: midgard_inactive_vault_events + - name: midgard_mayaname_change_events + - name: midgard_new_node_events + - name: midgard_outbound_events + - name: midgard_pending_liquidity_events + - name: midgard_pool_balance_change_events + - name: midgard_pool_events + - name: midgard_refund_events + - name: midgard_reserve_events + - name: midgard_rewards_events + - name: midgard_rewards_event_entries + - name: midgard_send_messages + - name: midgard_set_ip_address_events + - name: midgard_set_mimir_events + - name: midgard_set_node_keys_events + - name: midgard_set_node_mimir_events + - name: midgard_set_version_events + - name: midgard_slash_events + - name: midgard_slash_liquidity_events + - name: midgard_slash_points_events + - name: midgard_stake_events + - name: midgard_streaming_swap_details_events + - name: midgard_swap_events + - name: midgard_transfer_events + - name: midgard_update_node_account_status_events + - name: midgard_validator_request_leave_events + - name: midgard_withdraw_events + - name: crosschain + database: "{{ 'crosschain' if target.database == 'MAYA' else 'crosschain_dev' }}" + schema: core + tables: + - name: address_tags + - name: dim_dates + - name: crosschain_silver + database: "{{ 'crosschain' if target.database == 'MAYA' else 'crosschain_dev' }}" + schema: silver + tables: + - name: number_sequence + - name: complete_native_prices + - name: labels_combined \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..10b641c --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,16 @@ +packages: + - package: calogica/dbt_expectations + version: 0.8.5 + - git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: d3cf679e079f0cf06142de9386f215e55fe26b3b + - package: get-select/dbt_snowflake_query_tags + version: 2.5.0 + - package: dbt-labs/dbt_external_tables + version: 0.8.2 + - package: dbt-labs/dbt_utils + version: 1.0.0 + - package: calogica/dbt_date + version: 0.7.2 + - git: https://github.com/FlipsideCrypto/livequery-models.git + revision: b024188be4e9c6bc00ed77797ebdc92d351d620e +sha1_hash: 733d4f1fb94f4356106bab2f9af580d0898b3b50 diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..2ab7d3b --- /dev/null +++ b/packages.yml @@ -0,0 +1,11 @@ +packages: + - package: calogica/dbt_expectations + version: [">=0.4.0", "<0.9.0"] + - git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: v1.32.0 + - package: get-select/dbt_snowflake_query_tags + version: [">=2.0.0", "<3.0.0"] + - package: dbt-labs/dbt_external_tables + version: 0.8.2 + - package: dbt-labs/dbt_utils + version: 1.0.0 \ No newline at end of file diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..f69764a --- /dev/null +++ b/profiles.yml @@ -0,0 +1,29 @@ +maya: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + prod: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + config: + send_anonymous_usage_stats: False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..235f1a5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +dbt-core>=1.8,<1.9 +dbt-snowflake>=1.8,<1.9 +protobuf==4.25.3 \ No newline at end of file diff --git a/scripts/update_bronze_models.py b/scripts/update_bronze_models.py new file mode 100644 index 0000000..e69de29