diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml new file mode 100644 index 0000000..fc20ba8 --- /dev/null +++ b/.github/workflows/dbt_docs_update.yml @@ -0,0 +1,71 @@ +name: docs_update + +on: + push: + branches: + - "main" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: checkout docs branch + run: | + git checkout -B docs origin/main + - name: generate dbt docs + run: | + dbt ls -t prod + dbt docs generate --no-compile -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..435a87d --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,67 @@ +name: dbt_run_adhoc +run-name: dbt_run_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..32db163 --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,44 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + workflow_dispatch: +# schedule: +# - cron: '27 8 * * *' + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_incremental_core.yml b/.github/workflows/dbt_run_incremental_core.yml new file mode 100644 index 0000000..8242cdf --- /dev/null +++ b/.github/workflows/dbt_run_incremental_core.yml @@ -0,0 +1,46 @@ +name: dbt_run_incremental_core +run-name: dbt_run_incremental_core + +on: + workflow_dispatch: + schedule: + - cron: '*/95 * * * *' + + +env: + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -s "maya_models,tag:scheduled_core" + diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml new file mode 100644 index 0000000..52ad2b8 --- /dev/null +++ b/.github/workflows/dbt_test.yml @@ -0,0 +1,42 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 1 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10.x" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "maya_models,models/bronze" "maya_models,models/silver" "maya_models,models/gold" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2769f0f --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +target/ +dbt_modules/ +# newer versions of dbt use this directory instead of dbt_modules for test dependencies +dbt_packages/ +logs/ + +.venv/ +.python-version +dbt-env/ +venv/ + +# Visual Studio Code files +*/.vscode +*.code-workspace +.history/ +**/.DS_Store +.vscode/ +.env +.DS_Store +.user.yml \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..00ae875 --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +## Profile Set Up + +#### Use the following within profiles.yml +---- + +```yml +maya: + target: dev + outputs: + dev: + type: snowflake + account: + role: + user: + password: + region: + database: maya_DEV + warehouse: + schema: silver + threads: 4 + client_session_keep_alive: False + query_tag: +``` + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices + +## Applying Model Tags + +### Database / Schema level tags + +Database and schema tags are applied via the `add_database_or_schema_tags` macro. These tags are inherited by their downstream objects. To add/modify tags call the appropriate tag set function within the macro. + +``` +{{ set_database_tag_value('SOME_DATABASE_TAG_KEY','SOME_DATABASE_TAG_VALUE') }} +{{ set_schema_tag_value('SOME_SCHEMA_TAG_KEY','SOME_SCHEMA_TAG_VALUE') }} +``` + +### Model tags + +To add/update a model's snowflake tags, add/modify the `meta` model property under `config`. Only table level tags are supported at this time via DBT. + +``` +{{ config( + ..., + meta={ + 'database_tags':{ + 'table': { + 'PURPOSE': 'SOME_PURPOSE' + } + } + }, + ... +) }} +``` + +By default, model tags are not pushed to snowflake on each load. You can push a tag update for a model by specifying the `UPDATE_SNOWFLAKE_TAGS` project variable during a run. + +``` +dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.sql +``` + +### Querying for existing tags on a model in snowflake + +``` +select * +from table(maya.information_schema.tag_references('maya.core.fact_blocks', 'table')); +``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..1ff63ba --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,73 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "maya_models" +version: "1.0.0" +config-version: 2 + +require-dbt-version: ">=1.8.0" + +# This setting configures which "profile" dbt uses for this project. +profile: "maya" + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + - "dbt_packages" + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. +models: + +copy_grants: true + +persist_docs: + relation: true + columns: true + +on_schema_change: "append_new_columns" + livequery_models: + deploy: + core: + materialized: ephemeral + + +data_tests: + maya_models: # replace with the name of the chain + +store_failures: true # all tests + +on-run-start: + - '{{create_sps()}}' + - '{{create_udfs()}}' + + +dispatch: + - macro_namespace: dbt + search_order: + - maya-models + - dbt_snowflake_query_tags + - dbt + +query-comment: + comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' + append: true # Snowflake removes prefixed comments. + +vars: + "dbt_date:time_zone": GMT + OBSERV_FULL_TEST: False + START_GHA_TASKS: False + UPDATE_SNOWFLAKE_TAGS: True + UPDATE_UDFS_AND_SPS: True + diff --git a/docs/.gitkeep b/docs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/create_sps.sql b/macros/create_sps.sql new file mode 100644 index 0000000..3bdff2e --- /dev/null +++ b/macros/create_sps.sql @@ -0,0 +1,7 @@ +{% macro create_sps() %} + {% if target.database == 'MAYA' %} + CREATE schema IF NOT EXISTS _internal; +{{ sp_create_prod_clone('_internal') }}; + {% endif %} +{% endmacro %} +{% endmacro %} diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql new file mode 100644 index 0000000..56fe0ff --- /dev/null +++ b/macros/create_udfs.sql @@ -0,0 +1,2 @@ +{% macro create_udfs() %} +{% endmacro %} diff --git a/macros/custom_naming_macros.sql b/macros/custom_naming_macros.sql new file mode 100644 index 0000000..53fe8c7 --- /dev/null +++ b/macros/custom_naming_macros.sql @@ -0,0 +1,33 @@ +{% macro generate_schema_name( + custom_schema_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name [0] | trim }} +{%- endmacro %} + +{% macro generate_alias_name( + custom_alias_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {% if split_name | length < 2 %} + {{ split_name [0] | trim }} + {% else %} + {{ split_name [1] | trim }} + {% endif %} +{%- endmacro %} + +{% macro generate_tmp_view_name(model_name) -%} + {% set node_name = model_name.name %} + {% set split_name = node_name.split('__') %} + {{ target.database ~ '.' ~ split_name[0] ~ '.' ~ split_name [1] ~ '__dbt_tmp' | trim }} +{%- endmacro %} + +{% macro generate_view_name(model_name) -%} + {% set node_name = model_name.name %} + {% set split_name = node_name.split('__') %} + {{ target.database ~ '.' ~ split_name[0] ~ '.' ~ split_name [1] | trim }} +{%- endmacro %} \ No newline at end of file diff --git a/macros/custom_query_tag.sql b/macros/custom_query_tag.sql new file mode 100644 index 0000000..809e1bb --- /dev/null +++ b/macros/custom_query_tag.sql @@ -0,0 +1,11 @@ +{% macro set_query_tag() -%} + {% set new_json = {"repo":project_name, "object":this.table, "profile":target.profile_name, "env":target.name, "existing_tag":get_current_query_tag() } %} +{% set new_query_tag = tojson(new_json) | as_text %} + {% if new_query_tag %} + {% set original_query_tag = get_current_query_tag() %} + {{ log("Setting query_tag to '" ~ new_query_tag ~ "'. Will reset to '" ~ original_query_tag ~ "' after materialization.") }} + {% do run_query("alter session set query_tag = '{}'".format(new_query_tag)) %} + {{ return(original_query_tag)}} + {% endif %} + {{ return(none)}} +{% endmacro %} \ No newline at end of file diff --git a/macros/dbt/get_merge.sql b/macros/dbt/get_merge.sql new file mode 100644 index 0000000..7563d88 --- /dev/null +++ b/macros/dbt/get_merge.sql @@ -0,0 +1,30 @@ +-- incremental_strategy="merge" +{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {% set merge_sql = fsc_utils.get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %} + {{ return(merge_sql) }} +{% endmacro %} + +-- incremental_strategy="delete+insert" +{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%} + {% set predicate_override = "" %} + -- get the min value of column + {% if incremental_predicates[0] == "min_value_predicate" %} + {% set min_column_name = incremental_predicates[1] %} + {% set query %} + select min({{ min_column_name }}) from {{ source }} + {% endset %} + {% set min_block = run_query(query).columns[0][0] %} + + {% if min_block is not none %} + {% set predicate_override %} + round({{ target }}.{{ min_column_name }},-5) >= round({{ min_block }},-5) + {% endset %} + {% else %} + {% set predicate_override = "1=1" %} + {% endif %} + {% endif %} + {% set predicates = [predicate_override] + incremental_predicates[2:] if predicate_override else incremental_predicates %} + -- standard delete+insert from here + {% set merge_sql = dbt.get_delete_insert_merge_sql(target, source, unique_key, dest_columns, predicates) %} + {{ return(merge_sql) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/dbt/get_tmp_relation_type.sql b/macros/dbt/get_tmp_relation_type.sql new file mode 100644 index 0000000..e7c2d77 --- /dev/null +++ b/macros/dbt/get_tmp_relation_type.sql @@ -0,0 +1,4 @@ +{% macro dbt_snowflake_get_tmp_relation_type(strategy, unique_key, language) %} + -- always table + {{ return('table') }} +{% endmacro %} \ No newline at end of file diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql new file mode 100644 index 0000000..dff3c11 --- /dev/null +++ b/macros/run_sp_create_prod_clone.sql @@ -0,0 +1,10 @@ +{% macro run_sp_create_prod_clone() %} + {% set clone_query %} + call maya._internal.create_prod_clone( + 'maya', + 'maya_dev', + 'internal_dev' + ); +{% endset %} + {% do run_query(clone_query) %} +{% endmacro %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql new file mode 100644 index 0000000..b29c4d7 --- /dev/null +++ b/macros/sp_create_prod_clone.sql @@ -0,0 +1,49 @@ +{% macro sp_create_prod_clone(target_schema) -%} + +create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string) +returns boolean +language javascript +execute as caller +as +$$ + snowflake.execute({sqlText: `BEGIN TRANSACTION;`}); + try { + snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); + snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `REVOKE OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} FROM ROLE DBT_CLOUD_MAYA;`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE PROCEDURES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}) + + var existing_tags = snowflake.execute({sqlText: `SHOW TAGS IN DATABASE ${DESTINATION_DB_NAME};`}); + while (existing_tags.next()) { + var schema = existing_tags.getColumnValue(4); + var tag_name = existing_tags.getColumnValue(2) + snowflake.execute({sqlText: `GRANT OWNERSHIP ON TAG ${DESTINATION_DB_NAME}.${schema}.${tag_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + } + + snowflake.execute({sqlText: `COMMIT;`}); + } catch (err) { + snowflake.execute({sqlText: `ROLLBACK;`}); + throw(err); + } + + return true +$$ + +{%- endmacro %} \ No newline at end of file diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql new file mode 100644 index 0000000..fcff3d7 --- /dev/null +++ b/macros/tags/add_database_or_schema_tags.sql @@ -0,0 +1,6 @@ +{% macro add_database_or_schema_tags() %} + {{ set_database_tag_value( + 'BLOCKCHAIN_NAME', + 'MAYA' + ) }} +{% endmacro %} diff --git a/macros/tags/snowflake_tagging.sql b/macros/tags/snowflake_tagging.sql new file mode 100644 index 0000000..bc25e69 --- /dev/null +++ b/macros/tags/snowflake_tagging.sql @@ -0,0 +1,127 @@ +{% macro apply_meta_as_tags(results) %} + {% if var("UPDATE_SNOWFLAKE_TAGS") %} + {{ log('apply_meta_as_tags', info=False) }} + {{ log(results, info=False) }} + {% if execute %} + + {%- set tags_by_schema = {} -%} + {% for res in results -%} + {% if res.node.meta.database_tags %} + + {%- set model_database = res.node.database -%} + {%- set model_schema = res.node.schema -%} + {%- set model_schema_full = model_database+'.'+model_schema -%} + {%- set model_alias = res.node.alias -%} + + {% if model_schema_full not in tags_by_schema.keys() %} + {{ log('need to fetch tags for schema '+model_schema_full, info=False) }} + {%- call statement('main', fetch_result=True) -%} + show tags in {{model_database}}.{{model_schema}} + {%- endcall -%} + {%- set _ = tags_by_schema.update({model_schema_full: load_result('main')['table'].columns.get('name').values()|list}) -%} + {{ log('Added tags to cache', info=False) }} + {% else %} + {{ log('already have tag info for schema', info=False) }} + {% endif %} + + {%- set current_tags_in_schema = tags_by_schema[model_schema_full] -%} + {{ log('current_tags_in_schema:', info=False) }} + {{ log(current_tags_in_schema, info=False) }} + {{ log("========== Processing tags for "+model_schema_full+"."+model_alias+" ==========", info=False) }} + + {% set line -%} + node: {{ res.node.unique_id }}; status: {{ res.status }} (message: {{ res.message }}) + node full: {{ res.node}} + meta: {{ res.node.meta}} + materialized: {{ res.node.config.materialized }} + {%- endset %} + {{ log(line, info=False) }} + + {%- call statement('main', fetch_result=True) -%} + select LEVEL,UPPER(TAG_NAME) as TAG_NAME,TAG_VALUE from table(information_schema.tag_references_all_columns('{{model_schema}}.{{model_alias}}', 'table')) + {%- endcall -%} + {%- set existing_tags_for_table = load_result('main')['data'] -%} + {{ log('Existing tags for table:', info=False) }} + {{ log(existing_tags_for_table, info=False) }} + + {{ log('--', info=False) }} + {% for table_tag in res.node.meta.database_tags.table %} + + {{ create_tag_if_missing(current_tags_in_schema,table_tag|upper) }} + {% set desired_tag_value = res.node.meta.database_tags.table[table_tag] %} + + {{set_table_tag_value_if_different(model_schema,model_alias,table_tag,desired_tag_value,existing_tags_for_table)}} + {% endfor %} + {{ log("========== Finished processing tags for "+model_alias+" ==========", info=False) }} + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +{% endmacro %} + + +{% macro create_tag_if_missing(all_tag_names,table_tag) %} + {% if table_tag not in all_tag_names %} + {{ log('Creating missing tag '+table_tag, info=False) }} + {%- call statement('main', fetch_result=True) -%} + create tag if not exists silver.{{table_tag}} + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% else %} + {{ log('Tag already exists: '+table_tag, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_table_tag_value_if_different(model_schema,table_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at table level', info=False) }} + {%- set existing_tag_for_table = existing_tags|selectattr('0','equalto','TABLE')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for table:', info=False) }} + {{ log(existing_tag_for_table[0], info=False) }} + {% if existing_tag_for_table|length > 0 and existing_tag_for_table[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{model_schema}}.{{table_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_column_tag_value_if_different(table_name,column_name,tag_name,desired_tag_value,existing_tags) %} + {{ log('Ensuring tag '+tag_name+' has value '+desired_tag_value+' at column level', info=False) }} + {%- set existing_tag_for_column = existing_tags|selectattr('0','equalto','COLUMN')|selectattr('1','equalto',tag_name|upper)|list -%} + {{ log('Filtered tags for column:', info=False) }} + {{ log(existing_tag_for_column[0], info=False) }} + {% if existing_tag_for_column|length > 0 and existing_tag_for_column[0][2]==desired_tag_value %} + {{ log('Correct tag value already exists', info=False) }} + {% else %} + {{ log('Setting tag value for '+tag_name+' to value '+desired_tag_value, info=False) }} + {%- call statement('main', fetch_result=True) -%} + alter table {{table_name}} modify column {{column_name}} set tag {{tag_name}} = '{{desired_tag_value}}' + {%- endcall -%} + {{ log(load_result('main').data, info=False) }} + {% endif %} +{% endmacro %} + +{% macro set_database_tag_value(tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter database {{target.database}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} + +{% macro set_schema_tag_value(target_schema,tag_name,tag_value) %} + {% set query %} + create tag if not exists silver.{{tag_name}} + {% endset %} + {% do run_query(query) %} + {% set query %} + alter schema {{target.database}}.{{target_schema}} set tag {{target.database}}.silver.{{tag_name}} = '{{tag_value}}' + {% endset %} + {% do run_query(query) %} +{% endmacro %} \ No newline at end of file diff --git a/macros/tests/compare_model_subset.sql b/macros/tests/compare_model_subset.sql new file mode 100644 index 0000000..18aa624 --- /dev/null +++ b/macros/tests/compare_model_subset.sql @@ -0,0 +1,29 @@ +{% test compare_model_subset(model, compare_model, compare_columns, model_condition) %} + +{% set compare_cols_csv = compare_columns | join(', ') %} + +with a as ( + select {{compare_cols_csv}} from {{ model }} + {{ model_condition }} +), +b as ( + select {{compare_cols_csv}} from {{ compare_model }} +), +a_minus_b as ( + select * from a + except + select * from b +), +b_minus_a as ( + select * from b + except + select * from a +), +unioned as ( + select 'in_actual_not_in_expected' as which_diff, a_minus_b.* from a_minus_b + union all + select 'in_expected_not_in_actual' as which_diff, b_minus_a.* from b_minus_a +) +select * from unioned + +{% endtest %} \ No newline at end of file diff --git a/macros/tests/date_gaps.sql b/macros/tests/date_gaps.sql new file mode 100644 index 0000000..0f298a2 --- /dev/null +++ b/macros/tests/date_gaps.sql @@ -0,0 +1,58 @@ +{% macro date_gaps( + table, + partition_by, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + {{ table }} + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + DATEDIFF( + days, + {{ previous_column }}, + {{ column }} + ) - 1 AS gap +FROM + source + {% if varargs -%} +LEFT JOIN ( + {% for x in varargs %} + ( + {{ dbt_utils.date_spine( + datepart = "day", + start_date = x.start_date, + end_date = x.end_date + ) }} + ) + {{- "UNION ALL" if not loop.last -}} + {% endfor %} +) exclude + ON source.day = exclude.date_day + {%- endif %} +WHERE + DATEDIFF( + days, + {{ previous_column }}, + {{ column }} + ) > 1 + {{ "AND source.day != exclude.date_day" if varargs }} +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/negative_one.sql b/macros/tests/negative_one.sql new file mode 100644 index 0000000..22193aa --- /dev/null +++ b/macros/tests/negative_one.sql @@ -0,0 +1,10 @@ +{% test negative_one( + model, + column_name +) %} +SELECT + * +FROM + {{ model }} +WHERE + {{ column_name }} = '-1' {% endtest %} diff --git a/macros/tests/sequence_distinct_gaps.sql b/macros/tests/sequence_distinct_gaps.sql new file mode 100644 index 0000000..6cd0ccc --- /dev/null +++ b/macros/tests/sequence_distinct_gaps.sql @@ -0,0 +1,34 @@ +{% macro sequence_distinct_gaps( + table, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM ( + SELECT DISTINCT {{ column }} FROM {{ table }} + ) + + ) +SELECT + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/sequence_distinct_gaps_dim_block_id.sql b/macros/tests/sequence_distinct_gaps_dim_block_id.sql new file mode 100644 index 0000000..955ca66 --- /dev/null +++ b/macros/tests/sequence_distinct_gaps_dim_block_id.sql @@ -0,0 +1,43 @@ +{% macro sequence_distinct_gaps_dim_block_id( + table, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + ( + SELECT + DISTINCT {{ column }} + FROM + {{ table }} A + JOIN {{ ref('core__dim_block') }} + b + ON A.dim_block_id = b.dim_block_id + WHERE + A.dim_block_id <> '-1' + AND b.block_timestamp :: DATE < CURRENT_DATE + ) + ) +SELECT + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/sequence_gaps.sql b/macros/tests/sequence_gaps.sql new file mode 100644 index 0000000..84a9aa9 --- /dev/null +++ b/macros/tests/sequence_gaps.sql @@ -0,0 +1,37 @@ +{% macro sequence_gaps( + table, + partition_by, + column + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }} + FROM + {{ table }} + WHERE + block_timestamp::date <= current_date - 1 + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +ORDER BY + gap DESC +{% endmacro %} diff --git a/macros/tests/sequence_gaps_buffered_look_back.sql b/macros/tests/sequence_gaps_buffered_look_back.sql new file mode 100644 index 0000000..ea9deeb --- /dev/null +++ b/macros/tests/sequence_gaps_buffered_look_back.sql @@ -0,0 +1,52 @@ +{% macro sequence_gaps_buffered_look_back( + table, + partition_by, + column, + delayed_column, + delayed_period + ) %} + {%- set partition_sql = partition_by | join(", ") -%} + {%- set previous_column = "prev_" ~ column -%} + WITH source AS ( + SELECT + {{ partition_sql + "," if partition_sql }} + {{ column }}, + LAG( + {{ column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ previous_column }}, + LAG( + {{ delayed_column }}, + 1 + ) over ( + {{ "PARTITION BY " ~ partition_sql if partition_sql }} + ORDER BY + {{ column }} ASC + ) AS {{ delayed_column }} + FROM + {{ table }} + ) +SELECT + {{ partition_sql + "," if partition_sql }} + {{ previous_column }}, + {{ column }}, + {{ column }} - {{ previous_column }} + - 1 AS gap +FROM + source +WHERE + {{ column }} - {{ previous_column }} <> 1 +AND + {{ delayed_column }} < ( + SELECT + MAX( + {{ delayed_column }} + ) + FROM + {{ this }} + ) - INTERVAL '{{ delayed_period }}' +{% endmacro %} diff --git a/models/bronze/bronze__active_vault_events.sql b/models/bronze/bronze__active_vault_events.sql new file mode 100644 index 0000000..48fb269 --- /dev/null +++ b/models/bronze/bronze__active_vault_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_active_vault_events' + ) }} diff --git a/models/bronze/bronze__add_events.sql b/models/bronze/bronze__add_events.sql new file mode 100644 index 0000000..181a54d --- /dev/null +++ b/models/bronze/bronze__add_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + cacao_e8, + pool, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_add_events' + ) }} diff --git a/models/bronze/bronze__block_log.sql b/models/bronze/bronze__block_log.sql new file mode 100644 index 0000000..96c7c19 --- /dev/null +++ b/models/bronze/bronze__block_log.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + HASH, + height, + TIMESTAMP, + agg_state, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'maya_midgard', + 'midgard_block_log' + ) }} diff --git a/models/bronze/bronze__block_pool_depths.sql b/models/bronze/bronze__block_pool_depths.sql new file mode 100644 index 0000000..7a7f97e --- /dev/null +++ b/models/bronze/bronze__block_pool_depths.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset_e8, + cacao_e8, + synth_e8, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'maya_midgard', + 'midgard_block_pool_depths' + ) }} diff --git a/models/bronze/bronze__bond_events.sql b/models/bronze/bronze__bond_events.sql new file mode 100644 index 0000000..9cfa583 --- /dev/null +++ b/models/bronze/bronze__bond_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + bond_type, + e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'maya_midgard', + 'midgard_bond_events' + ) }} diff --git a/models/bronze/bronze__cacao_price.sql b/models/bronze/bronze__cacao_price.sql new file mode 100644 index 0000000..ac2c068 --- /dev/null +++ b/models/bronze/bronze__cacao_price.sql @@ -0,0 +1,16 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + cacao_price_e8, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_cacao_price' + ) }} diff --git a/models/bronze/bronze__constants.sql b/models/bronze/bronze__constants.sql new file mode 100644 index 0000000..c7715dd --- /dev/null +++ b/models/bronze/bronze__constants.sql @@ -0,0 +1,16 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + key, + VALUE, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_constants' + ) }} diff --git a/models/bronze/bronze__failed_deposit_messages.sql b/models/bronze/bronze__failed_deposit_messages.sql new file mode 100644 index 0000000..c03acb5 --- /dev/null +++ b/models/bronze/bronze__failed_deposit_messages.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + code, + memo, + asset, + amount_e8, + from_addr, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_failed_deposit_messages' + ) }} diff --git a/models/bronze/bronze__fee_events.sql b/models/bronze/bronze__fee_events.sql new file mode 100644 index 0000000..fa2a5a4 --- /dev/null +++ b/models/bronze/bronze__fee_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + asset, + asset_e8, + pool_deduct, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_fee_events' + ) }} diff --git a/models/bronze/bronze__gas_events.sql b/models/bronze/bronze__gas_events.sql new file mode 100644 index 0000000..8e12d1c --- /dev/null +++ b/models/bronze/bronze__gas_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + asset_e8, + cacao_e8, + tx_count, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_gas_events' + ) }} diff --git a/models/bronze/bronze__inactive_vault_events.sql b/models/bronze/bronze__inactive_vault_events.sql new file mode 100644 index 0000000..5984eec --- /dev/null +++ b/models/bronze/bronze__inactive_vault_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_inactive_vault_events' + ) }} diff --git a/models/bronze/bronze__mayaname_change_events.sql b/models/bronze/bronze__mayaname_change_events.sql new file mode 100644 index 0000000..28abe83 --- /dev/null +++ b/models/bronze/bronze__mayaname_change_events.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + NAME, + chain, + address, + registration_fee_e8, + fund_amount_e8, + height, + expire, + owner, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_mayaname_change_events' + ) }} diff --git a/models/bronze/bronze__new_node_events.sql b/models/bronze/bronze__new_node_events.sql new file mode 100644 index 0000000..2b87cef --- /dev/null +++ b/models/bronze/bronze__new_node_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_new_node_events' + ) }} diff --git a/models/bronze/bronze__outbound_events.sql b/models/bronze/bronze__outbound_events.sql new file mode 100644 index 0000000..124e2ff --- /dev/null +++ b/models/bronze/bronze__outbound_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + in_tx, + internal, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_outbound_events' + ) }} diff --git a/models/bronze/bronze__pending_liquidity_events.sql b/models/bronze/bronze__pending_liquidity_events.sql new file mode 100644 index 0000000..b3aaa74 --- /dev/null +++ b/models/bronze/bronze__pending_liquidity_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset_tx, + asset_chain, + asset_addr, + asset_e8, + cacao_tx, + cacao_addr, + cacao_e8, + pending_type, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_pending_liquidity_events' + ) }} diff --git a/models/bronze/bronze__pool_balance_change_events.sql b/models/bronze/bronze__pool_balance_change_events.sql new file mode 100644 index 0000000..6114c16 --- /dev/null +++ b/models/bronze/bronze__pool_balance_change_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + cacao_amt, + cacao_add, + asset_amt, + asset_add, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_pool_balance_change_events' + ) }} diff --git a/models/bronze/bronze__pool_events.sql b/models/bronze/bronze__pool_events.sql new file mode 100644 index 0000000..ef98d7e --- /dev/null +++ b/models/bronze/bronze__pool_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + status, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_pool_events' + ) }} diff --git a/models/bronze/bronze__refund_events.sql b/models/bronze/bronze__refund_events.sql new file mode 100644 index 0000000..c0226eb --- /dev/null +++ b/models/bronze/bronze__refund_events.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_refund_events' + ) }} diff --git a/models/bronze/bronze__reserve_events.sql b/models/bronze/bronze__reserve_events.sql new file mode 100644 index 0000000..7aef888 --- /dev/null +++ b/models/bronze/bronze__reserve_events.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + memo, + addr, + e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_reserve_events' + ) }} diff --git a/models/bronze/bronze__rewards_event_entries.sql b/models/bronze/bronze__rewards_event_entries.sql new file mode 100644 index 0000000..7ca83b6 --- /dev/null +++ b/models/bronze/bronze__rewards_event_entries.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + cacao_e8, + saver_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_rewards_event_entries' + ) }} diff --git a/models/bronze/bronze__rewards_events.sql b/models/bronze/bronze__rewards_events.sql new file mode 100644 index 0000000..6b35002 --- /dev/null +++ b/models/bronze/bronze__rewards_events.sql @@ -0,0 +1,17 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_rewards_events' + ) }} diff --git a/models/bronze/bronze__send_messages.sql b/models/bronze/bronze__send_messages.sql new file mode 100644 index 0000000..424aabb --- /dev/null +++ b/models/bronze/bronze__send_messages.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + amount_e8, + asset, + from_addr, + to_addr, + memo, + tx_id, + raw_log, + code, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_send_messages' + ) }} diff --git a/models/bronze/bronze__set_ip_address_events.sql b/models/bronze/bronze__set_ip_address_events.sql new file mode 100644 index 0000000..26c33ba --- /dev/null +++ b/models/bronze/bronze__set_ip_address_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + ip_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_ip_address_events' + ) }} diff --git a/models/bronze/bronze__set_mimir_events.sql b/models/bronze/bronze__set_mimir_events.sql new file mode 100644 index 0000000..de9592f --- /dev/null +++ b/models/bronze/bronze__set_mimir_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + key, + VALUE, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_mimir_events' + ) }} diff --git a/models/bronze/bronze__set_node_keys_events.sql b/models/bronze/bronze__set_node_keys_events.sql new file mode 100644 index 0000000..f9d3373 --- /dev/null +++ b/models/bronze/bronze__set_node_keys_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + secp256k1, + ed25519, + validator_consensus, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_node_keys_events' + ) }} diff --git a/models/bronze/bronze__set_node_mimir_events.sql b/models/bronze/bronze__set_node_mimir_events.sql new file mode 100644 index 0000000..4043ac7 --- /dev/null +++ b/models/bronze/bronze__set_node_mimir_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + address, + key, + VALUE, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_node_mimir_events' + ) }} diff --git a/models/bronze/bronze__set_version_events.sql b/models/bronze/bronze__set_version_events.sql new file mode 100644 index 0000000..75957a3 --- /dev/null +++ b/models/bronze/bronze__set_version_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + version, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_set_version_events' + ) }} diff --git a/models/bronze/bronze__slash_events.sql b/models/bronze/bronze__slash_events.sql new file mode 100644 index 0000000..2a84579 --- /dev/null +++ b/models/bronze/bronze__slash_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset, + asset_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_slash_events' + ) }} diff --git a/models/bronze/bronze__slash_liquidity_events.sql b/models/bronze/bronze__slash_liquidity_events.sql new file mode 100644 index 0000000..e3c12df --- /dev/null +++ b/models/bronze/bronze__slash_liquidity_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_slash_liquidity_events' + ) }} diff --git a/models/bronze/bronze__slash_points_events.sql b/models/bronze/bronze__slash_points_events.sql new file mode 100644 index 0000000..f8667e3 --- /dev/null +++ b/models/bronze/bronze__slash_points_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_address, + slash_points, + reason, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_slash_points_events' + ) }} diff --git a/models/bronze/bronze__stake_events.sql b/models/bronze/bronze__stake_events.sql new file mode 100644 index 0000000..e193afc --- /dev/null +++ b/models/bronze/bronze__stake_events.sql @@ -0,0 +1,27 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool, + asset_tx, + asset_chain, + asset_addr, + asset_e8, + stake_units, + cacao_tx, + cacao_addr, + cacao_e8, + _ASSET_IN_CACAO_E8, + memo, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_stake_events' + ) }} diff --git a/models/bronze/bronze__streamling_swap_details_events.sql b/models/bronze/bronze__streamling_swap_details_events.sql new file mode 100644 index 0000000..ae44289 --- /dev/null +++ b/models/bronze/bronze__streamling_swap_details_events.sql @@ -0,0 +1,29 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swap_reasons, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_streaming_swap_details_events' + ) }} diff --git a/models/bronze/bronze__swap_events.sql b/models/bronze/bronze__swap_events.sql new file mode 100644 index 0000000..ac43e78 --- /dev/null +++ b/models/bronze/bronze__swap_events.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + _STREAMING, + streaming_count, + streaming_quantity, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_swap_events' + ) }} diff --git a/models/bronze/bronze__transfer_events.sql b/models/bronze/bronze__transfer_events.sql new file mode 100644 index 0000000..ea7ab0c --- /dev/null +++ b/models/bronze/bronze__transfer_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + from_addr, + to_addr, + asset, + amount_e8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_transfer_events' + ) }} diff --git a/models/bronze/bronze__update_node_account_status_events.sql b/models/bronze/bronze__update_node_account_status_events.sql new file mode 100644 index 0000000..abf763f --- /dev/null +++ b/models/bronze/bronze__update_node_account_status_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr, + former, + "CURRENT", + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_update_node_account_status_events' + ) }} diff --git a/models/bronze/bronze__validator_request_leave_events.sql b/models/bronze/bronze__validator_request_leave_events.sql new file mode 100644 index 0000000..7648cd1 --- /dev/null +++ b/models/bronze/bronze__validator_request_leave_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + from_addr, + node_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_validator_request_leave_events' + ) }} diff --git a/models/bronze/bronze__withdraw_events.sql b/models/bronze/bronze__withdraw_events.sql new file mode 100644 index 0000000..71f6142 --- /dev/null +++ b/models/bronze/bronze__withdraw_events.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx, + chain, + from_addr, + to_addr, + asset, + asset_e8, + emit_asset_e8, + emit_cacao_e8, + memo, + pool, + stake_units, + basis_points, + asymmetry, + imp_loss_protection_e8, + _EMIT_ASSET_IN_CACAO_E8, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, +FROM + {{ source( + 'maya_midgard', + 'midgard_withdraw_events' + ) }} diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md new file mode 100644 index 0000000..6f7d34b --- /dev/null +++ b/models/descriptions/__overview__.md @@ -0,0 +1,123 @@ +{% docs __overview__ %} + +# Welcome to the Flipside Crypto Maya Models Documentation + +## **What does this documentation cover?** + +The documentation included here details the design of the Maya +tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/insights/dashboards/maya) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/maya-models) + +## **How do I use these docs?** + +The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. + +If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. + +There is more information on how to use dbt docs in the last section of this document. + +## **Quick Links to Table Documentation** + +**Click on the links below to jump to the documentation for each schema.** + +### Core Schema (`maya`.`CORE`.``) + +- [core.dim_block](#!/model/model.maya_models.core__dim_block) +- [core.dim_midgard](#!/model/model.maya_models.core__dim_midgard) +- [core.fact_mayaname_change_events](#!/model/model.maya_models.core__fact_mayaname_change_events) +- [core.fact_set_mimir_events](#!/model/model.maya_models.core__fact_set_mimir_events) +- [core.fact_transfer_events](#!/model/model.maya_models.core__fact_transfer_events) +- [core.fact_transfers](#!/model/model.maya_models.core__fact_transfers) + +### Defi Schema + +- [defi.fact_active_vault_events](#!/model/model.maya_models.defi__fact_active_vault_events) +- [defi.fact_add_events](#!/model/model.maya_models.defi__fact_add_events) +- [defi.fact_block_pool_depths](#!/model/model.maya_models.defi__fact_block_pool_depths) +- [defi.fact_block_rewards](#!/model/model.maya_models.defi__fact_block_rewards) +- [defi.fact_bond_actions](#!/model/model.maya_models.defi__fact_bond_actions) +- [defi.fact_bond_events](#!/model/model.maya_models.defi__fact_bond_events) +- [defi.fact_daily_earnings](#!/model/model.maya_models.defi__fact_daily_earnings) +- [defi.fact_daily_pool_stats](#!/model/model.maya_models.defi__fact_daily_pool_stats) +- [defi.fact_daily_tvl](#!/model/model.maya_models.defi__fact_daily_tvl) +- [defi.fact_failed_deposit_messages](#!/model/model.maya_models.defi__fact_failed_deposit_messages) +- [defi.fact_fee_events](#!/model/model.maya_models.defi__fact_fee_events) +- [defi.fact_gas_events](#!/model/model.maya_models.defi__fact_gas_events) +- [defi.fact_inactive_vault_events](#!/model/model.maya_models.defi__fact_inactive_vault_events) +- [defi.fact_liquidity_actions](#!/model/model.maya_models.defi__fact_liquidity_actions) +- [defi.fact_outbound_events](#!/model/model.maya_models.defi__fact_outbound_events) +- [defi.fact_pending_liquidity_events](#!/model/model.maya_models.defi__fact_pending_liquidity_events) +- [defi.fact_pool_balance_change_events](#!/model/model.maya_models.defi__fact_pool_balance_change_events) +- [defi.fact_pool_block_balances](#!/model/model.maya_models.defi__fact_pool_block_balances) +- [defi.fact_pool_block_fees](#!/model/model.maya_models.defi__fact_pool_block_fees) +- [defi.fact_pool_block_statistics](#!/model/model.maya_models.defi__fact_pool_block_statistics) +- [defi.fact_pool_events](#!/model/model.maya_models.defi__fact_pool_events) +- [defi.fact_refund_events](#!/model/model.maya_models.defi__fact_refund_events) +- [defi.fact_reserve_events](#!/model/model.maya_models.defi__fact_reserve_events) +- [defi.fact_rewards_event_entries](#!/model/model.maya_models.defi__fact_rewards_event_entries) +- [defi.fact_rewards_events](#!/model/model.maya_models.defi__fact_rewards_events) +- [defi.fact_send_message_events](#!/model/model.maya_models.defi__fact_send_message_events) +- [defi.fact_slash_liquidity_events](#!/model/model.maya_models.defi__fact_slash_liquidity_events) +- [defi.fact_stake_events](#!/model/model.maya_models.defi__fact_stake_events) +- [defi.fact_streamling_swap_details_events](#!/model/model.maya_models.defi__fact_streamling_swap_details_events) +- [defi.fact_swaps](#!/model/model.maya_models.defi__fact_swaps) +- [defi.fact_swaps_events](#!/model/model.maya_models.defi__fact_swaps_events) +- [defi.fact_total_block_rewards](#!/model/model.maya_models.defi__fact_total_block_rewards) +- [defi.fact_total_value_locked](#!/model/model.maya_models.defi__fact_total_value_locked) +- [defi.fact_update_node_account_status_events](#!/model/model.maya_models.defi__fact_update_node_account_status_events) +- [defi.fact_withdraw_events](#!/model/model.maya_models.defi__fact_withdraw_events) + +### Governance Schema + +- [gov.fact_new_node_events](#!/model/model.maya_models.gov__fact_new_node_events) +- [gov.fact_set_ip_address_events](#!/model/model.maya_models.gov__fact_set_ip_address_events) +- [gov.fact_set_node_keys_events](#!/model/model.maya_models.gov__fact_set_node_keys_events) +- [gov.fact_set_version_events](#!/model/model.maya_models.gov__fact_set_version_events) +- [gov.fact_slash_amounts](#!/model/model.maya_models.gov__fact_slash_amounts) +- [gov.fact_slash_points](#!/model/model.maya_models.gov__fact_slash_points) +- [gov.fact_validator_request_leave_events](#!/model/model.maya_models.gov__fact_validator_request_leave_events) + +### Price Schema + + - [price.factcacao_price](#!/model/model.maya_models.price__fact_cacao_price) + + +## **Data Model Overview** + +While maya models are built a few different ways, they are organized into three layers of sql models: **bronze, silver, and gold (or core).** + +- Bronze: Data is loaded in from the source as a view +- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here +- Gold (or core): Final views and tables that are available publicly + +The dimension tables are sourced from a variety of on-chain and off-chain sources. + +Convenience views (denoted ez\_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. + +## **Using dbt docs** + +### Navigation + +You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models in the project. + +### Database Tab + +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown in this interface, as they do not exist in the database. + +### Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. + +Once expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + +### **More information** + +- [maya on Flipside Crypto](https://flipsidecrypto.xyz/insights/dashboards/maya) +- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Github](https://github.com/FlipsideCrypto/maya-models) +- [What is dbt?](https://docs.getdbt.com/docs/introduction) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/address.md b/models/descriptions/address.md new file mode 100644 index 0000000..8ee2615 --- /dev/null +++ b/models/descriptions/address.md @@ -0,0 +1,5 @@ +{% docs address %} + +The account public key + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/asset.md b/models/descriptions/asset.md new file mode 100644 index 0000000..22564e6 --- /dev/null +++ b/models/descriptions/asset.md @@ -0,0 +1,5 @@ +{% docs asset %} + +Asset name or pool name + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_date.md b/models/descriptions/block_date.md new file mode 100644 index 0000000..3bc3977 --- /dev/null +++ b/models/descriptions/block_date.md @@ -0,0 +1,5 @@ +{% docs block_date %} + +Date of block minting(without a timezone) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_id.md b/models/descriptions/block_id.md new file mode 100644 index 0000000..86b272d --- /dev/null +++ b/models/descriptions/block_id.md @@ -0,0 +1,5 @@ +{% docs block_id %} + +ID of the confirmed block + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 0000000..1792c06 --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs block_timestamp %} + +Timestamp of block minting(without a timezone) + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/blockchain.md b/models/descriptions/blockchain.md new file mode 100644 index 0000000..b5098dc --- /dev/null +++ b/models/descriptions/blockchain.md @@ -0,0 +1,5 @@ +{% docs blockchain %} + +The name of the blockchain + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/bond_type.md b/models/descriptions/bond_type.md new file mode 100644 index 0000000..6675b27 --- /dev/null +++ b/models/descriptions/bond_type.md @@ -0,0 +1,5 @@ +{% docs bond_type %} + +There are 4 different types of bond:bond_reward, bond_paid, bond_cost, bond_returned + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/from_address.md b/models/descriptions/from_address.md new file mode 100644 index 0000000..97a82e3 --- /dev/null +++ b/models/descriptions/from_address.md @@ -0,0 +1,5 @@ +{% docs from_address %} + +The sending address for this event + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/id.md b/models/descriptions/id.md new file mode 100644 index 0000000..4075e01 --- /dev/null +++ b/models/descriptions/id.md @@ -0,0 +1,5 @@ +{% docs id %} + +The primary key for the table. + +{% enddocs %} diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md new file mode 100644 index 0000000..14703c3 --- /dev/null +++ b/models/descriptions/inserted_timestamp.md @@ -0,0 +1,5 @@ +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/memo.md b/models/descriptions/memo.md new file mode 100644 index 0000000..59ef667 --- /dev/null +++ b/models/descriptions/memo.md @@ -0,0 +1,5 @@ +{% docs memo %} + +The memo for this event + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/modified_timestamp.md b/models/descriptions/modified_timestamp.md new file mode 100644 index 0000000..c4d18ee --- /dev/null +++ b/models/descriptions/modified_timestamp.md @@ -0,0 +1,5 @@ +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/pk.md b/models/descriptions/pk.md new file mode 100644 index 0000000..8dfdfb5 --- /dev/null +++ b/models/descriptions/pk.md @@ -0,0 +1,5 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/pool_name.md b/models/descriptions/pool_name.md new file mode 100644 index 0000000..3a171b3 --- /dev/null +++ b/models/descriptions/pool_name.md @@ -0,0 +1,5 @@ +{% docs pool_name %} + +Name of the pool -- also asset name in other tables + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/prices.md b/models/descriptions/prices.md new file mode 100644 index 0000000..7c7f6a1 --- /dev/null +++ b/models/descriptions/prices.md @@ -0,0 +1,137 @@ +{% docs prices_dim_asset_metadata_table_doc %} + +A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. + +{% enddocs %} + +{% docs prices_ez_asset_metadata_table_doc %} + +A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. + +{% enddocs %} + +{% docs prices_fact_prices_ohlc_hourly_table_doc %} + +A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. + +{% enddocs %} + +{% docs prices_ez_prices_hourly_table_doc %} + +A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. + +{% enddocs %} + +{% docs prices_provider %} + +The provider or source of the data. + +{% enddocs %} + +{% docs prices_asset_id %} + +The unique identifier representing the asset. + +{% enddocs %} + +{% docs prices_name %} + +The name of asset. + +{% enddocs %} + +{% docs prices_symbol %} + +The symbol of asset. + +{% enddocs %} + +{% docs prices_token_address %} + +The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. + +{% enddocs %} + +{% docs prices_blockchain %} + +The Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_blockchain_id %} + +The unique identifier of the Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_decimals %} + +The number of decimals for the asset. May be NULL. + +{% enddocs %} + +{% docs prices_is_native %} + +A flag indicating assets native to the respective blockchain. + +{% enddocs %} + +{% docs prices_is_deprecated %} + +A flag indicating if the asset is deprecated or no longer supported by the provider. + +{% enddocs %} + +{% docs prices_id_deprecation %} + +Deprecating soon! Please use the `asset_id` column instead. + +{% enddocs %} + +{% docs prices_decimals_deprecation %} + +Deprecating soon! Please use the decimals column in `ez_asset_metadata` or join in `dim_contracts` instead. + +{% enddocs %} + +{% docs prices_hour %} + +Hour that the price was recorded at. + +{% enddocs %} + +{% docs prices_price %} + +Closing price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_is_imputed %} + +A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. + +{% enddocs %} + +{% docs prices_open %} + +Opening price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_high %} + +Highest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_low %} + +Lowest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_close %} + +Closing price of the recorded hour in USD + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sk.md b/models/descriptions/sk.md new file mode 100644 index 0000000..7a011e4 --- /dev/null +++ b/models/descriptions/sk.md @@ -0,0 +1,5 @@ +{% docs sk %} + +The surrogate key for the table. Will be unique and is used as a foreign key in other tables + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/to_address.md b/models/descriptions/to_address.md new file mode 100644 index 0000000..cda3c1b --- /dev/null +++ b/models/descriptions/to_address.md @@ -0,0 +1,5 @@ +{% docs to_address %} + +The receiving address for this event + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_id.md b/models/descriptions/tx_id.md new file mode 100644 index 0000000..18ac5e1 --- /dev/null +++ b/models/descriptions/tx_id.md @@ -0,0 +1,5 @@ +{% docs tx_id %} + +The unique transaction id + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_block.sql b/models/gold/core/core__dim_block.sql new file mode 100644 index 0000000..1f15154 --- /dev/null +++ b/models/gold/core/core__dim_block.sql @@ -0,0 +1,86 @@ +{{ config( + materialized = 'incremental', + unique_key = 'dim_block_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_id >= (select min(block_id) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + {{ dbt_utils.generate_surrogate_key( + ['height'] + ) }} AS dim_block_id, + height AS block_id, + block_timestamp, + block_date, + block_hour, + block_week, + block_month, + block_quarter, + block_year, + block_DAYOFMONTH, + block_DAYOFWEEK, + block_DAYOFYEAR, + TIMESTAMP, + HASH, + agg_state, + _INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + {{ ref('silver__block_log') }} + +{% if is_incremental() %} +WHERE + block_id >= ( + SELECT + MAX( + block_id - 600 --about 1 hour + ) + FROM + {{ this }} + ) +UNION ALL +SELECT + '-1' AS dim_block_id, + -1 AS block_id, + '1900-01-01' :: datetime AS block_timestamp, + NULL AS block_date, + NULL AS block_hour, + NULL AS block_week, + NULL AS block_month, + NULL AS block_quarter, + NULL AS block_year, + NULL AS block_DAYOFMONTH, + NULL AS block_DAYOFWEEK, + NULL AS block_DAYOFYEAR, + NULL AS TIMESTAMP, + NULL AS HASH, + NULL AS agg_state, + '1900-01-01' :: DATE AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + '1900-01-01' :: DATE AS inserted_timestamp, + '1900-01-01' :: DATE AS modified_timestamp +UNION ALL +SELECT + '-2' AS dim_block_id, + -2 AS block_id, + NULL AS block_timestamp, + NULL AS block_date, + NULL AS block_hour, + NULL AS block_week, + NULL AS block_month, + NULL AS block_quarter, + NULL AS block_year, + NULL AS block_DAYOFMONTH, + NULL AS block_DAYOFWEEK, + NULL AS block_DAYOFYEAR, + NULL AS TIMESTAMP, + NULL AS HASH, + NULL AS agg_state, + '1900-01-01' :: DATE AS _inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + '1900-01-01' :: DATE AS inserted_timestamp, + '1900-01-01' :: DATE AS modified_timestamp +{% endif %} diff --git a/models/gold/core/core__dim_block.yml b/models/gold/core/core__dim_block.yml new file mode 100644 index 0000000..7d9b23f --- /dev/null +++ b/models/gold/core/core__dim_block.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: core__dim_block + description: "Records of all blocks that have occurred on Thorchain, dating back to the genesis block. " + columns: + - name: DIM_BLOCK_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BLOCK_ID + description: "{{ doc('block_id') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_DATE + description: "{{ doc('block_date') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_HOUR + description: "{{ doc('block_date') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_WEEK + description: "{{ doc('block_date') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: BLOCK_MONTH + description: "The numeric month of block minting(without a timezone)" + - name: BLOCK_QUARTER + description: "The numeric quarter of block minting(without a timezone)" + - name: BLOCK_YEAR + description: "The numeric year of block minting(without a timezone)" + - name: BLOCK_DAYOFMONTH + description: "The numeric day of month of block minting(without a timezone)" + - name: BLOCK_DAYOFWEEK + description: "The numeric day of week of block minting(without a timezone)" + - name: BLOCK_DAYOFYEAR + description: "The numeric day of year of block minting(without a timezone)" + - name: TIMESTAMP + description: "integer value of the block_teimstamp" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: HASH + description: "block hash" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: agg_state + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: DIM_BLOCK_ID diff --git a/models/gold/core/core__dim_midgard.sql b/models/gold/core/core__dim_midgard.sql new file mode 100644 index 0000000..73df47a --- /dev/null +++ b/models/gold/core/core__dim_midgard.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + '2.10.0' AS midgard_version diff --git a/models/gold/core/core__fact_mayaname_change_events.sql b/models/gold/core/core__fact_mayaname_change_events.sql new file mode 100644 index 0000000..01e89f0 --- /dev/null +++ b/models/gold/core/core__fact_mayaname_change_events.sql @@ -0,0 +1,77 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_mayaname_change_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + NAME, + chain, + address, + registration_fee_e8, + fund_amount_e8, + height, + expire, + owner, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__mayaname_change_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.name'] + ) }} AS fact_mayaname_change_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + NAME, + chain, + address, + registration_fee_e8, + fund_amount_e8, + height, + expire, + owner, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + event_id, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_mayaname_change_events.yml b/models/gold/core/core__fact_mayaname_change_events.yml new file mode 100644 index 0000000..3764602 --- /dev/null +++ b/models/gold/core/core__fact_mayaname_change_events.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: core__fact_mayaname_change_events + description: "Fact table that shows name change events" + columns: + - name: FACT_MAYANAME_CHANGE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: OWNER + description: "" + - name: CHAIN + description: "{{ doc('blockchain') }}" + - name: ADDRESS + description: "{{ doc('address') }}" + - name: expire + description: "" + - name: NAME + description: "" + - name: FUND_AMOUNT_E8 + description: "" + - name: REGISTRATION_FEE_E8 + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_MAYANAME_CHANGE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/core/core__fact_set_mimir_events.sql b/models/gold/core/core__fact_set_mimir_events.sql new file mode 100644 index 0000000..958d8b0 --- /dev/null +++ b/models/gold/core/core__fact_set_mimir_events.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_mimir_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + key, + VALUE, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_mimir_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.key','a.block_timestamp'] + ) }} AS fact_set_mimir_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + key, + VALUE, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_set_mimir_events.yml b/models/gold/core/core__fact_set_mimir_events.yml new file mode 100644 index 0000000..e20ab40 --- /dev/null +++ b/models/gold/core/core__fact_set_mimir_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: core__fact_set_mimir_events + description: "Fact table that shows set mimir events" + columns: + - name: FACT_SET_MIMIR_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: KEY + description: "" + - name: VALUE + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_MIMIR_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/core/core__fact_transfer_events.sql b/models/gold/core/core__fact_transfer_events.sql new file mode 100644 index 0000000..3c10ba5 --- /dev/null +++ b/models/gold/core/core__fact_transfer_events.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_transfer_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + from_address, + to_address, + asset, + amount_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__transfer_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.from_address', 'a.to_address', 'a.asset', 'a.amount_e8'] + ) }} AS fact_transfer_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + from_address, + to_address, + asset, + amount_e8, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/core/core__fact_transfer_events.yml b/models/gold/core/core__fact_transfer_events.yml new file mode 100644 index 0000000..b7ef274 --- /dev/null +++ b/models/gold/core/core__fact_transfer_events.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: core__fact_transfer_events + description: "Fact table containing stake events" + columns: + - name: FACT_TRANSFER_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: FROM_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: AMOUNT_E8 + description: "The asset amount for this event" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TRANSFER_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/core/core__fact_transfers.sql b/models/gold/core/core__fact_transfers.sql new file mode 100644 index 0000000..a669a72 --- /dev/null +++ b/models/gold/core/core__fact_transfers.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_transfers_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + from_address, + to_address, + asset, + cacao_amount, + cacao_amount_usd, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__transfers') }} + +{% if is_incremental() %} +WHERE + block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_transfers_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + from_address, + to_address, + asset, + cacao_amount, + cacao_amount_usd, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id diff --git a/models/gold/core/core__fact_transfers.yml b/models/gold/core/core__fact_transfers.yml new file mode 100644 index 0000000..6b41db7 --- /dev/null +++ b/models/gold/core/core__fact_transfers.yml @@ -0,0 +1,52 @@ +version: 2 +models: + - name: core__fact_transfers + description: "Fact table shows the transfer action between different address" + columns: + - name: FACT_TRANSFERS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: FROM_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('address') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: cacao_AMOUNT + description: "The transferred cacao amount" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "The transferred cacao amount in USD" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TRANSFERS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_active_vault_events.sql b/models/gold/defi/defi__fact_active_vault_events.sql new file mode 100644 index 0000000..60f7b2b --- /dev/null +++ b/models/gold/defi/defi__fact_active_vault_events.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_active_vault_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + add_asgard_addr, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__active_vault_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp','a.add_asgard_addr'] + ) }} AS fact_active_vault_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + add_asgard_addr, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_active_vault_events.yml b/models/gold/defi/defi__fact_active_vault_events.yml new file mode 100644 index 0000000..bf40224 --- /dev/null +++ b/models/gold/defi/defi__fact_active_vault_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: defi__fact_active_vault_events + description: "Fact table containing the events triggered by the churning activities" + columns: + - name: FACT_ACTIVE_VAULT_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ADD_ASGARD_ADDR + description: "The asgard address added to the vault" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_ACTIVE_VAULT_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_add_events.sql b/models/gold/defi/defi__fact_add_events.sql new file mode 100644 index 0000000..d6ef799 --- /dev/null +++ b/models/gold/defi/defi__fact_add_events.sql @@ -0,0 +1,75 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_add_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + e.block_timestamp, + e.tx_id, + e.cacao_e8, + e.blockchain, + e.asset_e8, + e.pool_name, + e.memo, + e.to_address, + e.from_address, + e.asset, + e.event_id, + _inserted_timestamp + FROM + {{ ref('silver__add_events') }} + e +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.from_address','a.to_address','a.asset','a.memo','a.block_timestamp'] + ) }} AS fact_add_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + A.tx_id, + A.cacao_e8, + A.blockchain, + A.asset_e8, + A.pool_name, + A.memo, + A.to_address, + A.from_address, + A.asset, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_add_events.yml b/models/gold/defi/defi__fact_add_events.yml new file mode 100644 index 0000000..701cc9d --- /dev/null +++ b/models/gold/defi/defi__fact_add_events.yml @@ -0,0 +1,58 @@ +version: 2 +models: + - name: defi__fact_add_events + description: "Fact table containing add events" + columns: + - name: FACT_ADD_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: cacao_E8 + description: "The amount of cacao for this add event" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: ASSET_E8 + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_ADD_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_block_pool_depths.sql b/models/gold/defi/defi__fact_block_pool_depths.sql new file mode 100644 index 0000000..7bbcb36 --- /dev/null +++ b/models/gold/defi/defi__fact_block_pool_depths.sql @@ -0,0 +1,63 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_depths_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset_e8, + cacao_e8, + synth_e8, + block_timestamp, + _inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.pool_name','a.block_timestamp'] + ) }} AS fact_pool_depths_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + cacao_e8, + asset_e8, + synth_e8, + pool_name, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR pool_name IN ( + SELECT + pool_name + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_block_pool_depths.yml b/models/gold/defi/defi__fact_block_pool_depths.yml new file mode 100644 index 0000000..3713ed4 --- /dev/null +++ b/models/gold/defi/defi__fact_block_pool_depths.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_block_pool_depths + description: "Fact table containing all the available pools and its cacao/asset depth at each block interval" + columns: + - name: FACT_POOL_DEPTHS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: cacao_E8 + description: "The cacao depth for this pool at this block" + tests: + - not_null + - name: ASSET_E8 + description: "The asset depth for this pool at this block" + tests: + - not_null + - name: SYNTH_E8 + description: "" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_DEPTHS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_block_rewards.sql b/models/gold/defi/defi__fact_block_rewards.sql new file mode 100644 index 0000000..e5be0a7 --- /dev/null +++ b/models/gold/defi/defi__fact_block_rewards.sql @@ -0,0 +1,51 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_block_rewards_id', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge' +) }} + +WITH base AS ( + + SELECT + DAY, + liquidity_fee, + block_rewards, + earnings, + bonding_earnings, + liquidity_earnings, + avg_node_count, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__block_rewards') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_block_rewards_id, + DAY, + liquidity_fee, + block_rewards, + earnings, + bonding_earnings, + liquidity_earnings, + avg_node_count, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_block_rewards.yml b/models/gold/defi/defi__fact_block_rewards.yml new file mode 100644 index 0000000..3266bbb --- /dev/null +++ b/models/gold/defi/defi__fact_block_rewards.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: defi__fact_block_rewards + description: "The summarized rewards information for each block per day" + columns: + - name: FACT_BLOCK_REWARDS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The recorded day" + tests: + - not_null + - unique + - name: LIQUIDITY_FEE + description: "The summarized liquidity cost fee within this day" + tests: + - not_null + - name: BLOCK_REWARDS + description: "The summarized total block rewards within this day" + tests: + - not_null + - name: EARNINGS + description: "The summarized earnings within this day" + tests: + - not_null + - name: BONDING_EARNINGS + description: "The summarized bonding earnings within this day" + tests: + - not_null + - name: LIQUIDITY_EARNINGS + description: "The summarized liquidity earnings fee within this day" + tests: + - not_null + - name: AVG_NODE_COUNT + description: "The summarized average node operators number within this day" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_BLOCK_REWARDS_ID + + diff --git a/models/gold/defi/defi__fact_bond_actions.sql b/models/gold/defi/defi__fact_bond_actions.sql new file mode 100644 index 0000000..779921b --- /dev/null +++ b/models/gold/defi/defi__fact_bond_actions.sql @@ -0,0 +1,88 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = "fact_bond_actions_id", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH block_prices AS ( + + SELECT + AVG(cacao_usd) AS cacao_usd, + block_id + FROM + {{ ref('silver__prices') }} + GROUP BY + block_id +), +bond_events AS ( + SELECT + block_timestamp, + tx_id, + from_address, + to_address, + asset, + blockchain, + bond_type, + asset_e8, + e8, + memo, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__bond_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['be.event_id'] + ) }} AS fact_bond_actions_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + from_address, + to_address, + asset, + blockchain, + bond_type, + COALESCE(e8 / pow(10, 8), 0) AS asset_amount, + COALESCE( + cacao_usd * asset_e8, + 0 + ) AS asset_usd, + memo, + be._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + bond_events be + JOIN {{ ref('core__dim_block') }} + b + ON be.block_timestamp = b.timestamp + LEFT JOIN block_prices p + ON b.block_id = p.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_bond_actions.yml b/models/gold/defi/defi__fact_bond_actions.yml new file mode 100644 index 0000000..b06cbc6 --- /dev/null +++ b/models/gold/defi/defi__fact_bond_actions.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: defi__fact_bond_actions + description: "Fact table containing the clean bond action table to record the node operators' behaviors" + columns: + - name: FACT_BOND_ACTIONS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 4 + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: ASSET + description: "Same as pool name, which pool this bond events happens" + - name: bond_type + description: "{{ doc('bond_type') }}" + tests: + - not_null + - name: ASSET_AMOUNT + description: "The asset amount for this bond event" + tests: + - not_null + - name: ASSET_USD + description: "Used the price table to calculate the asset in the usd" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_BOND_ACTIONS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_bond_events.sql b/models/gold/defi/defi__fact_bond_events.sql new file mode 100644 index 0000000..e33da15 --- /dev/null +++ b/models/gold/defi/defi__fact_bond_events.sql @@ -0,0 +1,74 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_bond_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + bond_type, + e8, + block_timestamp, + event_id, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__bond_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id'] + ) }} AS fact_bond_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + bond_type, + e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_bond_events.yml b/models/gold/defi/defi__fact_bond_events.yml new file mode 100644 index 0000000..656279d --- /dev/null +++ b/models/gold/defi/defi__fact_bond_events.yml @@ -0,0 +1,71 @@ +version: 2 +models: + - name: defi__fact_bond_events + description: "Fact table containing all the information about the bond activities. THORChain uses a spinoff of the Proof of Stake consensus mechanism called Proof of Bond." + columns: + - name: FACT_BOND_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 4 + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: ASSET + description: "Same as pool name, which pool this bond events happens" + - name: ASSET_E8 + description: "The asset amount for this bond, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + tests: + - not_null: + where: tx_id <> '0000000000000000000000000000000000000000000000000000000000000000' + - name: bond_type + description: "{{ doc('bond_type') }}" + tests: + - not_null + - name: E8 + description: "The cacao amount for this bond event" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_BOND_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_daily_earnings.sql b/models/gold/defi/defi__fact_daily_earnings.sql new file mode 100644 index 0000000..5852666 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_earnings.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_daily_earnings_id', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge' +) }} + +WITH base AS ( + + SELECT + DAY, + liquidity_fees, + liquidity_fees_usd, + block_rewards, + block_rewards_usd, + total_earnings, + total_earnings_usd, + earnings_to_nodes, + earnings_to_nodes_usd, + earnings_to_pools, + earnings_to_pools_usd, + avg_node_count, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__daily_earnings') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_daily_earnings_id, + DAY, + liquidity_fees, + liquidity_fees_usd, + block_rewards, + block_rewards_usd, + total_earnings, + total_earnings_usd, + earnings_to_nodes, + earnings_to_nodes_usd, + earnings_to_pools, + earnings_to_pools_usd, + avg_node_count, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_daily_earnings.yml b/models/gold/defi/defi__fact_daily_earnings.yml new file mode 100644 index 0000000..0818383 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_earnings.yml @@ -0,0 +1,66 @@ +version: 2 +models: + - name: defi__fact_daily_earnings + description: "The earnings information for the whole THORChain network broken down by daily" + columns: + - name: FACT_DAILY_EARNINGS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The recorded day" + tests: + - not_null + - unique + - name: LIQUIDITY_FEES + description: "The summarized liquidity cost fee within this day" + tests: + - not_null + - name: LIQUIDITY_FEES_USD + description: "The summarized liquidity cost fee within this day in USD" + tests: + - not_null + - name: BLOCK_REWARDS + description: "The summarized total block rewards within this day" + tests: + - not_null + - name: BLOCK_REWARDS_USD + description: "The summarized total block rewards within this day in USD" + tests: + - not_null + - name: TOTAL_EARNINGS + description: "The summarized total earnings within this day" + tests: + - not_null + - name: TOTAL_EARNINGS_USD + description: "The summarized total earnings within this day in USD" + tests: + - not_null + - name: EARNINGS_TO_NODES + description: "The summarized bonding earnings within this day" + tests: + - not_null + - name: EARNINGS_TO_NODES_USD + description: "The summarized bonding earnings within this day in USD" + tests: + - not_null + - name: EARNINGS_TO_POOLS + description: "The summarized liquidity earnings fee within this day" + tests: + - not_null + - name: EARNINGS_TO_POOLS_USD + description: "The summarized liquidity earnings fee within this day in USD" + tests: + - not_null + - name: AVG_NODE_COUNT + description: "The summarized average node operators number within this day" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_DAILY_EARNINGS_ID + + diff --git a/models/gold/defi/defi__fact_daily_pool_stats.sql b/models/gold/defi/defi__fact_daily_pool_stats.sql new file mode 100644 index 0000000..aa43991 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_pool_stats.sql @@ -0,0 +1,115 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_daily_pool_stats_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + pool_name, + system_rewards, + system_rewards_usd, + asset_liquidity, + asset_price, + asset_price_usd, + cacao_liquidity, + cacao_price, + cacao_price_usd, + add_liquidity_count, + add_asset_liquidity, + add_asset_liquidity_usd, + add_cacao_liquidity, + add_cacao_liquidity_usd, + withdraw_count, + withdraw_asset_liquidity, + withdraw_asset_liquidity_usd, + withdraw_cacao_liquidity, + withdraw_cacao_liquidity_usd, + il_protection_paid, + il_protection_paid_usd, + average_slip, + to_asset_average_slip, + to_cacao_average_slip, + swap_count, + to_asset_swap_count, + to_cacao_swap_count, + swap_volume_cacao, + swap_volume_cacao_usd, + to_asset_swap_volume, + to_cacao_swap_volume, + total_swap_fees_cacao, + total_swap_fees_usd, + total_asset_swap_fees, + total_asset_cacao_fees, + unique_member_count, + unique_swapper_count, + liquidity_units, + _UNIQUE_KEY + FROM + {{ ref('silver__daily_pool_stats') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day','a.pool_name'] + ) }} AS fact_daily_pool_stats_id, + DAY, + pool_name, + system_rewards, + system_rewards_usd, + asset_liquidity, + asset_price, + asset_price_usd, + cacao_liquidity, + cacao_price, + cacao_price_usd, + add_liquidity_count, + add_asset_liquidity, + add_asset_liquidity_usd, + add_cacao_liquidity, + add_cacao_liquidity_usd, + withdraw_count, + withdraw_asset_liquidity, + withdraw_asset_liquidity_usd, + withdraw_cacao_liquidity, + withdraw_cacao_liquidity_usd, + il_protection_paid, + il_protection_paid_usd, + average_slip, + to_asset_average_slip, + to_cacao_average_slip, + swap_count, + to_asset_swap_count, + to_cacao_swap_count, + swap_volume_cacao, + swap_volume_cacao_usd, + to_asset_swap_volume, + to_cacao_swap_volume, + total_swap_fees_cacao, + total_swap_fees_usd, + total_asset_swap_fees, + total_asset_cacao_fees, + unique_member_count, + unique_swapper_count, + liquidity_units, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_daily_pool_stats.yml b/models/gold/defi/defi__fact_daily_pool_stats.yml new file mode 100644 index 0000000..31999e7 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_pool_stats.yml @@ -0,0 +1,97 @@ +version: 2 +models: + - name: defi__fact_daily_pool_stats + description: "Fact table that shows the total valued locked and the total value bonded/pooled" + columns: + - name: FACT_DAILY_POOL_STATS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: SYSTEM_REWARDS + description: "The total daily system rewards in cacao to the pool" + - name: SYSTEM_REWARDS_USD + description: "The total daily system rewards in USD to the pool" + - name: ASSET_LIQUIDITY + description: "The total asset liquidity for this pool" + - name: ASSET_PRICE + description: "Current asset price" + - name: ASSET_PRICE_USD + description: "Current asset price in USD" + - name: cacao_LIQUIDITY + description: "The total cacao liquidity for this pool" + - name: cacao_PRICE + description: "Current cacao price" + - name: cacao_PRICE_USD + description: "Current cacao price in USD" + - name: ADD_LIQUIDITY_COUNT + description: "How many times to add liquidity to the pool" + - name: ADD_ASSET_LIQUIDITY + description: "The total amount of asset liquidity added to the pool" + - name: ADD_ASSET_LIQUIDITY_USD + description: "The total amount in usd of asset liquidity added to the pool" + - name: ADD_cacao_LIQUIDITY + description: "The total amount of cacao liquidity added to the pool" + - name: ADD_cacao_LIQUIDITY_USD + description: "The total amount in usd of cacao liquidity added to the pool" + - name: WITHDRAW_COUNT + description: "Time to withdraw from the pool" + - name: WITHDRAW_ASSET_LIQUIDITY + description: "The total amount of asset withdrawn from the pool" + - name: WITHDRAW_ASSET_LIQUIDITY_USD + description: "The total amount in USD of asset withdrawn from the pool" + - name: WITHDRAW_cacao_LIQUIDITY + description: "The total amount of cacao withdrawn from the pool" + - name: WITHDRAW_cacao_LIQUIDITY_USD + description: "The total amount in USD of cacao withdrawn from the pool" + - name: IL_PROTECTION_PAID + description: "Impermanent loss protection cost paid to the network" + - name: IL_PROTECTION_PAID_USD + description: "Impermanent loss protection cost in USD paid to the network" + - name: AVERAGE_SLIP + description: "Average slippage " + - name: TO_ASSET_AVERAGE_SLIP + description: "When swap from cacao to Asset, the average slippage" + - name: TO_cacao_AVERAGE_SLIP + description: "When swap from Asset to cacao, the average slippage" + - name: SWAP_COUNT + description: "Total number of swaps" + - name: TO_ASSET_SWAP_COUNT + description: "Total number of swaps from cacao TO Asset" + - name: TO_cacao_SWAP_COUNT + description: "Total number of swaps from Asset TO cacao" + - name: SWAP_VOLUME_cacao + description: "The swap amount of cacao" + - name: SWAP_VOLUME_cacao_USD + description: "The swap amount of cacao in USD" + - name: TO_ASSET_SWAP_VOLUME + description: "The swap volume from cacao to Asset" + - name: TO_cacao_SWAP_VOLUME + description: "The swap volume from Asset to cacao" + - name: TOTAL_SWAP_FEES_cacao + description: "Total swap fees in cacao" + - name: TOTAL_SWAP_FEES_USD + description: "Total swap fees in USD" + - name: TOTAL_ASSET_SWAP_FEES + description: "Total asset swap fees in asset" + - name: TOTAL_ASSET_cacao_FEES + description: "Total asset swap fees in cacao" + - name: UNIQUE_MEMBER_COUNT + description: "All memberships with a cacao address. Take the member from cacao and asset and then subtract the balance = 0 then get the results" + - name: UNIQUE_SWAPPER_COUNT + description: "The unique swap address for this pool" + - name: LIQUIDITY_UNITS + description: "The amount of units for the liquidity in the pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_DAILY_POOL_STATS_ID diff --git a/models/gold/defi/defi__fact_daily_tvl.sql b/models/gold/defi/defi__fact_daily_tvl.sql new file mode 100644 index 0000000..70b239e --- /dev/null +++ b/models/gold/defi/defi__fact_daily_tvl.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_daily_tvl_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + total_value_pooled, + total_value_pooled_usd, + total_value_bonded, + total_value_bonded_usd, + total_value_locked, + total_value_locked_usd + FROM + {{ ref('silver__daily_tvl') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_daily_tvl_id, + DAY, + total_value_pooled, + total_value_pooled_usd, + total_value_bonded, + total_value_bonded_usd, + total_value_locked, + total_value_locked_usd, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_daily_tvl.yml b/models/gold/defi/defi__fact_daily_tvl.yml new file mode 100644 index 0000000..3baa9c7 --- /dev/null +++ b/models/gold/defi/defi__fact_daily_tvl.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: defi__fact_daily_tvl + description: "Fact table that shows the total valued locked and the total value bonded/pooled" + columns: + - name: FACT_DAILY_TVL_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: TOTAL_VALUE_POOLED + description: "The total amount of cacao provided by the liquidity provides and pooled in the pool" + - name: TOTAL_VALUE_POOLED_USD + description: "The total USD amount of cacao provided by the liquidity provides and pooled in the pool" + - name: TOTAL_VALUE_BONDED + description: "The total amount of cacao provided by the node operators and bonded in the pool" + - name: TOTAL_VALUE_BONDED_USD + description: "The total USD amount of cacao provided by the node operators and bonded in the pool" + - name: TOTAL_VALUE_LOCKED + description: "The total cacao value locked in the pool" + - name: TOTAL_VALUE_LOCKED_USD + description: "The total USD value locked in the pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_DAILY_TVL_ID diff --git a/models/gold/defi/defi__fact_failed_deposit_messages.sql b/models/gold/defi/defi__fact_failed_deposit_messages.sql new file mode 100644 index 0000000..2415962 --- /dev/null +++ b/models/gold/defi/defi__fact_failed_deposit_messages.sql @@ -0,0 +1,71 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_failed_deposit_messages_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + amount_e8, + asset, + from_address, + memo, + code, + reason, + tx_id, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__failed_deposit_messages') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp'] + ) }} AS fact_failed_deposit_messages_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + amount_e8, + asset, + from_address, + memo, + code, + reason, + tx_id, + event_id, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR event_id IN ( + SELECT + event_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_failed_deposit_messages.yml b/models/gold/defi/defi__fact_failed_deposit_messages.yml new file mode 100644 index 0000000..157f2e3 --- /dev/null +++ b/models/gold/defi/defi__fact_failed_deposit_messages.yml @@ -0,0 +1,51 @@ +version: 2 +models: + - name: defi__fact_failed_deposit_messages + description: "Fact table containing failed deposit messages" + columns: + - name: FACT_FAILED_DEPOSIT_MESSAGES_ID + description: "{{ doc('sk') }}" + tests: + - not_null + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "{{ doc('sk') }}" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: AMOUNT_E8 + description: "The amount of the failed deposit in E8 notation" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: MEMO + description: "{{ doc('memo') }}" + - name: CODE + description: "Error code associated with the failed deposit" + - name: REASON + description: "Detailed reason for the failed deposit" + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: EVENT_ID + description: "Unique identifier for the failed deposit event" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_FAILED_DEPOSIT_MESSAGES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_fee_events.sql b/models/gold/defi/defi__fact_fee_events.sql new file mode 100644 index 0000000..2953f92 --- /dev/null +++ b/models/gold/defi/defi__fact_fee_events.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_fee_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + asset, + pool_deduct, + asset_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__fee_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.asset_e8','a.pool_deduct','a.block_timestamp','a.tx_id '] + ) }} AS fact_fee_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + asset, + pool_deduct, + asset_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR tx_id IN ( + SELECT + tx_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_fee_events.yml b/models/gold/defi/defi__fact_fee_events.yml new file mode 100644 index 0000000..493fefd --- /dev/null +++ b/models/gold/defi/defi__fact_fee_events.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_fee_events + description: "Fact table containing all related fees that happen in the Thorchain network" + columns: + - name: FACT_FEE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: POOL_DEDUCT + description: "The amount deducted from the pool related to the fee" + tests: + - not_null + - name: ASSET_E8 + description: "The asset amount for this fee, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_FEE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_gas_events.sql b/models/gold/defi/defi__fact_gas_events.sql new file mode 100644 index 0000000..900a5f9 --- /dev/null +++ b/models/gold/defi/defi__fact_gas_events.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_gas_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + asset, + asset_e8, + cacao_e8, + tx_count, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__gas_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.block_timestamp'] + ) }} AS fact_gas_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + asset, + asset_e8, + cacao_e8, + tx_count, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR asset IN ( + SELECT + asset + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_gas_events.yml b/models/gold/defi/defi__fact_gas_events.yml new file mode 100644 index 0000000..2cbdd4a --- /dev/null +++ b/models/gold/defi/defi__fact_gas_events.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_gas_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_GAS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ASSET + description: "{{ doc('asset') }}" + tests: + - not_null + - name: ASSET_E8 + description: "The asset amount for this fee, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: cacao_E8 + description: "The total gas amount in cacao" + tests: + - not_null + - name: TX_COUNT + description: "The total count of transactions within this block id" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_GAS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_inactive_vault_events.sql b/models/gold/defi/defi__fact_inactive_vault_events.sql new file mode 100644 index 0000000..5129fb7 --- /dev/null +++ b/models/gold/defi/defi__fact_inactive_vault_events.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_inactive_vault_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + add_asgard_address, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__inactive_vault_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.add_asgard_address','a.block_timestamp'] + ) }} AS fact_inactive_vault_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + add_asgard_address, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_inactive_vault_events.yml b/models/gold/defi/defi__fact_inactive_vault_events.yml new file mode 100644 index 0000000..47763bd --- /dev/null +++ b/models/gold/defi/defi__fact_inactive_vault_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: defi__fact_inactive_vault_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_INACTIVE_VAULT_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ADD_ASGARD_ADDRESS + description: "The asgard address in the vault which are inactive" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_INACTIVE_VAULT_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_liquidity_actions.sql b/models/gold/defi/defi__fact_liquidity_actions.sql new file mode 100644 index 0000000..eb8cb98 --- /dev/null +++ b/models/gold/defi/defi__fact_liquidity_actions.sql @@ -0,0 +1,82 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_liquidity_actions_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + tx_id, + lp_action, + pool_name, + from_address, + to_address, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + il_protection, + il_protection_usd, + unstake_asymmetry, + unstake_basis_points, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__liquidity_actions') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_liquidity_actions_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + lp_action, + pool_name, + from_address, + to_address, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + il_protection, + il_protection_usd, + unstake_asymmetry, + unstake_basis_points, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_liquidity_actions.yml b/models/gold/defi/defi__fact_liquidity_actions.yml new file mode 100644 index 0000000..669ba88 --- /dev/null +++ b/models/gold/defi/defi__fact_liquidity_actions.yml @@ -0,0 +1,79 @@ +version: 2 +models: + - name: defi__fact_liquidity_actions + description: "Fact table containing the actions the liquidity providers do in the THORChain, with the amount in cacao/Asset" + columns: + - name: FACT_LIQUIDITY_ACTIONS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: LP_ACTION + description: "The direction of the liquidity providers, add or remove the liquidity" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('address') }}" + - name: TO_ADDRESS + description: "{{ doc('address') }}" + + - name: cacao_AMOUNT + description: "How many cacao liquidity added/removed" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "How many cacao liquidity added/removed in USD" + tests: + - not_null + - name: ASSET_AMOUNT + description: "How many Asset liquidity added/removed" + tests: + - not_null + - name: ASSET_AMOUNT_USD + description: "How many Asset liquidity added/removed in USD" + - name: STAKE_UNITS + description: "Stake units are a way of representing how much liquidity an address has in the pool. THORChain converts the raw amounts you are depositing / withdrawing into stake_units to represent what % of the pool you own a claim to." + tests: + - not_null + - name: asset_tx_id + description: "The transaction id for adding/removing the asset" + - name: asset_address + description: "The asset address of the liquidity provider" + - name: asset_blockchain + description: "{{ doc('blockchain') }}" + - name: il_protection + description: "The total impermanent loss protection paid for this pool on this day" + - name: il_protection_usd + description: "The total impermanent loss protection paid in USD for this pool on this day" + - name: unstake_asymmetry + description: "Only exists in unstake, or removing the liquidity" + - name: unstake_basis_points + description: "The basis points for unstaking, or removing the liquidity" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_LIQUIDITY_ACTIONS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_outbound_events.sql b/models/gold/defi/defi__fact_outbound_events.sql new file mode 100644 index 0000000..776b818 --- /dev/null +++ b/models/gold/defi/defi__fact_outbound_events.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_outbound_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + in_tx, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__outbound_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id ','a.blockchain','a.from_address','a.to_address','a.asset','a.memo ','a.in_tx','a.block_timestamp'] + ) }} AS fact_outbound_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + in_tx, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_outbound_events.yml b/models/gold/defi/defi__fact_outbound_events.yml new file mode 100644 index 0000000..d58d781 --- /dev/null +++ b/models/gold/defi/defi__fact_outbound_events.yml @@ -0,0 +1,47 @@ +version: 2 +models: + - name: defi__fact_outbound_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_OUTBOUND_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + - name: ASSET + description: "Same as pool name, which pool this outbound event happens" + - name: ASSET_E8 + description: "The asset amount for this outbound event, using the price table we can calculate the cacao amount by asset amount" + - name: MEMO + description: "{{ doc('memo') }}" + - name: IN_TX + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_OUTBOUND_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_pending_liquidity_events.sql b/models/gold/defi/defi__fact_pending_liquidity_events.sql new file mode 100644 index 0000000..8833da7 --- /dev/null +++ b/models/gold/defi/defi__fact_pending_liquidity_events.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pending_liquidity_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + cacao_tx_id, + cacao_address, + cacao_e8, + pending_type, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pending_liquidity_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.pool_name ','a.asset_tx_id','a.asset_blockchain','a.asset_address','a.cacao_tx_id','a.cacao_address ','a.pending_type','a.block_timestamp'] + ) }} AS fact_pending_liquidity_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + cacao_tx_id, + cacao_address, + cacao_e8, + pending_type, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pending_liquidity_events.yml b/models/gold/defi/defi__fact_pending_liquidity_events.yml new file mode 100644 index 0000000..a00d8cb --- /dev/null +++ b/models/gold/defi/defi__fact_pending_liquidity_events.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: defi__fact_pending_liquidity_events + description: "Fact table that provides the summary of the gas events for each block" + columns: + - name: FACT_PENDING_LIQUIDITY_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: ASSET_TX_ID + description: "The unique transaction id for the cacao event, the liquidity type can be add/remove the cacao/asset, if the event related to cacao" + - name: ASSET_BLOCKCHAIN + description: "The blockchain of the asset" + - name: ASSET_ADDRESS + description: "The address of asset" + - name: ASSET_E8 + description: "The amount of asset for the liquidity events" + - name: cacao_TX_ID + description: "The unique transaction id for the cacao event, the liquidity type can be add/remove the cacao/asset, if the event related to cacao" + - name: cacao_ADDRESS + description: "The address of cacao" + - name: cacao_E8 + description: "The amount of cacao for the liquidity events" + - name: PENDING_TYPE + description: "The type of liquidity, can be 'add' or 'withdraw'" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_PENDING_LIQUIDITY_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_pool_balance_change_events.sql b/models/gold/defi/defi__fact_pool_balance_change_events.sql new file mode 100644 index 0000000..5b774be --- /dev/null +++ b/models/gold/defi/defi__fact_pool_balance_change_events.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_balance_change_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + asset, + cacao_amount, + cacao_add, + asset_amount, + asset_add, + reason, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_balance_change_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.block_timestamp'] + ) }} AS fact_pool_balance_change_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + asset, + cacao_amount, + cacao_add, + asset_amount, + asset_add, + reason, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR asset IN ( + SELECT + asset + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pool_balance_change_events.yml b/models/gold/defi/defi__fact_pool_balance_change_events.yml new file mode 100644 index 0000000..f1cc4bd --- /dev/null +++ b/models/gold/defi/defi__fact_pool_balance_change_events.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: defi__fact_pool_balance_change_events + description: "Fact table that shows the change of the pool balance" + columns: + - name: FACT_POOL_BALANCE_CHANGE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ASSET + description: "Same as pool name, which pool this bond events happens" + - name: cacao_AMOUNT + description: "The amount of cacao for the pool balance change" + - name: cacao_ADD + description: "False or True, if True, then the event is to add cacao not asset" + - name: ASSET_AMOUNT + description: "The amount of asset for the pool balance change" + - name: ASSET_ADD + description: "False or True, if True, then the event is to add asset not cacao" + - name: REASON + description: "The reason for the pool balance change" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BALANCE_CHANGE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_pool_block_balances.sql b/models/gold/defi/defi__fact_pool_block_balances.sql new file mode 100644 index 0000000..850ea2a --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_balances.sql @@ -0,0 +1,62 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_block_balances_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + pool_name, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + synth_amount, + synth_amount_usd, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_block_balances') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_pool_block_balances_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + cacao_amount, + cacao_amount_usd, + asset_amount, + asset_amount_usd, + synth_amount, + synth_amount_usd, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pool_block_balances.yml b/models/gold/defi/defi__fact_pool_block_balances.yml new file mode 100644 index 0000000..e507b56 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_balances.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: defi__fact_pool_block_balances + description: "Fact table containing the actions the liquidity providers do in the THORChain, with the amount in cacao/Asset" + columns: + - name: FACT_POOL_BLOCK_BALANCES_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: cacao_AMOUNT + description: "The cacao amount balance for this pool name" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "The cacao amount balance in USD for this pool name" + tests: + - not_null + - name: ASSET_AMOUNT + description: "The asset amount balance for this pool name" + tests: + - not_null + - name: ASSET_AMOUNT_USD + description: "The asset amount balance in USD for this pool name" + - name: SYNTH_AMOUNT + description: "The synth amount balance for this pool name" + tests: + - not_null + - name: SYNTH_AMOUNT_USD + description: "The synth amount balance in USD for this pool name" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BLOCK_BALANCES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_pool_block_fees.sql b/models/gold/defi/defi__fact_pool_block_fees.sql new file mode 100644 index 0000000..b4ccf74 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_fees.sql @@ -0,0 +1,53 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_block_fees_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + pool_name, + rewards, + total_liquidity_fees_cacao, + asset_liquidity_fees, + cacao_liquidity_fees, + earnings, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_block_fees') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_pool_block_fees_id, + DAY, + pool_name, + rewards, + total_liquidity_fees_cacao, + asset_liquidity_fees, + cacao_liquidity_fees, + earnings, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_pool_block_fees.yml b/models/gold/defi/defi__fact_pool_block_fees.yml new file mode 100644 index 0000000..a18806f --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_fees.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: defi__fact_pool_block_fees + description: "Fact table that shows the fee paid by each pool at different timestamp" + columns: + - name: FACT_POOL_BLOCK_FEES_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: REWARDS + description: "The total rewards" + - name: TOTAL_LIQUIDITY_FEES_cacao + description: "The total liquidity fees paid in cacao" + - name: ASSET_LIQUIDITY_FEES + description: "The liquidity fees paid in Asset" + - name: cacao_LIQUIDITY_FEES + description: "The liquidity fees paid in cacao" + - name: EARNINGS + description: "The total earnings for this pool at this time" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BLOCK_FEES_ID diff --git a/models/gold/defi/defi__fact_pool_block_statistics.sql b/models/gold/defi/defi__fact_pool_block_statistics.sql new file mode 100644 index 0000000..80f4636 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_statistics.sql @@ -0,0 +1,111 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_block_statistics_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH base AS ( + + SELECT + DAY, + add_asset_liquidity_volume, + add_liquidity_count, + add_liquidity_volume, + add_cacao_liquidity_volume, + asset, + asset_depth, + asset_price, + asset_price_usd, + average_slip, + impermanent_loss_protection_paid, + cacao_depth, + status, + swap_count, + swap_volume, + to_asset_average_slip, + to_asset_count, + to_asset_fees, + to_asset_volume, + to_cacao_average_slip, + to_cacao_count, + to_cacao_fees, + to_cacao_volume, + totalfees, + unique_member_count, + unique_swapper_count, + units, + withdraw_asset_volume, + withdraw_count, + withdraw_cacao_volume, + withdraw_volume, + total_stake, + depth_product, + synth_units, + pool_units, + liquidity_unit_value_index, + prev_liquidity_unit_value_index, + _UNIQUE_KEY + FROM + {{ ref('silver__pool_block_statistics') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_pool_block_statistics_id, + DAY, + add_asset_liquidity_volume, + add_liquidity_count, + add_liquidity_volume, + add_cacao_liquidity_volume, + asset, + asset_depth, + asset_price, + asset_price_usd, + average_slip, + impermanent_loss_protection_paid, + cacao_depth, + status, + swap_count, + swap_volume, + to_asset_average_slip, + to_asset_count, + to_asset_fees, + to_asset_volume, + to_cacao_average_slip, + to_cacao_count, + to_cacao_fees, + to_cacao_volume, + totalfees, + unique_member_count, + unique_swapper_count, + units, + withdraw_asset_volume, + withdraw_count, + withdraw_cacao_volume, + withdraw_volume, + total_stake, + depth_product, + synth_units, + pool_units, + liquidity_unit_value_index, + prev_liquidity_unit_value_index, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_pool_block_statistics.yml b/models/gold/defi/defi__fact_pool_block_statistics.yml new file mode 100644 index 0000000..34aa858 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_block_statistics.yml @@ -0,0 +1,90 @@ +version: 2 +models: + - name: defi__fact_pool_block_statistics + description: "Fact table containing the actions the liquidity providers do in the THORChain, with the amount in cacao/Asset" + columns: + - name: FACT_POOL_BLOCK_STATISTICS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The timestamp in day for the recorded of the block fee" + - name: ADD_ASSET_LIQUIDITY_VOLUME + description: "How much asset liquidity has been added to the pool at this time for this pool" + - name: ADD_LIQUIDITY_COUNT + description: "The count of add liquidity transactions" + - name: ADD_LIQUIDITY_VOLUME + description: "The asset volume of liquidity added to the pool" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_DEPTH + description: "The current pool depth, which is the total cacao pooled in the asset" + - name: ASSET_PRICE + description: "The asset price in cacao for this day" + - name: ASSET_PRICE_USD + description: "The asset price in USD" + - name: AVERAGE_SLIP + description: "The average slip point for this block within the day" + - name: IMPERMANENT_LOSS_PROTECTION_PAID + description: "The total impermanent loss protection paid for this pool on this day" + - name: cacao_DEPTH + description: "The pool depth" + - name: STATUS + description: "The pool status, which is active or not" + - name: SWAP_COUNT + description: "Total swap transactions count" + - name: SWAP_VOLUME + description: "Total swap volume" + - name: TO_ASSET_AVERAGE_SLIP + description: "If the transaction is from cacao to Asset, the average slip point" + - name: TO_ASSET_COUNT + description: "How many swaps happen from cacao to Asset" + - name: TO_ASSET_FEES + description: "The total swap fees paid to transfer from cacao to Asset" + - name: TO_ASSET_VOLUME + description: "The total volume transferred from cacao to Asset" + - name: TO_cacao_AVERAGE_SLIP + description: "If the transaction is from Asset to cacao, the average slip point" + - name: TO_cacao_COUNT + description: "How many swaps happen from Asset to cacao" + - name: TO_cacao_FEES + description: "The total swap fees paid to transfer from Asset to cacao" + - name: TO_cacao_VOLUME + description: "The total volume transferred from Asset to cacao" + - name: TOTALFEES + description: "The total fees paid for the swaps" + - name: UNIQUE_MEMBER_COUNT + description: "All memberships with a cacao address. Take the member from cacao and asset and then subtract the balance = 0 then get the results" + - name: UNIQUE_SWAPPER_COUNT + description: "The unique swap addresses for this pool " + - name: UNITS + description: "The swap units" + - name: WITHDRAW_ASSET_VOLUME + description: "How many assets get withdrawn from the liquidity pools" + - name: WITHDRAW_COUNT + description: "How many times the withdraw events happens" + - name: WITHDRAW_cacao_VOLUME + description: "How many cacao volume get withdrawn from the pool" + - name: WITHDRAW_VOLUME + description: "How many asset volume get withdrawn from the pool" + - name: TOTAL_STAKE + description: "" + - name: DEPTH_PRODUCT + description: "" + - name: SYNTH_UNITS + description: "" + - name: POOL_UNITS + description: "" + - name: LIQUIDITY_UNIT_VALUE_INDEX + description: "" + - name: PREV_LIQUIDITY_UNIT_VALUE_INDEX + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_BLOCK_STATISTICS_ID + \ No newline at end of file diff --git a/models/gold/defi/defi__fact_pool_events.sql b/models/gold/defi/defi__fact_pool_events.sql new file mode 100644 index 0000000..e5f87df --- /dev/null +++ b/models/gold/defi/defi__fact_pool_events.sql @@ -0,0 +1,60 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_pool_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + asset, + status, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__pool_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.asset','a.status','a.block_timestamp'] + ) }} AS fact_pool_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + asset, + status, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR asset IN ( + SELECT + asset + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_pool_events.yml b/models/gold/defi/defi__fact_pool_events.yml new file mode 100644 index 0000000..48b9d44 --- /dev/null +++ b/models/gold/defi/defi__fact_pool_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: defi__fact_pool_events + description: "Fact table that shows that the status of the pool at each block id" + columns: + - name: FACT_POOL_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: ASSET + description: "The asset/pool name" + - name: STATUS + description: "The current status for this pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_POOL_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_refund_events.sql b/models/gold/defi/defi__fact_refund_events.sql new file mode 100644 index 0000000..563ea25 --- /dev/null +++ b/models/gold/defi/defi__fact_refund_events.sql @@ -0,0 +1,70 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_refund_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__refund_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.from_address' ,'a.to_address','a. asset', 'a.asset_2nd', 'a.memo', 'a.code', 'a.reason', 'a.block_timestamp'] + ) }} AS fact_refund_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_refund_events.yml b/models/gold/defi/defi__fact_refund_events.yml new file mode 100644 index 0000000..1abd740 --- /dev/null +++ b/models/gold/defi/defi__fact_refund_events.yml @@ -0,0 +1,53 @@ +version: 2 +models: + - name: defi__fact_refund_events + description: "Fact table that shows that the refund events" + columns: + - name: FACT_REFUND_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: "" + - name: ASSET_2ND + description: "" + - name: ASSET_2ND_E8 + description: "" + - name: MEMO + description: "{{ doc('memo') }}" + - name: CODE + description: "" + - name: REASON + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_REFUND_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/defi/defi__fact_reserve_events.sql b/models/gold/defi/defi__fact_reserve_events.sql new file mode 100644 index 0000000..67a2e10 --- /dev/null +++ b/models/gold/defi/defi__fact_reserve_events.sql @@ -0,0 +1,66 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_reserve_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + address, + e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__reserve_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.from_address','a.to_address','a.asset','a.memo','a.address','a.block_timestamp'] + ) }} AS fact_reserve_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + asset, + asset_e8, + memo, + address, + e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_reserve_events.yml b/models/gold/defi/defi__fact_reserve_events.yml new file mode 100644 index 0000000..fef354a --- /dev/null +++ b/models/gold/defi/defi__fact_reserve_events.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: defi__fact_reserve_events + description: "Fact table that shows the amount of cacao reserved into the network" + columns: + - name: FACT_RESERVE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: "The asset amount" + - name: MEMO + description: "{{ doc('memo') }}" + - name: ADDRESS + description: "The address reserve the amount to the pool" + - name: E8 + description: "The cacao amount" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_RESERVE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_rewards_event_entries.sql b/models/gold/defi/defi__fact_rewards_event_entries.sql new file mode 100644 index 0000000..af143d7 --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_event_entries.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_rewards_event_entries_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + cacao_e8, + saver_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__rewards_event_entries') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.pool_name','a.block_timestamp'] + ) }} AS fact_rewards_event_entries_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + cacao_e8, + saver_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_rewards_event_entries.yml b/models/gold/defi/defi__fact_rewards_event_entries.yml new file mode 100644 index 0000000..1d3a9bf --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_event_entries.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: defi__fact_rewards_event_entries + description: "Fact table that hows the entries for the rewards" + columns: + - name: FACT_REWARDS_EVENT_ENTRIES_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: cacao_E8 + description: "The cacao amount of the rewards for this pool at this block" + - name: SAVER_E8 + description: "The savers amount for this pool at this block" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_REWARDS_EVENT_ENTRIES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_rewards_events.sql b/models/gold/defi/defi__fact_rewards_events.sql new file mode 100644 index 0000000..a64bd09 --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_events.sql @@ -0,0 +1,50 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_rewards_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + bond_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__rewards_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp'] + ) }} AS fact_rewards_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + bond_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_rewards_events.yml b/models/gold/defi/defi__fact_rewards_events.yml new file mode 100644 index 0000000..f020a02 --- /dev/null +++ b/models/gold/defi/defi__fact_rewards_events.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: defi__fact_rewards_events + description: "Fact table that shows the entries for the rewards" + columns: + - name: FACT_REWARDS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: BOND_E8 + description: "The cacao amount of the bond for this pool at this block" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_REWARDS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_send_messages.sql b/models/gold/defi/defi__fact_send_messages.sql new file mode 100644 index 0000000..821ac7c --- /dev/null +++ b/models/gold/defi/defi__fact_send_messages.sql @@ -0,0 +1,69 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'event_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + amount_e8, + asset, + from_address, + to_address, + memo, + tx_id, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__send_messages') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.amount_e8','a.asset','a.from_address','a.to_address','a.memo','a.tx_id','a.event_id','a.block_timestamp'] + ) }} AS fact_send_messages_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + amount_e8, + asset, + from_address, + to_address, + memo, + tx_id, + event_id, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR event_id IN ( + SELECT + event_id + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_send_messages.yml b/models/gold/defi/defi__fact_send_messages.yml new file mode 100644 index 0000000..36d44ad --- /dev/null +++ b/models/gold/defi/defi__fact_send_messages.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: defi__fact_send_messages + description: "Fact table that shows send messages" + columns: + - name: FACT_SEND_MESSAGES_ID + description: "{{ doc('sk') }}" + tests: + - not_null + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: AMOUNT_E8 + - name: ASSET + - name: FROM_ADDRESS + - name: TO_ADDRESS + - name: MEMO + - name: TX_ID + - name: EVENT_ID + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SEND_MESSAGES_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_slash_liquidity_events.sql b/models/gold/defi/defi__fact_slash_liquidity_events.sql new file mode 100644 index 0000000..7c21eca --- /dev/null +++ b/models/gold/defi/defi__fact_slash_liquidity_events.sql @@ -0,0 +1,61 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_slash_liquidity_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__slash_liquidity_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.bond_address', 'a.lp_address', 'a.asset'] + ) }} AS fact_slash_liquidity_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_slash_liquidity_events.yml b/models/gold/defi/defi__fact_slash_liquidity_events.yml new file mode 100644 index 0000000..7ea81c6 --- /dev/null +++ b/models/gold/defi/defi__fact_slash_liquidity_events.yml @@ -0,0 +1,46 @@ +version: 2 +models: + - name: defi__fact_slash_liquidity_events + description: "Fact table that shows the slash liquidity event entries" + columns: + - name: FACT_SLASH_LIQUIDITY_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: bond_address + description: "The address of the bond" + - name: lp_address + description: "The address of the liquidity pool" + - name: asset + description: "The asset that was slashed" + - name: lp_units + description: "The number of liquidity pool units that were slashed" + - name: asset_e8_loss + description: "The amount of asset lost in E8" + - name: cacao_e10_loss + description: "The amount of cacao lost in E10" + + + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SLASH_LIQUIDITY_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_stake_events.sql b/models/gold/defi/defi__fact_stake_events.sql new file mode 100644 index 0000000..cb26324 --- /dev/null +++ b/models/gold/defi/defi__fact_stake_events.sql @@ -0,0 +1,68 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, STAKING' }} }, + unique_key = 'fact_stake_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + stake_units, + cacao_tx_id, + cacao_address, + cacao_e8, + _ASSET_IN_cacao_E8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__stake_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.cacao_address', 'a.asset_address'] + ) }} AS fact_stake_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + asset_tx_id, + asset_blockchain, + asset_address, + asset_e8, + stake_units, + cacao_tx_id, + cacao_address, + cacao_e8, + _ASSET_IN_cacao_E8, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_stake_events.yml b/models/gold/defi/defi__fact_stake_events.yml new file mode 100644 index 0000000..1770945 --- /dev/null +++ b/models/gold/defi/defi__fact_stake_events.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: defi__fact_stake_events + description: "Fact table containing stake events" + columns: + - name: FACT_STAKE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: ASSET_TX_ID + description: "{{ doc('tx_id') }}" + - name: ASSET_BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: ASSET_ADDRESS + description: "{{ doc('address') }}" + - name: ASSET_E8 + description: "The asset amount for this event" + tests: + - not_null + - name: STAKE_UNITS + description: "" + tests: + - not_null + - name: cacao_TX_ID + description: "{{ doc('tx_id') }}" + - name: cacao_ADDRESS + description: "{{ doc('address') }}" + - name: cacao_E8 + description: "The asset amount for this event" + tests: + - not_null + - name: _ASSET_IN_cacao_E8 + description: "" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_STAKE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_streamling_swap_details_events.sql b/models/gold/defi/defi__fact_streamling_swap_details_events.sql new file mode 100644 index 0000000..0b55b42 --- /dev/null +++ b/models/gold/defi/defi__fact_streamling_swap_details_events.sql @@ -0,0 +1,77 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_streamling_swap_details_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swap_reasons AS failed_swaps_reasons, + event_id, + block_timestamp, + failed_swap_reasons, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__streamling_swap_details_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id'] + ) }} AS fact_streamling_swap_details_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swaps_reasons, + event_id, + failed_swap_reasons, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_streamling_swap_details_events.yml b/models/gold/defi/defi__fact_streamling_swap_details_events.yml new file mode 100644 index 0000000..5c5ded1 --- /dev/null +++ b/models/gold/defi/defi__fact_streamling_swap_details_events.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: defi__fact_streamling_swap_details_events + description: "Fact table that shows the streaming swap event transactions" + columns: + - name: FACT_STREAMLING_SWAP_DETAILS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: INTERVAL + description: "the time interval of the swap" + - name: quantity + description: "the quantity of the swap" + - name: COUNT + description: "the count of the swap" + - name: last_height + description: "the last height of the swap" + - name: deposit_asset + description: "the deposit asset of the swap" + - name: deposit_e8 + description: "the deposit amount of the swap" + - name: in_asset + description: "the in asset of the swap" + - name: in_e8 + description: "the in amount of the swap" + - name: out_asset + description: "the out asset of the swap" + - name: out_e8 + description: "the out amount of the swap" + - name: failed_swaps + description: "the count of failed swaps" + - name: failed_swaps_reasons + description: "DEPRECATING SOON! the reason of failed swaps" + - name: event_id + description: "" + - name: failed_swap_reasons + description: "the reason of failed swaps" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_STREAMLING_SWAP_DETAILS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_swaps.sql b/models/gold/defi/defi__fact_swaps.sql new file mode 100644 index 0000000..8ea6275 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps.sql @@ -0,0 +1,103 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_swaps_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + block_id, + tx_id, + blockchain, + pool_name, + from_address, + native_to_address, + to_pool_address, + affiliate_address, + affiliate_fee_basis_points, + affiliate_addresses_array, + affiliate_fee_basis_points_array, + from_asset, + to_asset, + from_amount, + to_amount, + min_to_amount, + from_amount_usd, + to_amount_usd, + cacao_usd, + asset_usd, + to_amount_min_usd, + swap_slip_bp, + liq_fee_cacao, + liq_fee_cacao_usd, + liq_fee_asset, + liq_fee_asset_usd, + streaming_count, + streaming_quantity, + _unique_key, + _inserted_timestamp + FROM + {{ ref('silver__swaps') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_swaps_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + pool_name, + from_address, + native_to_address, + to_pool_address, + affiliate_address, + affiliate_fee_basis_points, + affiliate_addresses_array, + affiliate_fee_basis_points_array, + from_asset, + to_asset, + from_amount, + to_amount, + min_to_amount, + from_amount_usd, + to_amount_usd, + cacao_usd, + asset_usd, + to_amount_min_usd, + swap_slip_bp, + liq_fee_cacao, + liq_fee_cacao_usd, + liq_fee_asset, + liq_fee_asset_usd, + streaming_count, + streaming_quantity, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_swaps.yml b/models/gold/defi/defi__fact_swaps.yml new file mode 100644 index 0000000..1de9854 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps.yml @@ -0,0 +1,86 @@ +version: 2 +models: + - name: defi__fact_swaps + description: "Fact table that shows the swap activity in Thorchain" + columns: + - name: FACT_SWAPS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: FROM_ADDRESS + description: "The user address that initiates the swap" + - name: NATIVE_TO_ADDRESS + description: "The user address that is receiving the result of the swap" + - name: TO_POOL_ADDRESS + description: "The pool address that processes the swap" + - name: AFFILIATE_ADDRESS + description: "The affiliate address that is receiving the affiliate fee (redundant with array)" + - name: AFFILIATE_FEE_BASIS_POINTS + description: "The affiliate fee basis points that is received by the affiliate address (redundant with array)" + - name: AFFILIATE_ADDRESSES_ARRAY + description: "The affiliate addresses that are receiving the affiliate fee" + - name: AFFILIATE_FEE_BASIS_POINTS_ARRAY + description: "The affiliate fee basis points that are received by the affiliate addresses" + - name: FROM_ASSET + description: "Initial asset to swap" + - name: TO_ASSET + description: "The asset swap to" + - name: FROM_AMOUNT + description: "Amount of the asset to swap from" + - name: TO_AMOUNT + description: "Amount of the asset to swap for" + - name: MIN_TO_AMOUNT + description: "Minimal amount to swap for" + - name: FROM_AMOUNT_USD + description: "Amount in USD of the asset to swap from" + - name: TO_AMOUNT_USD + description: "Amount in USD of the asset to swap for" + - name: cacao_USD + description: "Minimal amount in USD to swap for" + - name: ASSET_USD + description: "Asset amount in USD" + - name: TO_AMOUNT_MIN_USD + description: "Min asset amount in USD" + - name: SWAP_SLIP_BP + description: "The slippage during the swap process" + - name: LIQ_FEE_cacao + description: "The amount of cacao liquidity fee paid in cacao for the swaps" + - name: LIQ_FEE_cacao_USD + description: "The amount of cacao in USD liquidity fee paid in cacao for the swaps" + - name: LIQ_FEE_ASSET + description: "The amount of Asset liquidity fee paid in cacao for the swaps" + - name: LIQ_FEE_ASSET_USD + description: "The amount of Asset in USD liquidity fee paid in cacao for the swaps" + - name: STREAMING_COUNT + description: "The count of the streaming events" + - name: STREAMING_QUANTITY + description: "The quantity of the streaming events" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SWAPS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_swaps_events.sql b/models/gold/defi/defi__fact_swaps_events.sql new file mode 100644 index 0000000..36c4a00 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps_events.sql @@ -0,0 +1,81 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM, SWAPS' }} }, + unique_key = 'fact_swap_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + event_id, + block_timestamp, + streaming_count, + streaming_quantity, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__swap_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.tx_id','a.blockchain','a.to_address','a.from_address','a.from_asset','a.from_e8','a.to_asset','a.to_e8','a.memo','a.pool_name','a._direction'] + ) }} AS fact_swap_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + blockchain, + from_address, + to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + event_id, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_swaps_events.yml b/models/gold/defi/defi__fact_swaps_events.yml new file mode 100644 index 0000000..59dcea6 --- /dev/null +++ b/models/gold/defi/defi__fact_swaps_events.yml @@ -0,0 +1,96 @@ +version: 2 +models: + - name: defi__fact_swaps_events + description: "Fact table that shows the swap event entries table shows the entries for the swaps" + columns: + - name: FACT_SWAP_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "The user address that initiates the swap" + tests: + - not_null + - name: TO_ADDRESS + description: "The address we are swapping to" + tests: + - not_null + - name: FROM_ASSET + description: "The asset we are swapping from" + tests: + - not_null + - name: FROM_E8 + description: "The amount we are swapping from" + tests: + - not_null + - name: TO_ASSET + description: "The asset we are swapping to" + tests: + - not_null + - name: TO_E8 + description: "The asset amount we are swapping for (divide by 10^8 to get the decimal amount)" + tests: + - not_null + - name: MEMO + description: "The transaction memo" + tests: + - not_null + - name: POOL_NAME + description: "The name of the pool" + tests: + - not_null + - name: TO_E8_MIN + description: "The minimum amount the swapper will receive" + tests: + - not_null + - name: SWAP_SLIP_BP + description: "The slippage" + tests: + - not_null + - name: LIQ_FEE_E8 + description: "The fee (divide by 10^8 to get the decimal amount)" + tests: + - not_null + - name: LIQ_FEE_IN_cacao_E8 + description: "The fee in cacao (divide by 10^8 to get the decimal amount)" + tests: + - not_null + - name: _DIRECTION + description: "" + tests: + - not_null + - name: STREAMING_COUNT + description: "The count of the streaming events" + - name: STREAMING_QUANTITY + description: "The quantity of the streaming events" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SWAP_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_total_block_rewards.sql b/models/gold/defi/defi__fact_total_block_rewards.sql new file mode 100644 index 0000000..abc2246 --- /dev/null +++ b/models/gold/defi/defi__fact_total_block_rewards.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_total_block_rewards_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_id, + reward_entity, + cacao_amount, + cacao_amount_usd, + _unique_key, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__total_block_rewards') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a._unique_key'] + ) }} AS fact_total_block_rewards_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + reward_entity, + cacao_amount, + cacao_amount_usd, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_id = b.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_total_block_rewards.yml b/models/gold/defi/defi__fact_total_block_rewards.yml new file mode 100644 index 0000000..a90be7b --- /dev/null +++ b/models/gold/defi/defi__fact_total_block_rewards.yml @@ -0,0 +1,42 @@ +version: 2 +models: + - name: defi__fact_total_block_rewards + description: "Fact table containing stake events" + columns: + - name: FACT_TOTAL_BLOCK_REWARDS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: REWARD_ENTITY + description: "The asset or named as pool name" + tests: + - not_null + - name: cacao_AMOUNT + description: "The rewards measured in cacao amount" + tests: + - not_null + - name: cacao_AMOUNT_USD + description: "The rewards measured in cacao amount in the USD" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TOTAL_BLOCK_REWARDS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID + diff --git a/models/gold/defi/defi__fact_total_value_locked.sql b/models/gold/defi/defi__fact_total_value_locked.sql new file mode 100644 index 0000000..ed4058a --- /dev/null +++ b/models/gold/defi/defi__fact_total_value_locked.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_total_value_locked_id', + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge' +) }} + +WITH base AS ( + + SELECT + DAY, + total_value_pooled, + total_value_bonded, + total_value_locked, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__total_value_locked') }} + +{% if is_incremental() %} +WHERE + DAY >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.day'] + ) }} AS fact_total_value_locked_id, + DAY, + total_value_pooled, + total_value_bonded, + total_value_locked, + _INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A diff --git a/models/gold/defi/defi__fact_total_value_locked.yml b/models/gold/defi/defi__fact_total_value_locked.yml new file mode 100644 index 0000000..f798bd8 --- /dev/null +++ b/models/gold/defi/defi__fact_total_value_locked.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: defi__fact_total_value_locked + description: "Fact table that shows set mimir events" + columns: + - name: FACT_TOTAL_VALUE_LOCKED_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: DAY + description: "The timestamp in day for the recorded of the block fee" + tests: + - not_null + - unique + - name: TOTAL_VALUE_POOLED + description: "The total amount of cacao provided by the liquidity provides and pooled in the pool" + - name: TOTAL_VALUE_BONDED + description: "The total amount of cacao provided by the node operators and bonded in the pool" + - name: TOTAL_VALUE_LOCKED + description: "The total cacao value locked in the pool" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_TOTAL_VALUE_LOCKED_ID diff --git a/models/gold/defi/defi__fact_update_node_account_status_events.sql b/models/gold/defi/defi__fact_update_node_account_status_events.sql new file mode 100644 index 0000000..667f516 --- /dev/null +++ b/models/gold/defi/defi__fact_update_node_account_status_events.sql @@ -0,0 +1,53 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_update_node_account_status_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + former_status, + current_status, + node_address, + _inserted_timestamp + FROM + {{ ref('silver__update_node_account_status_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.node_address', 'a.block_timestamp', 'a.current_status', 'a.former_status'] + ) }} AS fact_update_node_account_status_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + former_status, + current_status, + node_address, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_update_node_account_status_events.yml b/models/gold/defi/defi__fact_update_node_account_status_events.yml new file mode 100644 index 0000000..89857ec --- /dev/null +++ b/models/gold/defi/defi__fact_update_node_account_status_events.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: defi__fact_update_node_account_status_events + description: "Fact table containing node account status events" + columns: + - name: FACT_UPDATE_NODE_ACCOUNT_STATUS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: FORMER_STATUS + description: "The amount of cacao for this add event" + tests: + - not_null + - name: CURRENT_STATUS + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: NODE_ADDRESS + description: "Address of node operator" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_UPDATE_NODE_ACCOUNT_STATUS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/defi/defi__fact_withdraw_events.sql b/models/gold/defi/defi__fact_withdraw_events.sql new file mode 100644 index 0000000..5f129ca --- /dev/null +++ b/models/gold/defi/defi__fact_withdraw_events.sql @@ -0,0 +1,79 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_withdraw_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + e.tx_id, + e.blockchain, + e.from_address, + e.to_address, + e.asset, + e.asset_e8, + e.emit_asset_e8, + e.emit_cacao_e8, + e.memo, + e.pool_name, + e.stake_units, + e.basis_points, + e.asymmetry, + e.imp_loss_protection_e8, + e._emit_asset_in_cacao_e8, + e.block_timestamp, + e.event_id, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__withdraw_events') }} + e +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id'] + ) }} AS fact_withdraw_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + A.tx_id, + A.blockchain, + A.from_address, + A.to_address, + A.asset, + A.asset_e8, + A.emit_asset_e8, + A.emit_cacao_e8, + A.memo, + A.pool_name, + A.stake_units, + A.basis_points, + A.asymmetry, + A.imp_loss_protection_e8, + A._emit_asset_in_cacao_e8, + A._inserted_timestamp, + '{{ invocation_id }}' AS _audit_run_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/defi/defi__fact_withdraw_events.yml b/models/gold/defi/defi__fact_withdraw_events.yml new file mode 100644 index 0000000..57001be --- /dev/null +++ b/models/gold/defi/defi__fact_withdraw_events.yml @@ -0,0 +1,89 @@ +version: 2 +models: + - name: defi__fact_withdraw_events + description: "Fact table containing all withdraw events" + columns: + - name: FACT_WITHDRAW_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: BLOCKCHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null + - name: TO_ADDRESS + description: "{{ doc('to_address') }}" + tests: + - not_null + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: "The asset amount for this bond, using the price table we can calculate the cacao amount by asset amount" + tests: + - not_null + - name: EMIT_ASSET_E8 + description: "" + tests: + - not_null + - name: EMIT_cacao_E8 + description: "" + tests: + - not_null + - name: MEMO + description: "{{ doc('memo') }}" + tests: + - not_null + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + tests: + - not_null + - name: STAKE_UNITS + description: "" + tests: + - not_null + - name: BASIS_POINTS + description: "" + tests: + - not_null + - name: ASYMMETRY + description: "" + tests: + - not_null + - name: IMP_LOSS_PROTECTION_E8 + description: "" + tests: + - not_null + - name: _EMIT_ASSET_IN_cacao_E8 + description: "" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_WITHDRAW_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_new_node_events.sql b/models/gold/gov/gov__fact_new_node_events.sql new file mode 100644 index 0000000..a307d1f --- /dev/null +++ b/models/gold/gov/gov__fact_new_node_events.sql @@ -0,0 +1,58 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_new_node_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + block_timestamp, + event_id, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__new_node_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.node_address','a.block_timestamp'] + ) }} AS fact_new_node_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR node_address IN ( + SELECT + node_address + FROM + {{ this }} + WHERE + dim_block_id = '-1' + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_new_node_events.yml b/models/gold/gov/gov__fact_new_node_events.yml new file mode 100644 index 0000000..08d668b --- /dev/null +++ b/models/gold/gov/gov__fact_new_node_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: gov__fact_new_node_events + description: "Fact table that shows new node events" + columns: + - name: FACT_NEW_NODE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "The asgard address in the vault which are inactive" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_NEW_NODE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/gold/gov/gov__fact_set_ip_address_events.sql b/models/gold/gov/gov__fact_set_ip_address_events.sql new file mode 100644 index 0000000..6529a32 --- /dev/null +++ b/models/gold/gov/gov__fact_set_ip_address_events.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_ip_address_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + ip_addr, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_ip_address_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.node_address','a.ip_addr','a.block_timestamp'] + ) }} AS fact_set_ip_address_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + ip_addr, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_set_ip_address_events.yml b/models/gold/gov/gov__fact_set_ip_address_events.yml new file mode 100644 index 0000000..b88a6ba --- /dev/null +++ b/models/gold/gov/gov__fact_set_ip_address_events.yml @@ -0,0 +1,35 @@ +version: 2 +models: + - name: gov__fact_set_ip_address_events + description: "Fact table that shows set ip address events" + columns: + - name: FACT_SET_IP_ADDRESS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "" + - name: IP_ADDR + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_IP_ADDRESS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_set_node_keys_events.sql b/models/gold/gov/gov__fact_set_node_keys_events.sql new file mode 100644 index 0000000..5ee2c2f --- /dev/null +++ b/models/gold/gov/gov__fact_set_node_keys_events.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_node_keys_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + secp256k1, + ed25519, + validator_consensus, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_node_keys_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.node_address', 'a.secp256k1', 'a.ed25519', 'a.block_timestamp', 'a.validator_consensus'] + ) }} AS fact_set_node_keys_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + secp256k1, + ed25519, + validator_consensus, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_set_node_keys_events.yml b/models/gold/gov/gov__fact_set_node_keys_events.yml new file mode 100644 index 0000000..c8c698e --- /dev/null +++ b/models/gold/gov/gov__fact_set_node_keys_events.yml @@ -0,0 +1,39 @@ +version: 2 +models: + - name: gov__fact_set_node_keys_events + description: "Fact table that shows set node key events" + columns: + - name: FACT_SET_NODE_KEYS_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "" + - name: SECP256K1 + description: "" + - name: ED25519 + description: "" + - name: VALIDATOR_CONSENSUS + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_NODE_KEYS_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_set_version_events.sql b/models/gold/gov/gov__fact_set_version_events.sql new file mode 100644 index 0000000..7dbc319 --- /dev/null +++ b/models/gold/gov/gov__fact_set_version_events.sql @@ -0,0 +1,52 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_set_version_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + version, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__set_version_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id', 'a.node_address', 'a.block_timestamp', 'a.version'] + ) }} AS fact_set_version_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + version, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_set_version_events.yml b/models/gold/gov/gov__fact_set_version_events.yml new file mode 100644 index 0000000..b0d92c4 --- /dev/null +++ b/models/gold/gov/gov__fact_set_version_events.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: gov__fact_set_version_events + description: "Fact table that shows set version events" + columns: + - name: FACT_SET_VERSION_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: VERSION + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SET_VERSION_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_slash_amounts.sql b/models/gold/gov/gov__fact_slash_amounts.sql new file mode 100644 index 0000000..a47fd57 --- /dev/null +++ b/models/gold/gov/gov__fact_slash_amounts.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_slash_amounts_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + pool_name, + asset, + asset_e8, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__slash_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.pool_name','a.asset'] + ) }} AS fact_slash_amounts_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + pool_name, + asset, + asset_e8, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_slash_amounts.yml b/models/gold/gov/gov__fact_slash_amounts.yml new file mode 100644 index 0000000..2c87f96 --- /dev/null +++ b/models/gold/gov/gov__fact_slash_amounts.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: gov__fact_slash_amounts + description: "Fact table that shows slash events" + columns: + - name: FACT_SLASH_AMOUNTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: POOL_NAME + description: "{{ doc('pool_name') }}" + - name: ASSET + description: "{{ doc('asset') }}" + - name: ASSET_E8 + description: " The asset amount" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SLASH_AMOUNTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_slash_points.sql b/models/gold/gov/gov__fact_slash_points.sql new file mode 100644 index 0000000..9a540bb --- /dev/null +++ b/models/gold/gov/gov__fact_slash_points.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_slash_points_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + node_address, + slash_points, + reason, + event_id, + block_timestamp, + _INSERTED_TIMESTAMP + FROM + {{ ref('silver__slash_points_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.node_address','a.slash_points'] + ) }} AS fact_slash_points_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + node_address, + slash_points, + reason, + A._INSERTED_TIMESTAMP, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_slash_points.yml b/models/gold/gov/gov__fact_slash_points.yml new file mode 100644 index 0000000..4a53829 --- /dev/null +++ b/models/gold/gov/gov__fact_slash_points.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: gov__fact_slash_points + description: "Fact table that shows set mimir events" + columns: + - name: FACT_SLASH_POINTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: NODE_ADDRESS + description: "" + - name: SLASH_POINTS + description: "" + - name: REASON + description: "" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_SLASH_POINTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/gov/gov__fact_validator_request_leave_events.sql b/models/gold/gov/gov__fact_validator_request_leave_events.sql new file mode 100644 index 0000000..e2be80e --- /dev/null +++ b/models/gold/gov/gov__fact_validator_request_leave_events.sql @@ -0,0 +1,54 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_validator_request_leave_events_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + block_timestamp, + tx_id, + from_address, + node_address, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__validator_request_leave_events') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.event_id','a.block_timestamp','a.tx_id', 'a.from_address', 'a.node_address'] + ) }} AS fact_validator_request_leave_events_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + tx_id, + from_address, + node_address, + A._inserted_timestamp, + '{{ invocation_id }}' AS _invocation_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/gov/gov__fact_validator_request_leave_events.yml b/models/gold/gov/gov__fact_validator_request_leave_events.yml new file mode 100644 index 0000000..fd4aed0 --- /dev/null +++ b/models/gold/gov/gov__fact_validator_request_leave_events.yml @@ -0,0 +1,43 @@ +version: 2 +models: + - name: gov__fact_validator_request_leave_events + description: "Fact table containing validator request leave events" + columns: + - name: FACT_VALIDATOR_REQUEST_LEAVE_EVENTS_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: _inserted_timestamp < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + - name: FROM_ADDRESS + description: "{{ doc('from_address') }}" + tests: + - not_null + - name: NODE_ADDRESS + description: "Address of node operator" + tests: + - not_null + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + tests: + - dbt_constraints.primary_key: + column_name: FACT_VALIDATOR_REQUEST_LEAVE_EVENTS_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID \ No newline at end of file diff --git a/models/gold/price/price__fact_cacao_price.sql b/models/gold/price/price__fact_cacao_price.sql new file mode 100644 index 0000000..955b169 --- /dev/null +++ b/models/gold/price/price__fact_cacao_price.sql @@ -0,0 +1,49 @@ +{{ config( + materialized = 'incremental', + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'DEX, AMM' }} }, + unique_key = 'fact_cacao_price_id', + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH base AS ( + + SELECT + cacao_price_e8, + block_timestamp, + _inserted_timestamp + FROM + {{ ref('silver__cacao_price') }} +) +SELECT + {{ dbt_utils.generate_surrogate_key( + ['a.block_timestamp'] + ) }} AS fact_cacao_price_id, + b.block_timestamp, + COALESCE( + b.dim_block_id, + '-1' + ) AS dim_block_id, + cacao_price_e8 AS cacao_usd, + A._inserted_timestamp, + '{{ invocation_id }}' AS _audit_run_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp +FROM + base A + JOIN {{ ref('core__dim_block') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/gold/price/price__fact_cacao_price.yml b/models/gold/price/price__fact_cacao_price.yml new file mode 100644 index 0000000..fe5033e --- /dev/null +++ b/models/gold/price/price__fact_cacao_price.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: price__fact_cacao_price + description: "Fact table that provides the cacao price at a given block height" + columns: + - name: FACT_cacao_PRICE_ID + description: "{{ doc('sk') }}" + tests: + - dbt_expectations.expect_column_to_exist + - unique + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null: + where: DIM_BLOCK_ID not in ('-1','-2') + - name: DIM_BLOCK_ID + description: "FK to DIM_BLOCK table" + tests: + - negative_one: + where: BLOCK_TIMESTAMP < (CURRENT_TIMESTAMP - INTERVAL '8 HOURS') + - name: cacao_USD + description: "The cacao price in USD" + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + + tests: + - dbt_constraints.primary_key: + column_name: FACT_cacao_PRICE_ID + - dbt_constraints.foreign_key: + fk_column_name: DIM_BLOCK_ID + pk_table_name: ref('core__dim_block') + pk_column_name: DIM_BLOCK_ID diff --git a/models/silver/silver__active_vault_events.sql b/models/silver/silver__active_vault_events.sql new file mode 100644 index 0000000..9f1af26 --- /dev/null +++ b/models/silver/silver__active_vault_events.sql @@ -0,0 +1,22 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr, + event_id, + block_timestamp, + __HEVO__DATABASE_NAME, + __HEVO__SCHEMA_NAME, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__active_vault_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__active_vault_events.yml b/models/silver/silver__active_vault_events.yml new file mode 100644 index 0000000..d9edbdc --- /dev/null +++ b/models/silver/silver__active_vault_events.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: silver__active_vault_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - BLOCK_TIMESTAMP + - ADD_ASGARD_ADDR + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ADD_ASGARD_ADDR + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__add_events.sql b/models/silver/silver__add_events.sql new file mode 100644 index 0000000..21109c1 --- /dev/null +++ b/models/silver/silver__add_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + cacao_e8, + pool AS pool_name, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__add_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__add_events.yml b/models/silver/silver__add_events.yml new file mode 100644 index 0000000..2244613 --- /dev/null +++ b/models/silver/silver__add_events.yml @@ -0,0 +1,38 @@ +version: 2 +models: + - name: silver__add_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - TX_ID + - BLOCKCHAIN + - POOL_NAME + - FROM_ADDRESS + - TO_ADDRESS + - MEMO + columns: + - name: TX_ID + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: cacao_E8 + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__block_log.sql b/models/silver/silver__block_log.sql new file mode 100644 index 0000000..8e4d7e6 --- /dev/null +++ b/models/silver/silver__block_log.sql @@ -0,0 +1,43 @@ +{{ config( + materialized = 'view' +) }} + +WITH base AS ( + + SELECT + height, + TO_TIMESTAMP( + TIMESTAMP / 1000000000 + ) AS block_timestamp, + TIMESTAMP, + HASH, + agg_state, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP + FROM + {{ ref('bronze__block_log') }} + qualify(ROW_NUMBER() over(PARTITION BY height + ORDER BY + __HEVO__LOADED_AT DESC)) = 1 +) +SELECT + height, + block_timestamp, + block_timestamp :: DATE AS block_date, + HOUR(block_timestamp) AS block_hour, + week(block_timestamp) AS block_week, + MONTH(block_timestamp) AS block_month, + quarter(block_timestamp) AS block_quarter, + YEAR(block_timestamp) AS block_year, + dayofmonth(block_timestamp) AS block_DAYOFMONTH, + dayofweek(block_timestamp) AS block_DAYOFWEEK, + dayofyear(block_timestamp) AS block_DAYOFYEAR, + TIMESTAMP, + HASH, + agg_state, + _INSERTED_TIMESTAMP +FROM + base diff --git a/models/silver/silver__block_log.yml b/models/silver/silver__block_log.yml new file mode 100644 index 0000000..18eb923 --- /dev/null +++ b/models/silver/silver__block_log.yml @@ -0,0 +1,20 @@ +version: 2 +models: + - name: silver__block_log + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HEIGHT + - TIMESTAMP + - HASH + - AGG_STATE + columns: + - name: TIMESTAMP + data_tests: + - not_null + - name: HEIGHT + data_tests: + - not_null + - name: HASH + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__block_pool_depths.sql b/models/silver/silver__block_pool_depths.sql new file mode 100644 index 0000000..1bdb920 --- /dev/null +++ b/models/silver/silver__block_pool_depths.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset_e8, + cacao_e8, + synth_e8, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__block_pool_depths') }} + qualify(ROW_NUMBER() over(PARTITION BY pool, block_timestamp +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__block_pool_depths.yml b/models/silver/silver__block_pool_depths.yml new file mode 100644 index 0000000..d02771d --- /dev/null +++ b/models/silver/silver__block_pool_depths.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__block_pool_depths + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + - POOL_NAME + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: SYNTH_E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__block_rewards.sql b/models/silver/silver__block_rewards.sql new file mode 100644 index 0000000..74e59d5 --- /dev/null +++ b/models/silver/silver__block_rewards.sql @@ -0,0 +1,160 @@ +{{ config( + materialized = 'table', + cluster_by = ['day'] +) }} + +WITH all_block_id AS ( + + SELECT + block_timestamp, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} + GROUP BY + block_timestamp +), +avg_nodes_tbl AS ( + SELECT + block_timestamp, + SUM( + CASE + WHEN current_status = 'Active' THEN 1 + WHEN former_status = 'Active' THEN -1 + ELSE 0 + END + ) AS delta + FROM + {{ ref('silver__update_node_account_status_events') }} + GROUP BY + block_timestamp +), +all_block_with_nodes AS ( + SELECT + all_block_id.block_timestamp, + delta, + SUM(delta) over ( + ORDER BY + all_block_id.block_timestamp ASC + ) AS avg_nodes, + _inserted_timestamp AS _inserted_timestamp + FROM + all_block_id + LEFT JOIN avg_nodes_tbl + ON all_block_id.block_timestamp = avg_nodes_tbl.block_timestamp +), +all_block_with_nodes_date AS ( + SELECT + b.block_timestamp :: DATE AS DAY, + AVG(avg_nodes) AS avg_nodes, + MAX( + A._inserted_timestamp + ) AS _inserted_timestamp + FROM + all_block_with_nodes A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY +), +liquidity_fee_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + COALESCE(SUM(liq_fee_in_cacao_e8), 0) AS liquidity_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + 1 +), +bond_earnings_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + SUM(bond_e8) AS bond_earnings + FROM + {{ ref('silver__rewards_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY +), +total_pool_rewards_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + SUM(cacao_e8) AS total_pool_rewards + FROM + {{ ref('silver__rewards_event_entries') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY +) +SELECT + all_block_with_nodes_date.day, + COALESCE((liquidity_fee_tbl.liquidity_fee / power(10, 8)), 0) AS liquidity_fee, + ( + ( + COALESCE( + total_pool_rewards_tbl.total_pool_rewards, + 0 + ) + COALESCE( + bond_earnings_tbl.bond_earnings, + 0 + ) + ) + ) / power( + 10, + 8 + ) AS block_rewards, + ( + ( + COALESCE( + total_pool_rewards_tbl.total_pool_rewards, + 0 + ) + COALESCE( + liquidity_fee_tbl.liquidity_fee, + 0 + ) + COALESCE( + bond_earnings_tbl.bond_earnings, + 0 + ) + ) + ) / power( + 10, + 8 + ) AS earnings, + COALESCE((bond_earnings_tbl.bond_earnings / power(10, 8)), 0) AS bonding_earnings, + ( + ( + COALESCE( + total_pool_rewards_tbl.total_pool_rewards, + 0 + ) + COALESCE( + liquidity_fee_tbl.liquidity_fee, + 0 + ) + ) + ) / power( + 10, + 8 + ) AS liquidity_earnings, + all_block_with_nodes_date.avg_nodes + 2 AS avg_node_count, + all_block_with_nodes_date._inserted_timestamp +FROM + all_block_with_nodes_date + LEFT JOIN liquidity_fee_tbl + ON all_block_with_nodes_date.day = liquidity_fee_tbl.day + LEFT JOIN total_pool_rewards_tbl + ON all_block_with_nodes_date.day = total_pool_rewards_tbl.day + LEFT JOIN bond_earnings_tbl + ON all_block_with_nodes_date.day = bond_earnings_tbl.day diff --git a/models/silver/silver__block_rewards.yml b/models/silver/silver__block_rewards.yml new file mode 100644 index 0000000..b7e8b50 --- /dev/null +++ b/models/silver/silver__block_rewards.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: silver__block_rewards + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: LIQUIDITY_FEE + data_tests: + - not_null + - name: BLOCK_REWARDS + data_tests: + - not_null + - name: EARNINGS + data_tests: + - not_null + - name: BONDING_EARNINGS + data_tests: + - not_null + - name: LIQUIDITY_EARNINGS + data_tests: + - not_null + - name: AVG_NODE_COUNT \ No newline at end of file diff --git a/models/silver/silver__bond_events.sql b/models/silver/silver__bond_events.sql new file mode 100644 index 0000000..40b09a2 --- /dev/null +++ b/models/silver/silver__bond_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + bond_type, + e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__bond_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__bond_events.yml b/models/silver/silver__bond_events.yml new file mode 100644 index 0000000..f086b3d --- /dev/null +++ b/models/silver/silver__bond_events.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__bond_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: TX_ID + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: BOND_TYPE + data_tests: + - not_null + - name: E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__cacao_price.sql b/models/silver/silver__cacao_price.sql new file mode 100644 index 0000000..26f9160 --- /dev/null +++ b/models/silver/silver__cacao_price.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + cacao_price_e8, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref( + 'bronze__cacao_price' + ) }} + qualify(ROW_NUMBER() over (PARTITION BY block_timestamp +ORDER BY + cacao_price_e8 DESC) = 1) diff --git a/models/silver/silver__cacao_price.yml b/models/silver/silver__cacao_price.yml new file mode 100644 index 0000000..c966256 --- /dev/null +++ b/models/silver/silver__cacao_price.yml @@ -0,0 +1,10 @@ +version: 2 +models: + - name: silver__cacao_price + columns: + - name: CACAO_PRICE_E8 + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__constants.sql b/models/silver/silver__constants.sql new file mode 100644 index 0000000..0558482 --- /dev/null +++ b/models/silver/silver__constants.sql @@ -0,0 +1,9 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + C.key, + C.value +FROM + {{ ref('bronze__constants') }} C diff --git a/models/silver/silver__daily_earnings.sql b/models/silver/silver__daily_earnings.sql new file mode 100644 index 0000000..dd4dcdb --- /dev/null +++ b/models/silver/silver__daily_earnings.sql @@ -0,0 +1,109 @@ +{{ config( + materialized = 'incremental', + unique_key = 'day', + incremental_strategy = 'merge', + cluster_by = ['day'] +) }} + +WITH max_daily_block AS ( + + SELECT + MAX( + block_id + ) AS block_id, + DATE_TRUNC( + 'day', + block_timestamp + ) AS DAY + FROM + {{ ref('silver__prices') }} A + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY +), +daily_cacao_price AS ( + SELECT + p.block_id, + DAY, + AVG(cacao_usd) AS cacao_usd + FROM + {{ ref('silver__prices') }} + p + JOIN max_daily_block mdb + ON p.block_id = mdb.block_id + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + p.block_id +) +SELECT + br.day, + COALESCE( + liquidity_fee, + 0 + ) AS liquidity_fees, + COALESCE( + liquidity_fee * cacao_usd, + 0 + ) AS liquidity_fees_usd, + block_rewards AS block_rewards, + block_rewards * cacao_usd AS block_rewards_usd, + COALESCE( + earnings, + 0 + ) AS total_earnings, + COALESCE( + earnings * cacao_usd, + 0 + ) AS total_earnings_usd, + bonding_earnings AS earnings_to_nodes, + bonding_earnings * cacao_usd AS earnings_to_nodes_usd, + COALESCE( + liquidity_earnings, + 0 + ) AS earnings_to_pools, + COALESCE( + liquidity_earnings * cacao_usd, + 0 + ) AS earnings_to_pools_usd, + avg_node_count, + br._inserted_timestamp +FROM + {{ ref('silver__block_rewards') }} + br + JOIN daily_cacao_price drp + ON br.day = drp.day + +{% if is_incremental() %} +WHERE + br.day >= ( + SELECT + MAX( + DAY + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__daily_earnings.yml b/models/silver/silver__daily_earnings.yml new file mode 100644 index 0000000..c5cd598 --- /dev/null +++ b/models/silver/silver__daily_earnings.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: silver__daily_earnings + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: LIQUIDITY_FEES + data_tests: + - not_null + - name: LIQUIDITY_FEES_USD + data_tests: + - not_null + - name: BLOCK_REWARDS + data_tests: + - not_null + - name: BLOCK_REWARDS_USD + data_tests: + - not_null + - name: TOTAL_EARNINGS + data_tests: + - not_null + - name: TOTAL_EARNINGS_USD + data_tests: + - not_null + - name: EARNINGS_TO_NODES + data_tests: + - not_null + - name: EARNINGS_TO_NODES_USD + data_tests: + - not_null + - name: EARNINGS_TO_POOLS + data_tests: + - not_null + - name: EARNINGS_TO_POOLS_USD + data_tests: + - not_null + - name: AVG_NODE_COUNT diff --git a/models/silver/silver__daily_pool_stats.sql b/models/silver/silver__daily_pool_stats.sql new file mode 100644 index 0000000..a00303e --- /dev/null +++ b/models/silver/silver__daily_pool_stats.sql @@ -0,0 +1,177 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['day'], + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'] +) }} + +WITH daily_cacao_price AS ( + + SELECT + pool_name, + block_timestamp :: DATE AS DAY, + AVG(cacao_usd) AS cacao_usd, + AVG(asset_usd) AS asset_usd + FROM + {{ ref('silver__prices') }} + p + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + pool_name, + DAY +), +pool_fees AS ( + SELECT + pbf.day, + pbf.pool_name, + rewards AS system_rewards, + rewards * cacao_usd AS system_rewards_usd, + asset_liquidity_fees, + asset_liquidity_fees * asset_usd AS asset_liquidity_fees_usd, + cacao_liquidity_fees, + cacao_liquidity_fees * cacao_usd AS cacao_liquidity_fees_usd + FROM + {{ ref('silver__pool_block_fees') }} + pbf + JOIN daily_cacao_price drp + ON pbf.day = drp.day + AND pbf.pool_name = drp.pool_name + +{% if is_incremental() %} +WHERE + pbf.day >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + pbs.day, + pbs.asset AS pool_name, + COALESCE( + system_rewards, + 0 + ) AS system_rewards, + COALESCE( + system_rewards_usd, + 0 + ) AS system_rewards_usd, + COALESCE(asset_depth / pow(10, 8), 0) AS asset_liquidity, + COALESCE( + asset_price, + 0 + ) AS asset_price, + COALESCE( + asset_price_usd, + 0 + ) AS asset_price_usd, + COALESCE(cacao_depth / pow(10, 8), 0) AS cacao_liquidity, + COALESCE(asset_price_usd / NULLIF(cacao_usd, 0), 0) AS cacao_price, + COALESCE( + cacao_usd, + 0 + ) AS cacao_price_usd, + COALESCE( + add_liquidity_count, + 0 + ) AS add_liquidity_count, + COALESCE(add_asset_liquidity_volume / pow(10, 8), 0) AS add_asset_liquidity, + COALESCE(add_asset_liquidity_volume / pow(10, 8) * asset_usd, 0) AS add_asset_liquidity_usd, + COALESCE(add_cacao_liquidity_volume / pow(10, 8), 0) AS add_cacao_liquidity, + COALESCE(add_cacao_liquidity_volume / pow(10, 8) * cacao_usd, 0) AS add_cacao_liquidity_usd, + COALESCE( + withdraw_count, + 0 + ) AS withdraw_count, + COALESCE(withdraw_asset_volume / pow(10, 8), 0) AS withdraw_asset_liquidity, + COALESCE(withdraw_asset_volume / pow(10, 8) * asset_usd, 0) AS withdraw_asset_liquidity_usd, + COALESCE(withdraw_cacao_volume / pow(10, 8), 0) AS withdraw_cacao_liquidity, + COALESCE(withdraw_cacao_volume / pow(10, 8) * cacao_usd, 0) AS withdraw_cacao_liquidity_usd, + COALESCE(impermanent_loss_protection_paid / pow(10, 8), 0) AS il_protection_paid, + COALESCE(impermanent_loss_protection_paid / pow(10, 8) * cacao_usd, 0) AS il_protection_paid_usd, + COALESCE( + average_slip, + 0 + ) AS average_slip, + COALESCE( + to_asset_average_slip, + 0 + ) AS to_asset_average_slip, + COALESCE( + to_cacao_average_slip, + 0 + ) AS to_cacao_average_slip, + COALESCE( + swap_count, + 0 + ) AS swap_count, + COALESCE( + to_asset_count, + 0 + ) AS to_asset_swap_count, + COALESCE( + to_cacao_count, + 0 + ) AS to_cacao_swap_count, + COALESCE(swap_volume / pow(10, 8), 0) AS swap_volume_cacao, + COALESCE(swap_volume / pow(10, 8) * cacao_usd, 0) AS swap_volume_cacao_usd, + COALESCE(to_asset_volume / pow(10, 8), 0) AS to_asset_swap_volume, + COALESCE(to_cacao_volume / pow(10, 8), 0) AS to_cacao_swap_volume, + COALESCE(totalfees / pow(10, 8), 0) AS total_swap_fees_cacao, + COALESCE(totalfees / pow(10, 8) * cacao_usd, 0) AS total_swap_fees_usd, + COALESCE(to_asset_fees / pow(10, 8), 0) AS total_asset_swap_fees, + COALESCE(to_cacao_fees / pow(10, 8), 0) AS total_asset_cacao_fees, + COALESCE( + unique_member_count, + 0 + ) AS unique_member_count, + COALESCE( + unique_swapper_count, + 0 + ) AS unique_swapper_count, + COALESCE( + units, + 0 + ) AS liquidity_units, + concat_ws( + '-', + pbs.day, + pbs.asset + ) AS _unique_key +FROM + {{ ref('silver__pool_block_statistics') }} + pbs + LEFT JOIN daily_cacao_price drp + ON pbs.day = drp.day + AND pbs.asset = drp.pool_name + LEFT JOIN pool_fees pf + ON pbs.day = pf.day + AND pbs.asset = pf.pool_name + +{% if is_incremental() %} +WHERE + pbs.day >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__daily_pool_stats.yml b/models/silver/silver__daily_pool_stats.yml new file mode 100644 index 0000000..e9f8e66 --- /dev/null +++ b/models/silver/silver__daily_pool_stats.yml @@ -0,0 +1,126 @@ +version: 2 +models: + - name: silver__daily_pool_stats + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + - POOL_NAME + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: POOL_NAME + data_tests: + - not_null + - name: SYSTEM_REWARDS + data_tests: + - not_null + - name: SYSTEM_REWARDS_USD + data_tests: + - not_null + - name: ASSET_LIQUIDITY + data_tests: + - not_null + - name: ASSET_PRICE + data_tests: + - not_null + - name: ASSET_PRICE_USD + data_tests: + - not_null + - name: cacao_LIQUIDITY + data_tests: + - not_null + - name: cacao_PRICE + data_tests: + - not_null + - name: cacao_PRICE_USD + data_tests: + - not_null + - name: ADD_LIQUIDITY_COUNT + data_tests: + - not_null + - name: ADD_ASSET_LIQUIDITY + data_tests: + - not_null + - name: ADD_ASSET_LIQUIDITY_USD + data_tests: + - not_null + - name: ADD_cacao_LIQUIDITY_USD + data_tests: + - not_null + - name: WITHDRAW_COUNT + data_tests: + - not_null + - name: WITHDRAW_ASSET_LIQUIDITY + data_tests: + - not_null + - name: WITHDRAW_ASSET_LIQUIDITY_USD + data_tests: + - not_null + - name: WITHDRAW_cacao_LIQUIDITY + data_tests: + - not_null + - name: WITHDRAW_cacao_LIQUIDITY_USD + data_tests: + - not_null + - name: IL_PROTECTION_PAID + data_tests: + - not_null + - name: IL_PROTECTION_PAID_USD + data_tests: + - not_null + - name: AVERAGE_SLIP + data_tests: + - not_null + - name: TO_ASSET_AVERAGE_SLIP + data_tests: + - not_null + - name: TO_cacao_AVERAGE_SLIP + data_tests: + - not_null + - name: SWAP_COUNT + data_tests: + - not_null + - name: TO_ASSET_SWAP_COUNT + data_tests: + - not_null + - name: TO_cacao_SWAP_COUNT + data_tests: + - not_null + - name: SWAP_VOLUME_cacao + data_tests: + - not_null + - name: SWAP_VOLUME_cacao_USD + data_tests: + - not_null + - name: TO_ASSET_SWAP_VOLUME + data_tests: + - not_null + - name: TO_cacao_SWAP_VOLUME + data_tests: + - not_null + - name: TOTAL_SWAP_FEES_cacao + data_tests: + - not_null + - name: TOTAL_SWAP_FEES_USD + data_tests: + - not_null + - name: TOTAL_ASSET_SWAP_FEES + data_tests: + - not_null + - name: TOTAL_ASSET_cacao_FEES + data_tests: + - not_null + - name: UNIQUE_MEMBER_COUNT + data_tests: + - not_null + - name: UNIQUE_SWAPPER_COUNT + data_tests: + - not_null + - name: LIQUIDITY_UNITS + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__daily_tvl.sql b/models/silver/silver__daily_tvl.sql new file mode 100644 index 0000000..d23f724 --- /dev/null +++ b/models/silver/silver__daily_tvl.sql @@ -0,0 +1,72 @@ +{{ config( + materialized = 'incremental', + unique_key = "day", + incremental_strategy = 'merge', + cluster_by = ['day'], + incremental_predicates = ['DBT_INTERNAL_DEST.day >= (select min(day) from ' ~ generate_tmp_view_name(this) ~ ')'] +) }} + +WITH max_daily_block AS ( + + SELECT + MAX(block_id) AS block_id, + DATE_TRUNC( + 'day', + block_timestamp + ) AS DAY + FROM + {{ ref('silver__prices') }} + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY +), +daily_cacao_price AS ( + SELECT + p.block_id, + DAY, + AVG(cacao_usd) AS cacao_usd + FROM + {{ ref('silver__prices') }} + p + JOIN max_daily_block mdb + ON p.block_id = mdb.block_id + +{% if is_incremental() %} +WHERE + block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + p.block_id +) +SELECT + br.day, + total_value_pooled AS total_value_pooled, + total_value_pooled * cacao_usd AS total_value_pooled_usd, + total_value_bonded AS total_value_bonded, + total_value_bonded * cacao_usd AS total_value_bonded_usd, + total_value_locked AS total_value_locked, + total_value_locked * cacao_usd AS total_value_locked_usd +FROM + {{ ref('silver__total_value_locked') }} + br + JOIN daily_cacao_price drp + ON br.day = drp.day diff --git a/models/silver/silver__daily_tvl.yml b/models/silver/silver__daily_tvl.yml new file mode 100644 index 0000000..f8b22fc --- /dev/null +++ b/models/silver/silver__daily_tvl.yml @@ -0,0 +1,32 @@ +version: 2 +models: + - name: silver__daily_tvl + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: TOTAL_VALUE_POOLED + data_tests: + - not_null + - name: TOTAL_VALUE_POOLED_USD + data_tests: + - not_null + - name: TOTAL_VALUE_BONDED + data_tests: + - not_null + - name: TOTAL_VALUE_BONDED_USD + data_tests: + - not_null + - name: TOTAL_VALUE_LOCKED + data_tests: + - not_null + - name: TOTAL_VALUE_LOCKED_USD + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__failed_deposit_messages.sql b/models/silver/silver__failed_deposit_messages.sql new file mode 100644 index 0000000..55caaef --- /dev/null +++ b/models/silver/silver__failed_deposit_messages.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + code, + memo, + asset, + amount_e8, + from_addr AS from_address, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref( + 'bronze__failed_deposit_messages' + ) }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__failed_deposit_messages.yml b/models/silver/silver__failed_deposit_messages.yml new file mode 100644 index 0000000..577d132 --- /dev/null +++ b/models/silver/silver__failed_deposit_messages.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: silver__failed_deposit_messages + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + columns: + - name: AMOUNT_E8 + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: MEMO + - name: CODE + - name: REASON + - name: TX_ID + data_tests: + - not_null + - name: EVENT_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__fee_events.sql b/models/silver/silver__fee_events.sql new file mode 100644 index 0000000..55af07f --- /dev/null +++ b/models/silver/silver__fee_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + asset, + asset_e8, + pool_deduct, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__fee_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__fee_events.yml b/models/silver/silver__fee_events.yml new file mode 100644 index 0000000..fab3b90 --- /dev/null +++ b/models/silver/silver__fee_events.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__fee_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: TX_ID + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: POOL_DEDUCT + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null diff --git a/models/silver/silver__gas_events.sql b/models/silver/silver__gas_events.sql new file mode 100644 index 0000000..3869018 --- /dev/null +++ b/models/silver/silver__gas_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + asset_e8, + cacao_e8, + tx_count, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__gas_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__gas_events.yml b/models/silver/silver__gas_events.yml new file mode 100644 index 0000000..f6900ee --- /dev/null +++ b/models/silver/silver__gas_events.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__gas_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - ASSET + - BLOCK_TIMESTAMP + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: TX_COUNT + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__inactive_vault_events.sql b/models/silver/silver__inactive_vault_events.sql new file mode 100644 index 0000000..803107b --- /dev/null +++ b/models/silver/silver__inactive_vault_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + add_asgard_addr AS add_asgard_address, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__inactive_vault_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__liquidity_actions.sql b/models/silver/silver__liquidity_actions.sql new file mode 100644 index 0000000..b589b3f --- /dev/null +++ b/models/silver/silver__liquidity_actions.sql @@ -0,0 +1,180 @@ +{{ config( + materialized = 'incremental', + unique_key = '_unique_key', + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH stakes AS ( + + SELECT + block_timestamp, + cacao_tx_id, + pool_name, + cacao_address, + cacao_e8, + asset_e8, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__stake_events') }} +), +unstakes AS ( + SELECT + block_timestamp, + tx_id, + pool_name, + from_address, + to_address, + emit_cacao_e8, + emit_asset_e8, + stake_units, + imp_loss_protection_e8, + asymmetry, + basis_points, + event_id, + _inserted_timestamp + FROM + {{ ref('silver__withdraw_events') }} +) +SELECT + b.block_timestamp, + b.height AS block_id, + cacao_tx_id AS tx_id, + 'add_liquidity' AS lp_action, + se.pool_name, + cacao_address AS from_address, + NULL AS to_address, + COALESCE((cacao_e8 / pow(10, 8)), 0) AS cacao_amount, + COALESCE((cacao_e8 / pow(10, 8) * cacao_usd), 0) AS cacao_amount_usd, + COALESCE((asset_e8 / pow(10, 8)), 0) AS asset_amount, + COALESCE((asset_e8 / pow(10, 8) * asset_usd), 0) AS asset_amount_usd, + stake_units, + asset_tx_id, + asset_address, + asset_blockchain, + NULL AS il_protection, + NULL AS il_protection_usd, + NULL AS unstake_asymmetry, + NULL AS unstake_basis_points, + concat_ws( + '-', + event_id, + se.block_timestamp, + COALESCE( + tx_id, + '' + ), + lp_action, + se.pool_name, + COALESCE( + from_address, + '' + ), + COALESCE( + to_address, + '' + ), + COALESCE( + asset_tx_id, + '' + ) + ) AS _unique_key, + se._inserted_timestamp +FROM + stakes se + JOIN {{ ref('silver__block_log') }} + b + ON se.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND se.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +UNION +SELECT + b.block_timestamp, + b.height AS block_id, + tx_id, + 'remove_liquidity' AS lp_action, + ue.pool_name, + from_address, + to_address, + COALESCE(emit_cacao_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(emit_cacao_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + COALESCE(emit_asset_e8 / pow(10, 8), 0) AS asset_amount, + COALESCE(emit_asset_e8 / pow(10, 8) * asset_usd, 0) AS asset_amount_usd, + stake_units, + NULL AS asset_tx_id, + NULL AS asset_address, + NULL AS asset_blockchain, + imp_loss_protection_e8 / pow( + 10, + 8 + ) AS il_protection, + imp_loss_protection_e8 / pow( + 10, + 8 + ) * cacao_usd AS il_protection_usd, + asymmetry AS unstake_asymmetry, + basis_points AS unstake_basis_points, + concat_ws( + '-', + event_id, + ue.block_timestamp, + COALESCE( + tx_id, + '' + ), + lp_action, + ue.pool_name, + COALESCE( + from_address, + '' + ), + COALESCE( + to_address, + '' + ), + COALESCE( + asset_tx_id, + '' + ) + ) AS _unique_key, + ue._inserted_timestamp +FROM + unstakes ue + JOIN {{ ref('silver__block_log') }} + b + ON ue.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND ue.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__liquidity_actions.yml b/models/silver/silver__liquidity_actions.yml new file mode 100644 index 0000000..9afcaa8 --- /dev/null +++ b/models/silver/silver__liquidity_actions.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: silver__liquidity_actions + + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + data_tests: + - not_null + - name: LP_ACTION + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= SYSDATE() - interval '2 day' AND BLOCK_TIMESTAMP >= '2021-04-13' + - name: ASSET_AMOUNT + data_tests: + - not_null + - name: STAKE_UNITS + data_tests: + - not_null diff --git a/models/silver/silver__mayaname_change_events.sql b/models/silver/silver__mayaname_change_events.sql new file mode 100644 index 0000000..32f3d41 --- /dev/null +++ b/models/silver/silver__mayaname_change_events.sql @@ -0,0 +1,31 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + owner, + chain, + address, + expire, + NAME, + fund_amount_e8, + registration_fee_e8, + event_id, + height, + tx_id, + memo, + sender, + preferred_asset, + affiliate_bps, + sub_affiliates, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__mayaname_change_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__new_node_events.sql b/models/silver/silver__new_node_events.sql new file mode 100644 index 0000000..0fab353 --- /dev/null +++ b/models/silver/silver__new_node_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__new_node_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__outbound_events.sql b/models/silver/silver__outbound_events.sql new file mode 100644 index 0000000..bcd71d2 --- /dev/null +++ b/models/silver/silver__outbound_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + in_tx, + internal, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__outbound_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, in_tx +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__outbound_events.yml b/models/silver/silver__outbound_events.yml new file mode 100644 index 0000000..b2ed12b --- /dev/null +++ b/models/silver/silver__outbound_events.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: silver__outbound_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - ASSET + - MEMO + - IN_TX + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: IN_TX + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null diff --git a/models/silver/silver__pending_liquidity_events.sql b/models/silver/silver__pending_liquidity_events.sql new file mode 100644 index 0000000..98626d8 --- /dev/null +++ b/models/silver/silver__pending_liquidity_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset_tx AS asset_tx_id, + asset_chain AS asset_blockchain, + asset_addr AS asset_address, + asset_e8, + cacao_tx AS cacao_tx_ID, + cacao_addr AS cacao_address, + cacao_e8, + pending_type, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__pending_liquidity_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, asset_addr, cacao_addr +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__pending_liquidity_events.yml b/models/silver/silver__pending_liquidity_events.yml new file mode 100644 index 0000000..1d31e85 --- /dev/null +++ b/models/silver/silver__pending_liquidity_events.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__pending_liquidity_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - ASSET_ADDRESS + - CACAO_ADDRESS + columns: + - name: POOL_NAME + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: CACAO_ADDRESS + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: PENDING_TYPE + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__pool_balance_change_events.sql b/models/silver/silver__pool_balance_change_events.sql new file mode 100644 index 0000000..e7383f1 --- /dev/null +++ b/models/silver/silver__pool_balance_change_events.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + cacao_amt AS cacao_amount, + cacao_add, + asset_amt AS asset_amount, + asset_add, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__pool_balance_change_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, asset +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__pool_balance_change_events.yml b/models/silver/silver__pool_balance_change_events.yml new file mode 100644 index 0000000..cde412c --- /dev/null +++ b/models/silver/silver__pool_balance_change_events.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__pool_balance_change_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: CACAO_AMOUNT + data_tests: + - not_null + - name: CACAO_ADD + data_tests: + - not_null + - name: ASSET_AMOUNT + data_tests: + - not_null + - name: ASSET_ADD + data_tests: + - not_null + - name: REASON + data_tests: + - not_null + diff --git a/models/silver/silver__pool_block_balances.sql b/models/silver/silver__pool_block_balances.sql new file mode 100644 index 0000000..a65f181 --- /dev/null +++ b/models/silver/silver__pool_block_balances.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + unique_key = '_unique_key', + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +SELECT + b.block_timestamp, + b.height AS block_id, + bpd.pool_name, + COALESCE(cacao_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(cacao_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + COALESCE(asset_e8 / pow(10, 8), 0) AS asset_amount, + COALESCE(asset_e8 / pow(10, 8) * asset_usd, 0) AS asset_amount_usd, + COALESCE(synth_e8 / pow(10, 8), 0) AS synth_amount, + COALESCE(synth_e8 / pow(10, 8) * asset_usd, 0) AS synth_amount_usd, + concat_ws( + '-', + bpd.block_timestamp, + bpd.pool_name + ) AS _unique_key, + bpd._inserted_timestamp +FROM + {{ ref('silver__block_pool_depths') }} + bpd + JOIN {{ ref('silver__block_log') }} + b + ON bpd.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND bpd.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__pool_block_balances.yml b/models/silver/silver__pool_block_balances.yml new file mode 100644 index 0000000..6ed6998 --- /dev/null +++ b/models/silver/silver__pool_block_balances.yml @@ -0,0 +1,40 @@ +version: 2 +models: + - name: silver__pool_block_balances + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - POOL_NAME + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= current_date -2 AND BLOCK_TIMESTAMP >= '2021-04-13' + - name: ASSET_AMOUNT + data_tests: + - not_null + - name: ASSET_AMOUNT_USD + data_tests: + - not_null + - name: SYNTH_AMOUNT + data_tests: + - not_null + - name: SYNTH_AMOUNT_USD + data_tests: + - not_null diff --git a/models/silver/silver__pool_block_fees.sql b/models/silver/silver__pool_block_fees.sql new file mode 100644 index 0000000..1ed1692 --- /dev/null +++ b/models/silver/silver__pool_block_fees.sql @@ -0,0 +1,200 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.DAY >= (select min(DAY) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['day'] +) }} + +WITH all_block_id AS ( + + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + MAX( + A._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + pool_name +), +total_pool_rewards_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + SUM(cacao_e8) AS rewards + FROM + {{ ref('silver__rewards_event_entries') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + pool_name +), +total_liquidity_fees_CACAO_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + SUM(liq_fee_in_CACAO_e8) AS total_liquidity_fees_CACAO + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +GROUP BY + DAY, + pool_name +), +liquidity_fees_asset_tbl AS ( + SELECT + DATE(block_timestamp) AS DAY, + pool_name, + SUM(asset_fee) AS assetLiquidityFees + FROM + ( + SELECT + b.block_timestamp, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +GROUP BY + DAY, + pool_name +), +liquidity_fees_cacao_tbl AS ( + SELECT + DATE(block_timestamp) AS DAY, + pool_name, + SUM(asset_fee) AS cacaoLiquidityFees + FROM + ( + SELECT + b.block_timestamp, + pool_name, + CASE + WHEN to_asset <> 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp :: DATE >= ( + SELECT + MAX( + DAY - INTERVAL '2 DAYS' --counteract clock skew + ) + FROM + {{ this }} + ) +{% endif %} +) +GROUP BY + DAY, + pool_name +) +SELECT + A.day, + A.pool_name, + COALESCE((rewards / power(10, 8)), 0) AS rewards, + COALESCE((total_liquidity_fees_cacao / power(10, 8)), 0) AS total_liquidity_fees_cacao, + COALESCE((assetLiquidityFees / power(10, 8)), 0) AS asset_liquidity_fees, + COALESCE((cacaoLiquidityFees / power(10, 8)), 0) AS cacao_liquidity_fees, + ( + (COALESCE(total_liquidity_fees_cacao, 0) + COALESCE(rewards, 0)) / power( + 10, + 8 + ) + ) AS earnings, + concat_ws( + '-', + A.day, + A.pool_name + ) AS _unique_key, + _inserted_timestamp +FROM + all_block_id A + LEFT JOIN total_pool_rewards_tbl b + ON A.day = b.day + AND A.pool_name = b.pool_name + LEFT JOIN total_liquidity_fees_cacao_tbl C + ON A.day = C.day + AND A.pool_name = C.pool_name + LEFT JOIN liquidity_fees_asset_tbl d + ON A.day = d.day + AND A.pool_name = d.pool_name + LEFT JOIN liquidity_fees_cacao_tbl e + ON A.day = e.day + AND A.pool_name = e.pool_name diff --git a/models/silver/silver__pool_block_fees.yml b/models/silver/silver__pool_block_fees.yml new file mode 100644 index 0000000..be00203 --- /dev/null +++ b/models/silver/silver__pool_block_fees.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: silver__pool_block_fees + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + - POOL_NAME + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: POOL_NAME + data_tests: + - not_null + - name: REWARDS + data_tests: + - not_null + - name: TOTAL_LIQUIDITY_FEES_cacao + data_tests: + - not_null + - name: ASSET_LIQUIDITY_FEES + data_tests: + - not_null + - name: cacao_LIQUIDITY_FEES + data_tests: + - not_null + - name: EARNINGS + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__pool_block_statistics.sql b/models/silver/silver__pool_block_statistics.sql new file mode 100644 index 0000000..9516a30 --- /dev/null +++ b/models/silver/silver__pool_block_statistics.sql @@ -0,0 +1,586 @@ +{{ config( + materialized = 'table', + cluster_by = ['day'] +) }} + +WITH pool_depth AS ( + + SELECT + DAY, + pool_name, + cacao_e8 AS cacao_depth, + asset_e8 AS asset_depth, + synth_e8 AS synth_depth, + cacao_e8 / NULLIF( + asset_e8, + 0 + ) AS asset_price + FROM + ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + b.height AS block_id, + pool_name, + cacao_e8, + synth_e8, + asset_e8, + MAX( + b.height + ) over (PARTITION BY pool_name, DATE(b.block_timestamp)) AS max_block_id + FROM + {{ ref("silver__block_pool_depths") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + asset_e8 > 0 + ) + WHERE + block_id = max_block_id +), +pool_status AS ( + SELECT + DAY, + pool_name, + status + FROM + ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + asset AS pool_name, + status, + ROW_NUMBER() over (PARTITION BY pool_name, DATE(b.block_timestamp) + ORDER BY + b.block_timestamp DESC, status) AS rn + FROM + {{ ref("silver__pool_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + WHERE + rn = 1 +), +add_liquidity_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + COUNT(*) AS add_liquidity_count, + SUM(cacao_e8) AS add_cacao_liquidity_volume, + SUM(asset_e8) AS add_asset_liquidity_volume, + SUM(stake_units) AS added_stake + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY, + pool_name +), +withdraw_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + COUNT(*) AS withdraw_count, + SUM(emit_cacao_e8) AS withdraw_cacao_volume, + SUM(emit_asset_e8) AS withdraw_asset_volume, + SUM(stake_units) AS withdrawn_stake, + SUM(imp_loss_protection_e8) AS impermanent_loss_protection_paid + FROM + {{ ref("silver__withdraw_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY, + pool_name +), +swap_total_tbl AS ( + SELECT + DAY, + pool_name, + SUM(volume) AS swap_volume + FROM + ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN to_e8 + ELSE from_e8 + END AS volume + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + GROUP BY + DAY, + pool_name +), +swap_to_asset_tbl AS ( + SELECT + DAY, + pool_name, + SUM(liq_fee_in_cacao_e8) AS to_asset_fees, + SUM(from_e8) AS to_asset_volume, + COUNT(*) AS to_asset_count, + AVG(swap_slip_bp) AS to_asset_average_slip + FROM( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 'to_cacao' + ELSE 'to_asset' + END AS to_tune_asset, + liq_fee_in_cacao_e8, + to_e8, + from_e8, + swap_slip_bp, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + GROUP BY + to_tune_asset, + pool_name, + DAY + HAVING + to_tune_asset = 'to_asset' +), +swap_to_cacao_tbl AS ( + SELECT + DAY, + pool_name, + SUM(liq_fee_in_cacao_e8) AS to_cacao_fees, + SUM(to_e8) AS to_cacao_volume, + COUNT(*) AS to_cacao_count, + AVG(swap_slip_bp) AS to_cacao_average_slip + FROM( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 'to_cacao' + ELSE 'to_asset' + END AS to_tune_asset, + liq_fee_in_cacao_e8, + to_e8, + from_e8, + swap_slip_bp, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN 0 + ELSE liq_fee_e8 + END AS asset_fee + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + ) + GROUP BY + to_tune_asset, + pool_name, + DAY + HAVING + to_tune_asset = 'to_cacao' +), +average_slip_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + AVG(swap_slip_bp) AS average_slip + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + pool_name, + DAY +), +unique_swapper_tbl AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + COUNT( + DISTINCT from_address + ) AS unique_swapper_count + FROM + {{ ref("silver__swap_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + pool_name, + DAY +), +stake_amount AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + pool_name, + SUM(stake_units) AS units + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + pool_name, + DAY +), +unstake_umc AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + from_address AS address, + pool_name, + SUM(stake_units) AS unstake_liquidity_units + FROM + {{ ref("silver__withdraw_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + from_address, + pool_name, + DAY +), +stake_umc AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + cacao_address AS address, + pool_name, + SUM(stake_units) AS liquidity_units + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + cacao_address IS NOT NULL + GROUP BY + cacao_address, + pool_name, + DAY + UNION ALL + SELECT + DATE( + b.block_timestamp + ) AS DAY, + asset_address AS address, + pool_name, + SUM(stake_units) AS liquidity_units + FROM + {{ ref("silver__stake_events") }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + asset_address IS NOT NULL + AND cacao_address IS NULL + GROUP BY + asset_address, + pool_name, + DAY +), +unique_member_count AS ( + SELECT + DAY, + pool_name, + COUNT( + DISTINCT address + ) AS unique_member_count + FROM + ( + SELECT + stake_umc.day, + stake_umc.pool_name, + stake_umc.address, + stake_umc.liquidity_units, + CASE + WHEN unstake_umc.unstake_liquidity_units IS NOT NULL THEN unstake_umc.unstake_liquidity_units + ELSE 0 + END AS unstake_liquidity_units + FROM + stake_umc + LEFT JOIN unstake_umc + ON stake_umc.address = unstake_umc.address + AND stake_umc.pool_name = unstake_umc.pool_name + ) + WHERE + liquidity_units - unstake_liquidity_units > 0 + GROUP BY + pool_name, + DAY +), +asset_price_usd_tbl AS ( + SELECT + DAY, + pool_name, + asset_usd AS asset_price_usd + FROM + ( + SELECT + DATE(block_timestamp) AS DAY, + block_id, + MAX(block_id) over (PARTITION BY pool_name, DATE(block_timestamp)) AS max_block_id, + pool_name, + asset_usd + FROM + {{ ref("silver__prices") }} + ) + WHERE + block_id = max_block_id +), +joined AS ( + SELECT + pool_depth.day AS DAY, + COALESCE( + add_asset_liquidity_volume, + 0 + ) AS add_asset_liquidity_volume, + COALESCE( + add_liquidity_count, + 0 + ) AS add_liquidity_count, + COALESCE( + ( + add_asset_liquidity_volume + add_cacao_liquidity_volume + ), + 0 + ) AS add_liquidity_volume, + COALESCE( + add_cacao_liquidity_volume, + 0 + ) AS add_cacao_liquidity_volume, + pool_depth.pool_name AS asset, + asset_depth, + COALESCE( + asset_price, + 0 + ) AS asset_price, + COALESCE( + asset_price_usd, + 0 + ) AS asset_price_usd, + COALESCE( + average_slip, + 0 + ) AS average_slip, + COALESCE( + impermanent_loss_protection_paid, + 0 + ) AS impermanent_loss_protection_paid, + COALESCE( + cacao_depth, + 0 + ) AS cacao_depth, + COALESCE( + synth_depth, + 0 + ) AS synth_depth, + COALESCE( + status, + 'no status' + ) AS status, + COALESCE((to_cacao_count + to_asset_count), 0) AS swap_count, + COALESCE( + swap_volume, + 0 + ) AS swap_volume, + COALESCE( + to_asset_average_slip, + 0 + ) AS to_asset_average_slip, + COALESCE( + to_asset_count, + 0 + ) AS to_asset_count, + COALESCE( + to_asset_fees, + 0 + ) AS to_asset_fees, + COALESCE( + to_asset_volume, + 0 + ) AS to_asset_volume, + COALESCE( + to_cacao_average_slip, + 0 + ) AS to_cacao_average_slip, + COALESCE( + to_cacao_count, + 0 + ) AS to_cacao_count, + COALESCE( + to_cacao_fees, + 0 + ) AS to_cacao_fees, + COALESCE( + to_cacao_volume, + 0 + ) AS to_cacao_volume, + COALESCE((to_cacao_fees + to_asset_fees), 0) AS totalFees, + COALESCE( + unique_member_count, + 0 + ) AS unique_member_count, + COALESCE( + unique_swapper_count, + 0 + ) AS unique_swapper_count, + COALESCE( + units, + 0 + ) AS units, + COALESCE( + withdraw_asset_volume, + 0 + ) AS withdraw_asset_volume, + COALESCE( + withdraw_count, + 0 + ) AS withdraw_count, + COALESCE( + withdraw_cacao_volume, + 0 + ) AS withdraw_cacao_volume, + COALESCE((withdraw_cacao_volume + withdraw_asset_volume), 0) AS withdraw_volume, + SUM(COALESCE(added_stake, 0) - COALESCE(withdrawn_stake, 0)) over ( + PARTITION BY pool_depth.pool_name + ORDER BY + pool_depth.day ASC + ) AS total_stake, + asset_depth * COALESCE( + cacao_depth, + 0 + ) AS depth_product, + total_stake * synth_depth / ((asset_depth * 2) - synth_depth) AS synth_units, + CASE + WHEN total_stake = 0 THEN 0 + WHEN depth_product < 0 THEN 0 + ELSE SQRT(depth_product) / ( + total_stake + synth_units + ) + END AS liquidity_unit_value_index + FROM + pool_depth + LEFT JOIN pool_status + ON pool_depth.pool_name = pool_status.pool_name + AND pool_depth.day = pool_status.day + LEFT JOIN add_liquidity_tbl + ON pool_depth.pool_name = add_liquidity_tbl.pool_name + AND pool_depth.day = add_liquidity_tbl.day + LEFT JOIN withdraw_tbl + ON pool_depth.pool_name = withdraw_tbl.pool_name + AND pool_depth.day = withdraw_tbl.day + LEFT JOIN swap_total_tbl + ON pool_depth.pool_name = swap_total_tbl.pool_name + AND pool_depth.day = swap_total_tbl.day + LEFT JOIN swap_to_asset_tbl + ON pool_depth.pool_name = swap_to_asset_tbl.pool_name + AND pool_depth.day = swap_to_asset_tbl.day + LEFT JOIN swap_to_cacao_tbl + ON pool_depth.pool_name = swap_to_cacao_tbl.pool_name + AND pool_depth.day = swap_to_cacao_tbl.day + LEFT JOIN unique_swapper_tbl + ON pool_depth.pool_name = unique_swapper_tbl.pool_name + AND pool_depth.day = unique_swapper_tbl.day + LEFT JOIN stake_amount + ON pool_depth.pool_name = stake_amount.pool_name + AND pool_depth.day = stake_amount.day + LEFT JOIN average_slip_tbl + ON pool_depth.pool_name = average_slip_tbl.pool_name + AND pool_depth.day = average_slip_tbl.day + LEFT JOIN unique_member_count + ON pool_depth.pool_name = unique_member_count.pool_name + AND pool_depth.day = unique_member_count.day + LEFT JOIN asset_price_usd_tbl + ON pool_depth.pool_name = asset_price_usd_tbl.pool_name + AND pool_depth.day = asset_price_usd_tbl.day +) +SELECT + DISTINCT DAY, + add_asset_liquidity_volume, + add_liquidity_count, + add_liquidity_volume, + add_cacao_liquidity_volume, + asset, + asset_depth, + asset_price, + asset_price_usd, + average_slip, + impermanent_loss_protection_paid, + cacao_depth, + status, + swap_count, + swap_volume, + to_asset_average_slip, + to_asset_count, + to_asset_fees, + to_asset_volume, + to_cacao_average_slip, + to_cacao_count, + to_cacao_fees, + to_cacao_volume, + totalFees, + unique_member_count, + unique_swapper_count, + units, + withdraw_asset_volume, + withdraw_count, + withdraw_cacao_volume, + withdraw_volume, + total_stake, + depth_product, + synth_units, + total_stake + synth_units AS pool_units, + liquidity_unit_value_index, + LAG( + liquidity_unit_value_index, + 1 + ) over ( + PARTITION BY asset + ORDER BY + DAY ASC + ) AS prev_liquidity_unit_value_index, + concat_ws( + '-', + DAY, + asset + ) AS _unique_key +FROM + joined diff --git a/models/silver/silver__pool_block_statistics.yml b/models/silver/silver__pool_block_statistics.yml new file mode 100644 index 0000000..45668e7 --- /dev/null +++ b/models/silver/silver__pool_block_statistics.yml @@ -0,0 +1,125 @@ +version: 2 +models: + - name: silver__pool_block_statistics + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + - ASSET + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: ADD_ASSET_LIQUIDITY_VOLUME + data_tests: + - not_null + - name: ADD_LIQUIDITY_COUNT + data_tests: + - not_null + - name: ADD_LIQUIDITY_VOLUME + data_tests: + - not_null + - name: ADD_cacao_LIQUIDITY_VOLUME + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_DEPTH + data_tests: + - not_null + - name: ASSET_PRICE + data_tests: + - not_null + - name: ASSET_PRICE_USD + data_tests: + - not_null + - name: AVERAGE_SLIP + data_tests: + - not_null + - name: IMPERMANENT_LOSS_PROTECTION_PAID + data_tests: + - not_null + - name: cacao_DEPTH + data_tests: + - not_null + - name: STATUS + data_tests: + - not_null + - name: SWAP_COUNT + data_tests: + - not_null + - name: SWAP_VOLUME + data_tests: + - not_null + - name: TO_ASSET_AVERAGE_SLIP + data_tests: + - not_null + - name: TO_ASSET_COUNT + data_tests: + - not_null + - name: TO_ASSET_FEES + data_tests: + - not_null + - name: TO_ASSET_VOLUME + data_tests: + - not_null + - name: TO_cacao_AVERAGE_SLIP + data_tests: + - not_null + - name: TO_cacao_COUNT + data_tests: + - not_null + - name: TO_cacao_FEES + data_tests: + - not_null + - name: TO_cacao_VOLUME + data_tests: + - not_null + - name: TOTALFEES + data_tests: + - not_null + - name: UNIQUE_MEMBER_COUNT + data_tests: + - not_null + - name: UNIQUE_SWAPPER_COUNT + data_tests: + - not_null + - name: UNITS + data_tests: + - not_null + - name: WITHDRAW_ASSET_VOLUME + data_tests: + - not_null + - name: WITHDRAW_COUNT + data_tests: + - not_null + - name: WITHDRAW_cacao_VOLUME + data_tests: + - not_null + - name: WITHDRAW_VOLUME + data_tests: + - not_null + - name: TOTAL_STAKE + data_tests: + - not_null + - name: DEPTH_PRODUCT + data_tests: + - not_null + - name: SYNTH_UNITS + data_tests: + - not_null + - name: POOL_UNITS + data_tests: + - not_null + - name: LIQUIDITY_UNIT_VALUE_INDEX + data_tests: + - not_null + - name: PREV_LIQUIDITY_UNIT_VALUE_INDEX + data_tests: + - not_null: + where: day <= '2021-04-10' + diff --git a/models/silver/silver__pool_events.sql b/models/silver/silver__pool_events.sql new file mode 100644 index 0000000..81c93b5 --- /dev/null +++ b/models/silver/silver__pool_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + asset, + status, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__pool_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__prices.sql b/models/silver/silver__prices.sql new file mode 100644 index 0000000..ee0e7a2 --- /dev/null +++ b/models/silver/silver__prices.sql @@ -0,0 +1,88 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.block_timestamp >= (select min(block_timestamp) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['block_timestamp::DATE'] +) }} +-- block level prices by pool +-- step 1 what is the USD pool with the highest balance (aka deepest pool) +WITH blocks AS ( + + SELECT + height AS block_id, + b.block_timestamp, + pool_name, + cacao_e8, + asset_e8 + FROM + {{ ref('silver__block_pool_depths') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +), +price AS ( + SELECT + height AS block_id, + b.block_timestamp, + cacao_price_e8 AS cacao_usd + FROM + {{ ref('silver__cacao_price') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +) -- step 3 calculate the prices of assets by pool, in terms of tokens per tokens +-- and in USD for both tokens +SELECT + DISTINCT b.block_id, + b.block_timestamp, + COALESCE( + cacao_e8 / asset_e8, + 0 + ) AS price_cacao_asset, + COALESCE( + asset_e8 / cacao_e8, + 0 + ) AS price_asset_cacao, + COALESCE(cacao_usd * (cacao_e8 / asset_e8), 0) AS asset_usd, + COALESCE( + cacao_usd, + 0 + ) AS cacao_usd, + pool_name, + concat_ws( + '-', + b.block_id :: STRING, + pool_name :: STRING + ) AS _unique_key +FROM + blocks b + JOIN price ru + ON b.block_id = ru.block_id +WHERE + cacao_e8 > 0 + AND asset_e8 > 0 diff --git a/models/silver/silver__prices.yml b/models/silver/silver__prices.yml new file mode 100644 index 0000000..206b7da --- /dev/null +++ b/models/silver/silver__prices.yml @@ -0,0 +1,33 @@ +version: 2 +models: + - name: silver__prices + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - POOL_NAME + columns: + - name: BLOCK_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: PRICE_cacao_ASSET + data_tests: + - not_null + - name: PRICE_ASSET_cacao + data_tests: + - not_null + - name: ASSET_USD + data_tests: + - not_null + - name: cacao_USD + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null diff --git a/models/silver/silver__refund_events.sql b/models/silver/silver__refund_events.sql new file mode 100644 index 0000000..dff481a --- /dev/null +++ b/models/silver/silver__refund_events.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + asset_2nd, + asset_2nd_e8, + memo, + code, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__refund_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__refund_events.yml b/models/silver/silver__refund_events.yml new file mode 100644 index 0000000..e454676 --- /dev/null +++ b/models/silver/silver__refund_events.yml @@ -0,0 +1,48 @@ +version: 2 +models: + - name: silver__refund_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - ASSET + - ASSET_2ND + - MEMO + - CODE + - REASON + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: ASSET_2ND_E8 + data_tests: + - not_null + - name: CODE + data_tests: + - not_null + - name: REASON + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__reserve_events.sql b/models/silver/silver__reserve_events.sql new file mode 100644 index 0000000..e946a1b --- /dev/null +++ b/models/silver/silver__reserve_events.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + memo, + addr AS address, + e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__reserve_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__rewards_event_entries.sql b/models/silver/silver__rewards_event_entries.sql new file mode 100644 index 0000000..b677d24 --- /dev/null +++ b/models/silver/silver__rewards_event_entries.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + cacao_e8, + saver_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__rewards_event_entries') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, pool +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__rewards_event_entries.yml b/models/silver/silver__rewards_event_entries.yml new file mode 100644 index 0000000..f61e874 --- /dev/null +++ b/models/silver/silver__rewards_event_entries.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__rewards_event_entries + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + - POOL_NAME + - EVENT_ID + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__rewards_events.sql b/models/silver/silver__rewards_events.sql new file mode 100644 index 0000000..bb8666e --- /dev/null +++ b/models/silver/silver__rewards_events.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__rewards_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__rewards_events.yml b/models/silver/silver__rewards_events.yml new file mode 100644 index 0000000..c5ae267 --- /dev/null +++ b/models/silver/silver__rewards_events.yml @@ -0,0 +1,14 @@ +version: 2 +models: + - name: silver__rewards_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: BOND_E8 + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__send_messages.sql b/models/silver/silver__send_messages.sql new file mode 100644 index 0000000..5b683ce --- /dev/null +++ b/models/silver/silver__send_messages.sql @@ -0,0 +1,25 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + amount_e8, + asset, + from_addr AS from_address, + to_addr AS to_address, + memo, + tx_id, + raw_log, + code, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__send_messages') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__send_messages.yml b/models/silver/silver__send_messages.yml new file mode 100644 index 0000000..7b39b40 --- /dev/null +++ b/models/silver/silver__send_messages.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: silver__send_messages + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + columns: + - name: AMOUNT_E8 + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: MEMO + - name: TX_ID + data_tests: + - not_null + - name: EVENT_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__set_ip_address_events.sql b/models/silver/silver__set_ip_address_events.sql new file mode 100644 index 0000000..541df67 --- /dev/null +++ b/models/silver/silver__set_ip_address_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + ip_addr, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_ip_address_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__set_mimir_events.sql b/models/silver/silver__set_mimir_events.sql new file mode 100644 index 0000000..7565b4c --- /dev/null +++ b/models/silver/silver__set_mimir_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + key, + VALUE, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_mimir_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__set_node_keys_events.sql b/models/silver/silver__set_node_keys_events.sql new file mode 100644 index 0000000..e3ac62a --- /dev/null +++ b/models/silver/silver__set_node_keys_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + secp256k1, + ed25519, + validator_consensus, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_node_keys_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__set_node_mimir_events.sql b/models/silver/silver__set_node_mimir_events.sql new file mode 100644 index 0000000..4f59ae7 --- /dev/null +++ b/models/silver/silver__set_node_mimir_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + address, + key, + VALUE, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_node_mimir_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__set_version_events.sql b/models/silver/silver__set_version_events.sql new file mode 100644 index 0000000..a539760 --- /dev/null +++ b/models/silver/silver__set_version_events.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + version, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__set_version_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__slash_events.sql b/models/silver/silver__slash_events.sql new file mode 100644 index 0000000..c754bc2 --- /dev/null +++ b/models/silver/silver__slash_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset, + asset_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__slash_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, pool, asset +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__slash_liquidity_events.sql b/models/silver/silver__slash_liquidity_events.sql new file mode 100644 index 0000000..525270b --- /dev/null +++ b/models/silver/silver__slash_liquidity_events.sql @@ -0,0 +1,23 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + bond_address, + lp_address, + asset, + lp_units, + asset_e8_loss, + cacao_e10_loss, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__slash_liquidity_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, bond_address, lp_address, asset +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__slash_points_events.sql b/models/silver/silver__slash_points_events.sql new file mode 100644 index 0000000..b535559 --- /dev/null +++ b/models/silver/silver__slash_points_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_address, + slash_points, + reason, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__slash_points_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__stake_events.sql b/models/silver/silver__stake_events.sql new file mode 100644 index 0000000..a905289 --- /dev/null +++ b/models/silver/silver__stake_events.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + pool AS pool_name, + asset_tx AS asset_tx_id, + asset_chain AS asset_blockchain, + asset_addr AS asset_address, + asset_e8, + stake_units, + cacao_tx AS cacao_tx_id, + cacao_addr AS cacao_address, + cacao_e8, + _ASSET_IN_CACAO_E8, + memo, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__stake_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, cacao_addr, asset_addr +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__stake_events.yml b/models/silver/silver__stake_events.yml new file mode 100644 index 0000000..d07d169 --- /dev/null +++ b/models/silver/silver__stake_events.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: silver__stake_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - POOL_NAME + - ASSET_TX_ID + - ASSET_BLOCKCHAIN + - ASSET_ADDRESS + - STAKE_UNITS + - CACAO_TX_ID + - CACAO_ADDRESS + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: POOL_NAME + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: STAKE_UNITS + data_tests: + - not_null + - name: CACAO_E8 + data_tests: + - not_null + - name: _ASSET_IN_CACAO_E8 + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__streamling_swap_details_events.sql b/models/silver/silver__streamling_swap_details_events.sql new file mode 100644 index 0000000..7e6304e --- /dev/null +++ b/models/silver/silver__streamling_swap_details_events.sql @@ -0,0 +1,30 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx_id, + INTERVAL, + quantity, + COUNT, + last_height, + deposit_asset, + deposit_e8, + in_asset, + in_e8, + out_asset, + out_e8, + failed_swaps, + failed_swap_reasons, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__streamling_swap_details_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__streamling_swap_details_events.yml b/models/silver/silver__streamling_swap_details_events.yml new file mode 100644 index 0000000..931a665 --- /dev/null +++ b/models/silver/silver__streamling_swap_details_events.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: silver__streamling_swap_details_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: INTERVAL + data_tests: + - not_null + - name: quantity + data_tests: + - not_null + - name: COUNT + data_tests: + - not_null + - name: last_height + data_tests: + - not_null + - name: deposit_asset + data_tests: + - not_null + - name: deposit_e8 + data_tests: + - not_null + - name: in_asset + data_tests: + - not_null + - name: in_e8 + data_tests: + - not_null + - name: out_asset + data_tests: + - not_null + - name: out_e8 + data_tests: + - not_null + - name: failed_swaps + - name: failed_swaps_reasons + - name: event_id + data_tests: + - not_null + - name: block_timestamp + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__swap_events.sql b/models/silver/silver__swap_events.sql new file mode 100644 index 0000000..9d7a2ef --- /dev/null +++ b/models/silver/silver__swap_events.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool AS pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + _STREAMING, + streaming_count, + streaming_quantity, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__swap_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__swap_events.yml b/models/silver/silver__swap_events.yml new file mode 100644 index 0000000..9c84f89 --- /dev/null +++ b/models/silver/silver__swap_events.yml @@ -0,0 +1,72 @@ +version: 2 +models: + - name: silver__swap_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - FROM_ASSET + - FROM_E8 + - TO_ASSET + - TO_E8 + - MEMO + - POOL_NAME + - TO_E8_MIN + - SWAP_SLIP_BP + - LIQ_FEE_E8 + - LIQ_FEE_IN_CACAO_E8 + - _DIRECTION + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: FROM_ASSET + data_tests: + - not_null + - name: FROM_E8 + data_tests: + - not_null + - name: TO_ASSET + data_tests: + - not_null + - name: TO_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: TO_E8_MIN + data_tests: + - not_null + - name: SWAP_SLIP_BP + data_tests: + - not_null + - name: LIQ_FEE_E8 + data_tests: + - not_null + - name: LIQ_FEE_IN_CACAO_E8 + data_tests: + - not_null + - name: _DIRECTION + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__swaps.sql b/models/silver/silver__swaps.sql new file mode 100644 index 0000000..4fc4804 --- /dev/null +++ b/models/silver/silver__swaps.sql @@ -0,0 +1,165 @@ +{{ config( + materialized = 'incremental', + unique_key = '_unique_key', + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH swaps AS ( + + SELECT + tx_id, + blockchain, + from_address, + to_address, + from_asset, + from_e8, + to_asset, + to_e8, + memo, + pool_name, + to_e8_min, + swap_slip_bp, + liq_fee_e8, + liq_fee_in_cacao_e8, + _DIRECTION, + event_id, + streaming_count, + streaming_quantity, + b.block_timestamp, + b.height AS block_id, + A._INSERTED_TIMESTAMP, + COUNT(1) over ( + PARTITION BY tx_id + ) AS n_tx, + RANK() over ( + PARTITION BY tx_id + ORDER BY + liq_fee_e8 ASC + ) AS rank_liq_fee + FROM + {{ ref('silver__swap_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} +) +SELECT + se.block_timestamp, + se.block_id, + tx_id, + blockchain, + se.pool_name, + from_address, + CASE + WHEN n_tx > 1 + AND rank_liq_fee = 1 + AND len(SPLIT(memo, ':') [4] :: STRING) = 43 THEN SPLIT( + memo, + ':' + ) [4] :: STRING + WHEN n_tx > 1 + AND LOWER(LEFT(memo, 1)) IN ( + 's', + '=' + ) + AND len(COALESCE(SPLIT(memo, ':') [2] :: STRING, '')) = 0 THEN from_address + ELSE SPLIT( + memo, + ':' + ) [2] :: STRING + END AS native_to_address, + to_address AS to_pool_address, + CASE + WHEN COALESCE(SPLIT(memo, ':') [4], '') = '' THEN NULL + WHEN CONTAINS(SPLIT(memo, ':') [4], '/') THEN SPLIT(SPLIT(memo, ':') [4], '/') [0] + ELSE SPLIT( + memo, + ':' + ) [4] :: STRING + END AS affiliate_address, + TRY_CAST( + CASE + WHEN COALESCE(SPLIT(memo, ':') [5], '') = '' THEN NULL + WHEN CONTAINS(SPLIT(memo, ':') [5], '/') THEN SPLIT(SPLIT(memo, ':') [5], '/') [0] + ELSE SPLIT( + memo, + ':' + ) [5] + END :: STRING AS INT + ) AS affiliate_fee_basis_points, + SPLIT(COALESCE(SPLIT(SPLIT(memo, '|') [0], ':') [4], ''), '/') AS affiliate_addresses_array, + ARRAY_AGG(TRY_CAST(TRIM(f.value) AS INTEGER)) within GROUP ( + ORDER BY + f.index + ) AS affiliate_fee_basis_points_array, + from_asset, + to_asset, + COALESCE(from_e8 / pow(10, 8), 0) AS from_amount, + COALESCE(to_e8 / pow(10, 8), 0) AS to_amount, + COALESCE(to_e8_min / pow(10, 8), 0) AS min_to_amount, + CASE + WHEN from_asset = 'MAYA.CACAO' THEN COALESCE(from_e8 * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(from_e8 * asset_usd / pow(10, 8), 0) + END AS from_amount_usd, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(to_e8 * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(to_e8 * asset_usd / pow(10, 8), 0) + END AS to_amount_usd, + cacao_usd, + asset_usd, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(to_e8_min * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(to_e8_min * asset_usd / pow(10, 8), 0) + END AS to_amount_min_usd, + swap_slip_bp, + COALESCE(liq_fee_in_cacao_e8 / pow(10, 8), 0) AS liq_fee_cacao, + COALESCE(liq_fee_in_cacao_e8 / pow(10, 8) * cacao_usd, 0) AS liq_fee_cacao_usd, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(liq_fee_e8 / pow(10, 8), 0) + ELSE COALESCE(liq_fee_e8 / pow(10, 8), 0) + END AS liq_fee_asset, + CASE + WHEN to_asset = 'MAYA.CACAO' THEN COALESCE(liq_fee_e8 * cacao_usd / pow(10, 8), 0) + ELSE COALESCE(liq_fee_e8 * asset_usd / pow(10, 8), 0) + END AS liq_fee_asset_usd, + streaming_count, + streaming_quantity, + concat_ws( + '-', + tx_id, + se.block_id, + to_asset, + from_asset, + COALESCE( + native_to_address, + '' + ), + from_address, + se.pool_name, + to_pool_address, + event_id + ) AS _unique_key, + _INSERTED_TIMESTAMP +FROM + swaps se + LEFT JOIN {{ ref('silver__prices') }} + p + ON se.block_id = p.block_id + AND se.pool_name = p.pool_name, + LATERAL FLATTEN( + input => SPLIT(COALESCE(SPLIT(memo, ':') [5], ''), '/') + ) f +GROUP BY + ALL diff --git a/models/silver/silver__swaps.yml b/models/silver/silver__swaps.yml new file mode 100644 index 0000000..4221eda --- /dev/null +++ b/models/silver/silver__swaps.yml @@ -0,0 +1,80 @@ +version: 2 +models: + - name: silver__swaps + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - TX_ID + - POOL_NAME + - FROM_ADDRESS + - TO_POOL_ADDRESS + - FROM_ASSET + - TO_ASSET + - NATIVE_TO_ADDRESS + - FROM_AMOUNT + - TO_AMOUNT + columns: + - name: BLOCK_ID + data_tests: + - not_null + - name: BLOCK_TIMESTAMP::DATE + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_POOL_ADDRESS + data_tests: + - not_null + - name: FROM_ASSET + data_tests: + - not_null + - name: TO_ASSET + data_tests: + - not_null + - name: FROM_AMOUNT + data_tests: + - not_null + - name: TO_AMOUNT + data_tests: + - not_null + - name: MIN_TO_AMOUNT + data_tests: + - not_null + - name: FROM_AMOUNT_USD + data_tests: + - not_null + - name: TO_AMOUNT_USD + data_tests: + - not_null + - name: TO_AMOUNT_MIN_USD + data_tests: + - not_null + - name: SWAP_SLIP_BP + data_tests: + - not_null + - name: LIQ_FEE_cacao + data_tests: + - not_null + - name: LIQ_FEE_cacao_USD + data_tests: + - not_null + - name: LIQ_FEE_ASSET + data_tests: + - not_null + - name: LIQ_FEE_ASSET_USD + data_tests: + - not_null diff --git a/models/silver/silver__total_block_rewards.sql b/models/silver/silver__total_block_rewards.sql new file mode 100644 index 0000000..b2e7aa9 --- /dev/null +++ b/models/silver/silver__total_block_rewards.sql @@ -0,0 +1,137 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH block_prices AS ( + + SELECT + AVG(cacao_usd) AS cacao_usd, + block_id + FROM + {{ ref('silver__prices') }} + GROUP BY + block_id +), +fin AS ( + SELECT + b.block_timestamp, + b.height AS block_id, + ree.pool_name AS reward_entity, + COALESCE(cacao_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(cacao_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + concat_ws( + '-', + b.height, + reward_entity + ) AS _unique_key, + ree._inserted_timestamp + FROM + {{ ref('silver__rewards_event_entries') }} + ree + JOIN {{ ref('silver__block_log') }} + b + ON ree.block_timestamp = b.timestamp + LEFT JOIN {{ ref('silver__prices') }} + p + ON b.height = p.block_id + AND ree.pool_name = p.pool_name + +{% if is_incremental() %} +WHERE + ( + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR concat_ws( + '-', + b.height, + reward_entity + ) IN ( + SELECT + _unique_key + FROM + {{ this }} + WHERE + cacao_amount_USD IS NULL + ) + ) +{% endif %} +UNION +SELECT + b.block_timestamp, + b.height AS block_id, + 'bond_holders' AS reward_entity, + bond_e8 / pow( + 10, + 8 + ) AS cacao_amount, + bond_e8 / pow( + 10, + 8 + ) * cacao_usd AS cacao_amount_usd, + concat_ws( + '-', + b.height, + reward_entity + ) AS _unique_key, + re._inserted_timestamp +FROM + {{ ref('silver__rewards_events') }} + re + JOIN {{ ref('silver__block_log') }} + b + ON re.block_timestamp = b.timestamp + LEFT JOIN block_prices p + ON b.height = p.block_id + +{% if is_incremental() %} +WHERE + ( + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) + OR concat_ws( + '-', + b.height, + reward_entity + ) IN ( + SELECT + _unique_key + FROM + {{ this }} + WHERE + cacao_amount_USD IS NULL + ) + ) +{% endif %} +) +SELECT + block_timestamp, + block_id, + reward_entity, + SUM( + cacao_amount + ) AS cacao_amount, + SUM(cacao_amount_usd) AS cacao_amount_usd, + _unique_key, + MAX(_inserted_timestamp) AS _inserted_timestamp +FROM + fin +GROUP BY + block_timestamp, + block_id, + reward_entity, + _unique_key diff --git a/models/silver/silver__total_block_rewards.yml b/models/silver/silver__total_block_rewards.yml new file mode 100644 index 0000000..1d1e77e --- /dev/null +++ b/models/silver/silver__total_block_rewards.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__total_block_rewards + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_ID + - REWARD_ENTITY + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - name: BLOCK_ID + data_tests: + - not_null + - name: REWARD_ENTITY + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + diff --git a/models/silver/silver__total_value_locked.sql b/models/silver/silver__total_value_locked.sql new file mode 100644 index 0000000..b1f6030 --- /dev/null +++ b/models/silver/silver__total_value_locked.sql @@ -0,0 +1,122 @@ +{{ config( + materialized = 'table', + cluster_by = ['day'] +) }} + +WITH bond_type_day AS ( + + SELECT + DATE( + b.block_timestamp + ) AS DAY, + bond_type, + (SUM(e8) / pow(10, 8)) AS cacao_amount, + MAX( + A._inserted_timestamp + ) AS _inserted_timestamp + FROM + {{ ref('silver__bond_events') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + GROUP BY + DAY, + bond_type +), +bond_type_day_direction AS ( + SELECT + DAY, + bond_type, + CASE + WHEN bond_type IN ( + 'bond_returned', + 'bond_cost' + ) THEN -1 + ELSE 1 + END AS direction, + cacao_amount, + cacao_amount * direction AS abs_CACAO_amount, + _inserted_timestamp + FROM + bond_type_day +), +total_value_bonded_tbl AS ( + SELECT + DAY, + SUM(abs_CACAO_amount) AS total_value_bonded, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + bond_type_day_direction + GROUP BY + DAY +), +total_pool_depth AS ( + SELECT + DATE( + b.block_timestamp + ) AS DAY, + b.height AS block_id, + pool_name, + cacao_e8, + asset_e8, + MAX(height) over (PARTITION BY pool_name, DATE(b.block_timestamp)) AS max_block_id, + A._inserted_timestamp + FROM + {{ ref('silver__block_pool_depths') }} A + JOIN {{ ref('silver__block_log') }} + b + ON A.block_timestamp = b.timestamp + WHERE + LOWER(pool_name) NOT LIKE 'thor.%' +), +total_pool_depth_max AS ( + SELECT + DAY, + cacao_e8 AS cacao_depth, + asset_e8 AS asset_depth, + _inserted_timestamp + FROM + total_pool_depth + WHERE + block_id = max_block_id +), +total_value_pooled_tbl AS ( + SELECT + DAY, + SUM(cacao_depth) * 2 / power( + 10, + 8 + ) AS total_value_pooled, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + total_pool_depth_max + GROUP BY + DAY +) +SELECT + COALESCE( + total_value_bonded_tbl.day, + total_value_pooled_tbl.day + ) AS DAY, + COALESCE( + total_value_pooled, + 0 + ) AS total_value_pooled, + COALESCE(SUM(total_value_bonded) over ( +ORDER BY + COALESCE(total_value_bonded_tbl.day, total_value_pooled_tbl.day) ASC), 0) AS total_value_bonded, + COALESCE( + total_value_pooled, + 0 + ) + SUM(COALESCE(total_value_bonded, 0)) over ( + ORDER BY + COALESCE( + total_value_bonded_tbl.day, + total_value_pooled_tbl.day + ) ASC + ) AS total_value_locked, + total_value_bonded_tbl._inserted_timestamp +FROM + total_value_bonded_tbl full + JOIN total_value_pooled_tbl + ON total_value_bonded_tbl.day = total_value_pooled_tbl.day diff --git a/models/silver/silver__total_value_locked.yml b/models/silver/silver__total_value_locked.yml new file mode 100644 index 0000000..7b44679 --- /dev/null +++ b/models/silver/silver__total_value_locked.yml @@ -0,0 +1,23 @@ +version: 2 +models: + - name: silver__total_value_locked + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - DAY + columns: + - name: DAY + data_tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - name: TOTAL_VALUE_POOLED + data_tests: + - not_null + - name: TOTAL_VALUE_BONDED + data_tests: + - not_null + - name: TOTAL_VALUE_LOCKED + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__transfer_events.sql b/models/silver/silver__transfer_events.sql new file mode 100644 index 0000000..e22b103 --- /dev/null +++ b/models/silver/silver__transfer_events.sql @@ -0,0 +1,21 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + from_addr AS from_address, + to_addr AS to_address, + asset, + amount_e8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__transfer_events') }} + qualify(ROW_NUMBER() over(PARTITION BY event_id, from_addr, to_addr, asset, amount_e8 +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__transfer_events.yml b/models/silver/silver__transfer_events.yml new file mode 100644 index 0000000..0197937 --- /dev/null +++ b/models/silver/silver__transfer_events.yml @@ -0,0 +1,19 @@ +version: 2 +models: + - name: silver__transfer_events + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: AMOUNT_E8 + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null diff --git a/models/silver/silver__transfers.sql b/models/silver/silver__transfers.sql new file mode 100644 index 0000000..85f85ff --- /dev/null +++ b/models/silver/silver__transfers.sql @@ -0,0 +1,51 @@ +{{ config( + materialized = 'incremental', + unique_key = "_unique_key", + incremental_strategy = 'merge', + cluster_by = ['block_timestamp::DATE'] +) }} + +WITH block_prices AS ( + + SELECT + AVG(cacao_usd) AS cacao_usd, + block_id + FROM + {{ ref('silver__prices') }} + GROUP BY + block_id +) +SELECT + b.block_timestamp, + b.height AS block_id, + from_address, + to_address, + asset, + COALESCE(amount_e8 / pow(10, 8), 0) AS cacao_amount, + COALESCE(amount_e8 / pow(10, 8) * cacao_usd, 0) AS cacao_amount_usd, + event_id, + {{ dbt_utils.generate_surrogate_key( + ['se.event_id', 'se.from_address', 'se.to_address', 'se.asset', 'se.amount_e8'] + ) }} + _unique_key, + se._inserted_timestamp +FROM + {{ ref('silver__transfer_events') }} + se + JOIN {{ ref('silver__block_log') }} + b + ON se.block_timestamp = b.timestamp + LEFT JOIN block_prices p + ON b.height = p.block_id + +{% if is_incremental() %} +WHERE + b.block_timestamp >= ( + SELECT + MAX( + block_timestamp - INTERVAL '1 HOUR' + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/silver__transfers.yml b/models/silver/silver__transfers.yml new file mode 100644 index 0000000..017aee7 --- /dev/null +++ b/models/silver/silver__transfers.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__transfers + + columns: + - name: BLOCK_TIMESTAMP + data_tests: + - not_null + - name: BLOCK_ID + data_tests: + - not_null + - name: ASSET + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: cacao_AMOUNT + data_tests: + - not_null + - name: cacao_AMOUNT_USD + data_tests: + - not_null: + where: BLOCK_TIMESTAMP <= SYSDATE() - interval '2 day' diff --git a/models/silver/silver__update_node_account_status_events.sql b/models/silver/silver__update_node_account_status_events.sql new file mode 100644 index 0000000..1865abc --- /dev/null +++ b/models/silver/silver__update_node_account_status_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + node_addr AS node_address, + "CURRENT" AS current_status, + former AS former_status, + block_timestamp, + event_id, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__update_node_account_status_events') }} + e qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__update_node_account_status_events.yml b/models/silver/silver__update_node_account_status_events.yml new file mode 100644 index 0000000..73289b4 --- /dev/null +++ b/models/silver/silver__update_node_account_status_events.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__update_node_account_status_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - NODE_ADDRESS + - CURRENT_STATUS + - FORMER_STATUS + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: NODE_ADDRESS + data_tests: + - not_null + - name: CURRENT_STATUS + data_tests: + - not_null + - name: FORMER_STATUS + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__validator_request_leave_events.sql b/models/silver/silver__validator_request_leave_events.sql new file mode 100644 index 0000000..6bd53ff --- /dev/null +++ b/models/silver/silver__validator_request_leave_events.sql @@ -0,0 +1,20 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + from_addr AS from_address, + node_addr AS node_address, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref('bronze__validator_request_leave_events') }} + e qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__INGESTED_AT DESC)) = 1 diff --git a/models/silver/silver__validator_request_leave_events.yml b/models/silver/silver__validator_request_leave_events.yml new file mode 100644 index 0000000..9a3ff53 --- /dev/null +++ b/models/silver/silver__validator_request_leave_events.yml @@ -0,0 +1,27 @@ +version: 2 +models: + - name: silver__validator_request_leave_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - EVENT_ID + - TX_ID + - BLOCK_TIMESTAMP + - FROM_ADDRESS + - NODE_ADDRESS + columns: + - name: EVENT_ID + data_tests: + - not_null + - name: TX_ID + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: NODE_ADDRESS + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/silver/silver__withdraw_events.sql b/models/silver/silver__withdraw_events.sql new file mode 100644 index 0000000..9284337 --- /dev/null +++ b/models/silver/silver__withdraw_events.sql @@ -0,0 +1,34 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + tx AS tx_id, + chain AS blockchain, + from_addr AS from_address, + to_addr AS to_address, + asset, + asset_e8, + emit_asset_e8, + emit_cacao_e8, + memo, + pool AS pool_name, + stake_units, + basis_points, + asymmetry, + imp_loss_protection_e8, + _EMIT_ASSET_IN_CACAO_E8, + event_id, + block_timestamp, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS _INSERTED_TIMESTAMP +FROM + {{ ref( + 'bronze__withdraw_events' + ) }} + e qualify(ROW_NUMBER() over(PARTITION BY event_id +ORDER BY + __HEVO__LOADED_AT DESC)) = 1 diff --git a/models/silver/silver__withdraw_events.yml b/models/silver/silver__withdraw_events.yml new file mode 100644 index 0000000..35a8a9f --- /dev/null +++ b/models/silver/silver__withdraw_events.yml @@ -0,0 +1,64 @@ +version: 2 +models: + - name: silver__withdraw_events + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_ID + - BLOCKCHAIN + - FROM_ADDRESS + - TO_ADDRESS + - ASSET + - MEMO + - POOL_NAME + - BLOCK_TIMESTAMP + - EVENT_ID + columns: + - name: TX_ID + data_tests: + - not_null + - name: BLOCKCHAIN + data_tests: + - not_null + - name: FROM_ADDRESS + data_tests: + - not_null + - name: TO_ADDRESS + data_tests: + - not_null + - name: ASSET + data_tests: + - not_null + - name: ASSET_E8 + data_tests: + - not_null + - name: EMIT_ASSET_E8 + data_tests: + - not_null + - name: EMIT_cacao_E8 + data_tests: + - not_null + - name: MEMO + data_tests: + - not_null + - name: POOL_NAME + data_tests: + - not_null + - name: STAKE_UNITS + data_tests: + - not_null + - name: BASIS_POINTS + data_tests: + - not_null + - name: ASYMMETRY + data_tests: + - not_null + - name: IMP_LOSS_PROTECTION_E8 + data_tests: + - not_null + - name: _EMIT_ASSET_IN_cacao_E8 + data_tests: + - not_null + - name: BLOCK_TIMESTAMP + data_tests: + - not_null \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 0000000..2015920 --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,57 @@ +version: 2 + +sources: + - name: maya_midgard + database: HEVO + schema: MAYA_MIDGARD_2 + tables: + - name: midgard_active_vault_events + - name: midgard_add_events + - name: midgard_block_log + - name: midgard_block_pool_depths + - name: midgard_bond_events + - name: midgard_cacao_price + - name: midgard_constants + - name: midgard_failed_deposit_messages + - name: midgard_fee_events + - name: midgard_gas_events + - name: midgard_inactive_vault_events + - name: midgard_mayaname_change_events + - name: midgard_new_node_events + - name: midgard_outbound_events + - name: midgard_pending_liquidity_events + - name: midgard_pool_balance_change_events + - name: midgard_pool_events + - name: midgard_refund_events + - name: midgard_reserve_events + - name: midgard_rewards_events + - name: midgard_rewards_event_entries + - name: midgard_send_messages + - name: midgard_set_ip_address_events + - name: midgard_set_mimir_events + - name: midgard_set_node_keys_events + - name: midgard_set_node_mimir_events + - name: midgard_set_version_events + - name: midgard_slash_events + - name: midgard_slash_liquidity_events + - name: midgard_slash_points_events + - name: midgard_stake_events + - name: midgard_streaming_swap_details_events + - name: midgard_swap_events + - name: midgard_transfer_events + - name: midgard_update_node_account_status_events + - name: midgard_validator_request_leave_events + - name: midgard_withdraw_events + - name: crosschain + database: "{{ 'crosschain' if target.database == 'MAYA' else 'crosschain_dev' }}" + schema: core + tables: + - name: address_tags + - name: dim_dates + - name: crosschain_silver + database: "{{ 'crosschain' if target.database == 'MAYA' else 'crosschain_dev' }}" + schema: silver + tables: + - name: number_sequence + - name: complete_native_prices + - name: labels_combined \ No newline at end of file diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..9df15ab --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,18 @@ +packages: + - package: calogica/dbt_expectations + version: 0.8.5 + - git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: 5f6015eb8501e2b238cd455f494203385de38107 + - package: get-select/dbt_snowflake_query_tags + version: 2.5.0 + - package: dbt-labs/dbt_external_tables + version: 0.8.2 + - package: dbt-labs/dbt_utils + version: 1.0.0 + - package: Snowflake-Labs/dbt_constraints + version: 1.0.4 + - package: calogica/dbt_date + version: 0.7.2 + - git: https://github.com/FlipsideCrypto/livequery-base.git + revision: 884d6171cd6c97275ce387432b0e3e60972c7e9a +sha1_hash: f3c151c26472240d5520abcfdc0cb07ec687c4bd diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..7038e65 --- /dev/null +++ b/packages.yml @@ -0,0 +1,13 @@ +packages: + - package: calogica/dbt_expectations + version: [">=0.4.0", "<0.9.0"] + - git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: v1.33.1 + - package: get-select/dbt_snowflake_query_tags + version: [">=2.0.0", "<3.0.0"] + - package: dbt-labs/dbt_external_tables + version: 0.8.2 + - package: dbt-labs/dbt_utils + version: 1.0.0 + - package: Snowflake-Labs/dbt_constraints + version: [">=0.4.0"] \ No newline at end of file diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..f69764a --- /dev/null +++ b/profiles.yml @@ -0,0 +1,29 @@ +maya: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + prod: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + config: + send_anonymous_usage_stats: False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..235f1a5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +dbt-core>=1.8,<1.9 +dbt-snowflake>=1.8,<1.9 +protobuf==4.25.3 \ No newline at end of file