From 433e62a63cc827965047b2eab789a222b58e1413 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:22:24 -0600 Subject: [PATCH] AN-4553/deploy new defi gold views (#345) * bridge complete model * depr notice on ez. Tests on fact bridge. TODOs - naming convention for amount and use chain IDs?? * ez and fact bridging * dex swaps * fix id col name. add platform to dex * rm adj --- models/descriptions/amount_fee_usd.md | 5 + models/descriptions/amount_usd.md | 5 + models/descriptions/token_symbol.md | 5 + models/gold/defi/defi__ez_bridge_activity.sql | 56 ++++++++ models/gold/defi/defi__ez_bridge_activity.yml | 132 ++++++++++++++++++ .../defi/defi__ez_bridge_transactions.yml | 4 +- models/gold/defi/defi__ez_dex_swaps.sql | 69 +++++++++ models/gold/defi/defi__ez_dex_swaps.yml | 88 ++++++++++++ models/gold/defi/defi__ez_swaps.yml | 2 +- .../gold/defi/defi__fact_bridge_activity.sql | 24 ++++ .../gold/defi/defi__fact_bridge_activity.yml | 123 ++++++++++++++++ models/gold/defi/defi__fact_dex_swaps.sql | 27 ++++ models/gold/defi/defi__fact_dex_swaps.yml | 76 ++++++++++ .../transfers/silver__bridge_complete.sql | 99 +++++++++++++ 14 files changed, 712 insertions(+), 3 deletions(-) create mode 100644 models/descriptions/amount_fee_usd.md create mode 100644 models/descriptions/amount_usd.md create mode 100644 models/descriptions/token_symbol.md create mode 100644 models/gold/defi/defi__ez_bridge_activity.sql create mode 100644 models/gold/defi/defi__ez_bridge_activity.yml create mode 100644 models/gold/defi/defi__ez_dex_swaps.sql create mode 100644 models/gold/defi/defi__ez_dex_swaps.yml create mode 100644 models/gold/defi/defi__fact_bridge_activity.sql create mode 100644 models/gold/defi/defi__fact_bridge_activity.yml create mode 100644 models/gold/defi/defi__fact_dex_swaps.sql create mode 100644 models/gold/defi/defi__fact_dex_swaps.yml create mode 100644 models/silver/transfers/silver__bridge_complete.sql diff --git a/models/descriptions/amount_fee_usd.md b/models/descriptions/amount_fee_usd.md new file mode 100644 index 0000000..b165a9c --- /dev/null +++ b/models/descriptions/amount_fee_usd.md @@ -0,0 +1,5 @@ +{% docs amount_fee_usd %} + +Amount denominated in USD, where pricing data is available. + +{% enddocs %} diff --git a/models/descriptions/amount_usd.md b/models/descriptions/amount_usd.md new file mode 100644 index 0000000..020d640 --- /dev/null +++ b/models/descriptions/amount_usd.md @@ -0,0 +1,5 @@ +{% docs amount_usd %} + +Amount denominated in USD, where pricing data is available. + +{% enddocs %} diff --git a/models/descriptions/token_symbol.md b/models/descriptions/token_symbol.md new file mode 100644 index 0000000..023d5c8 --- /dev/null +++ b/models/descriptions/token_symbol.md @@ -0,0 +1,5 @@ +{% docs token_symbol %} + +Abbreviated symbol for the asset. + +{% enddocs %} diff --git a/models/gold/defi/defi__ez_bridge_activity.sql b/models/gold/defi/defi__ez_bridge_activity.sql new file mode 100644 index 0000000..c1dd383 --- /dev/null +++ b/models/gold/defi/defi__ez_bridge_activity.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'view', + tags = ['ez', 'bridge', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BRIDGE' }} } +) }} + +WITH prices AS ( + + SELECT + hour, + token_address, + symbol, + price + FROM + {{ ref('silver__complete_token_prices') }} + UNION ALL + SELECT + hour, + 'A.1654653399040a61.FlowToken' AS token_address, + symbol, + price + FROM + {{ ref('silver__complete_native_prices') }} +) +SELECT + tx_id, + block_timestamp, + block_height, + bridge_address, + b.token_address, + p.symbol AS token_symbol, + gross_amount AS amount, + amount_fee, + gross_amount * p.price AS amount_usd, + amount_fee * p.price AS amount_fee_usd, + source_address, + destination_address, + source_chain, + destination_chain, + platform, + bridge_complete_id AS ez_bridge_activity_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__bridge_complete') }} b + LEFT JOIN prices p + ON LOWER( + b.token_address + ) = LOWER( + p.token_address + ) + AND DATE_TRUNC( + 'hour', + b.block_timestamp + ) = p.hour + diff --git a/models/gold/defi/defi__ez_bridge_activity.yml b/models/gold/defi/defi__ez_bridge_activity.yml new file mode 100644 index 0000000..ec1d426 --- /dev/null +++ b/models/gold/defi/defi__ez_bridge_activity.yml @@ -0,0 +1,132 @@ +version: 2 + +models: + - name: defi__ez_bridge_activity + description: |- + This table parses transactions where tokens are bridged to or from the Flow network using Blocto teleport or the Celer bridge. + + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + where: block_height >= 55114467 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: BLOCK_HEIGHT + description: "{{ doc('block_height') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: BRIDGE_ADDRESS + description: "{{ doc('bridge_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TOKEN_ADDRESS + description: "{{ doc('token_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TOKEN_SYMBOL + description: "{{ doc('token_symbol') }}" + + - name: AMOUNT + description: "{{ doc('amount_adj') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - DOUBLE + - FLOAT + + - name: AMOUNT_FEE + description: "{{ doc('amount_fee') }}" + tests: + - not_null: + where: platform = 'blocto' + + - name: AMOUNT_USD + description: "{{ doc('amount_usd') }}" + + - name: AMOUNT_FEE_USD + description: "{{ doc('amount_fee_usd') }}" + + - name: SOURCE_ADDRESS + description: "{{ doc('flow_wallet_address') }}" + tests: + - not_null: + where: source_chain = 'flow' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: DESTINATION_ADDRESS + description: "{{ doc('flow_wallet_address') }}" + tests: + - not_null: + where: destination_chain = 'flow' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: SOURCE_CHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: DESTINATION_CHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: PLATFORM + description: "{{ doc('bridge') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: FACT_BRIDGE_TRANSACTIONS_ID + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/defi/defi__ez_bridge_transactions.yml b/models/gold/defi/defi__ez_bridge_transactions.yml index ce23d79..e153d3b 100644 --- a/models/gold/defi/defi__ez_bridge_transactions.yml +++ b/models/gold/defi/defi__ez_bridge_transactions.yml @@ -3,8 +3,8 @@ version: 2 models: - name: defi__ez_bridge_transactions description: |- - This table parses transactions where tokens are bridged to or from the Flow network using Blocto teleport or the Celer bridge. - + Deprecating Soon! This table will be deprecated on September 1st, please migrate work `defi__ez_bridge_activity` instead. + tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql new file mode 100644 index 0000000..77a1d57 --- /dev/null +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -0,0 +1,69 @@ +{{ config ( + materialized = 'view', + tags = ['ez', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } +) }} +WITH prices AS ( + + SELECT + hour, + token_address, + symbol, + price + FROM + {{ ref('silver__complete_token_prices') }} + UNION ALL + SELECT + hour, + 'A.1654653399040a61.FlowToken' AS token_address, + symbol, + price + FROM + {{ ref('silver__complete_native_prices') }} +) +SELECT + tx_id, + block_timestamp, + block_height, + swap_contract AS contract_address, + swap_index, + trader, + platform, + token_out_source AS origin_from_address, + token_out_contract AS token_out, + po.symbol AS token_out_symbol, + token_out_amount AS amount_out, + token_out_amount * po.price AS amount_out_usd, + token_in_destination AS origin_to_address, + token_in_contract AS token_in, + pi.symbol AS token_in_symbol, + token_in_amount AS amount_in, + token_in_amount * pi.price AS amount_in_usd, + swaps_final_id AS ez_dex_swaps_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__swaps_final') }} s + LEFT JOIN prices po + ON LOWER( + s.token_out_contract + ) = LOWER( + po.token_address + ) + AND DATE_TRUNC( + 'hour', + s.block_timestamp + ) = po.hour + LEFT JOIN prices pi + ON LOWER( + s.token_in_contract + ) = LOWER( + pi.token_address + ) + AND DATE_TRUNC( + 'hour', + s.block_timestamp + ) = pi.hour + +WHERE + token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__ez_dex_swaps.yml b/models/gold/defi/defi__ez_dex_swaps.yml new file mode 100644 index 0000000..06623b1 --- /dev/null +++ b/models/gold/defi/defi__ez_dex_swaps.yml @@ -0,0 +1,88 @@ +version: 2 + +models: + - name: defi__ez_dex_swaps + description: |- + This table records asset swaps on the Flow blockchain, with price and label information. + + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: BLOCK_HEIGHT + description: "{{ doc('block_height') }}" + tests: + - not_null + + - name: CONTRACT_ADDRESS + description: "{{ doc('swap_contract') }}" + tests: + - not_null + + - name: SWAP_INDEX + description: "{{ doc('swap_index') }}" + tests: + - not_null + + - name: TRADER + description: "{{ doc('trader') }}" + tests: + - not_null: + severity: warn + + - name: PLATFORM + description: "{{ doc('platform') }}" + + - name: ORIGIN_FROM_ADDRESS + description: "{{ doc('token_out_source') }}" + + - name: TOKEN_OUT + description: "{{ doc('token_out_contract') }}" + tests: + - not_null + + - name: TOKEN_OUT_SYMBOL + description: "{{ doc('symbol') }}" + + - name: AMOUNT_OUT + description: "{{ doc('token_out_amount') }}" + tests: + - not_null + + - name: AMOUNT_OUT_USD + description: "{{ doc('amount_usd') }}" + + - name: ORIGIN_TO_ADDRESS + description: "{{ doc('token_in_destination') }}" + + - name: TOKEN_IN + description: "{{ doc('token_in_contract') }}" + tests: + - not_null + + - name: TOKEN_IN_SYMBOL + description: "{{ doc('symbol') }}" + + - name: AMOUNT_IN + description: "{{ doc('token_in_amount') }}" + tests: + - not_null + + - name: AMOUNT_IN_USD + description: "{{ doc('amount_usd') }}" + + - name: EZ_DEX_SWAPS_ID + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/defi/defi__ez_swaps.yml b/models/gold/defi/defi__ez_swaps.yml index 42063ac..1f83bdd 100644 --- a/models/gold/defi/defi__ez_swaps.yml +++ b/models/gold/defi/defi__ez_swaps.yml @@ -3,7 +3,7 @@ version: 2 models: - name: defi__ez_swaps description: |- - This table records asset swaps on the Flow blockchain. + Deprecating Soon! Please migrate work to the new `defi.ez_dex_swaps` table by September 1st. columns: - name: TX_ID diff --git a/models/gold/defi/defi__fact_bridge_activity.sql b/models/gold/defi/defi__fact_bridge_activity.sql new file mode 100644 index 0000000..3ba03b9 --- /dev/null +++ b/models/gold/defi/defi__fact_bridge_activity.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'view', + tags = ['ez', 'bridge', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BRIDGE' }} } +) }} + +SELECT + tx_id, + block_timestamp, + block_height, + bridge_address, + token_address, + gross_amount AS amount, + amount_fee, + source_address, + destination_address, + source_chain, + destination_chain, + platform, + bridge_complete_id AS fact_bridge_activity_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__bridge_complete') }} diff --git a/models/gold/defi/defi__fact_bridge_activity.yml b/models/gold/defi/defi__fact_bridge_activity.yml new file mode 100644 index 0000000..ca8e69a --- /dev/null +++ b/models/gold/defi/defi__fact_bridge_activity.yml @@ -0,0 +1,123 @@ +version: 2 + +models: + - name: defi__fact_bridge_activity + description: |- + This table parses transactions where tokens are bridged to or from the Flow network using Blocto teleport or the Celer bridge. + + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + where: block_height >= 55114467 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: BLOCK_HEIGHT + description: "{{ doc('block_height') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: BRIDGE_ADDRESS + description: "{{ doc('bridge_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: TOKEN_ADDRESS + description: "{{ doc('token_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: AMOUNT + description: "{{ doc('amount') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - DOUBLE + - FLOAT + + - name: AMOUNT_FEE + description: "{{ doc('amount_fee') }}" + tests: + - not_null: + where: platform = 'blocto' + + - name: SOURCE_ADDRESS + description: "{{ doc('flow_wallet_address') }}" + tests: + - not_null: + where: source_chain = 'flow' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: DESTINATION_ADDRESS + description: "{{ doc('flow_wallet_address') }}" + tests: + - not_null: + where: destination_chain = 'flow' + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: SOURCE_CHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: DESTINATION_CHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: PLATFORM + description: "{{ doc('bridge') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: FACT_BRIDGE_TRANSACTIONS_ID + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/defi/defi__fact_dex_swaps.sql b/models/gold/defi/defi__fact_dex_swaps.sql new file mode 100644 index 0000000..51f4ffd --- /dev/null +++ b/models/gold/defi/defi__fact_dex_swaps.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['ez', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } +) }} + +SELECT + tx_id, + block_timestamp, + block_height, + swap_contract AS contract_address, + swap_index, + trader, + platform, + token_out_source AS origin_from_address, + token_out_contract AS token_out, + token_out_amount AS amount_out, + token_in_destination AS origin_to_address, + token_in_contract AS token_in, + token_in_amount AS amount_in, + swaps_final_id AS fact_dex_swaps_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__swaps_final') }} +WHERE + token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__fact_dex_swaps.yml b/models/gold/defi/defi__fact_dex_swaps.yml new file mode 100644 index 0000000..9397201 --- /dev/null +++ b/models/gold/defi/defi__fact_dex_swaps.yml @@ -0,0 +1,76 @@ +version: 2 + +models: + - name: defi__fact_dex_swaps + description: |- + This table records asset swaps on the Flow blockchain. + + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: BLOCK_HEIGHT + description: "{{ doc('block_height') }}" + tests: + - not_null + + - name: CONTRACT_ADDRESS + description: "{{ doc('swap_contract') }}" + tests: + - not_null + + - name: SWAP_INDEX + description: "{{ doc('swap_index') }}" + tests: + - not_null + + - name: TRADER + description: "{{ doc('trader') }}" + tests: + - not_null: + severity: warn + + - name: PLATFORM + description: "{{ doc('platform') }}" + + - name: ORIGIN_FROM_ADDRESS + description: "{{ doc('token_out_source') }}" + + - name: AMOUNT_OUT + description: "{{ doc('token_out_amount') }}" + tests: + - not_null + + - name: TOKEN_OUT + description: "{{ doc('token_out_contract') }}" + tests: + - not_null + + - name: ORIGIN_TO_ADDRESS + description: "{{ doc('token_in_destination') }}" + + - name: AMOUNT_IN + description: "{{ doc('token_in_amount') }}" + tests: + - not_null + + - name: TOKEN_IN + description: "{{ doc('token_in_contract') }}" + tests: + - not_null + + - name: FACT_DEX_SWAPS_ID + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/silver/transfers/silver__bridge_complete.sql b/models/silver/transfers/silver__bridge_complete.sql new file mode 100644 index 0000000..a248ad1 --- /dev/null +++ b/models/silver/transfers/silver__bridge_complete.sql @@ -0,0 +1,99 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::date'], + unique_key = 'bridge_complete_id', + tags = ['bridge', 'scheduled', 'streamline_scheduled', 'scheduled_non_core'] +) }} + +WITH +blocto AS ( + SELECT + tx_id, + block_timestamp, + block_height, + teleport_contract AS bridge_address, + token_contract AS token_address, + gross_amount, + amount_fee, + net_amount, + IFF(teleport_direction = 'inbound', flow_wallet_address, null) AS destination_address, + IFF(teleport_direction = 'outbound', flow_wallet_address, null) AS source_address, + IFF(teleport_direction = 'inbound', 'flow', blockchain) AS destination_chain, + IFF(teleport_direction = 'outbound', 'flow', blockchain) AS source_chain, + bridge AS platform, + inserted_timestamp, + modified_timestamp, + bridge_blocto_id AS bridge_complete_id + FROM + {{ ref('silver__bridge_blocto_s') }} +{% if is_incremental() %} +WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +celer AS ( + SELECT + tx_id, + block_timestamp, + block_height, + bridge_contract AS bridge_address, + token_contract AS token_address, + amount AS gross_amount, + NULL AS amount_fee, + amount AS net_amount, + IFF(direction = 'inbound', flow_wallet_address, counterparty) AS destination_address, + IFF(direction = 'outbound', flow_wallet_address, counterparty) AS source_address, + IFF(direction = 'inbound', 'flow', blockchain) AS destination_chain, + IFF(direction = 'outbound', 'flow', blockchain) AS source_chain, + bridge AS platform, + inserted_timestamp, + modified_timestamp, + bridge_celer_id AS bridge_complete_id + FROM + {{ ref('silver__bridge_celer_s') }} +{% if is_incremental() %} +WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +combo AS ( + SELECT + * + FROM + blocto + UNION ALL + SELECT + * + FROM + celer +) +SELECT + tx_id, + block_timestamp, + block_height, + bridge_address, + token_address, + gross_amount, + amount_fee, + net_amount, + source_address, + destination_address, + source_chain, + destination_chain, + platform, + bridge_complete_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id, +FROM + combo