diff --git a/models/bronze/bronze__ft_balances_daily.sql b/models/bronze/bronze__ft_balances_daily.sql new file mode 100644 index 0000000..66ab34b --- /dev/null +++ b/models/bronze/bronze__ft_balances_daily.sql @@ -0,0 +1,26 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + account_id, + epoch_block_height, + epoch_date, + liquid, + lockup_account_id, + lockup_liquid, + lockup_reward, + lockup_staked, + lockup_unstaked_not_liquid, + reward, + staked, + storage_usage, + unstaked_not_liquid, + __HEVO_ID, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT +FROM + {{ source( + 'hevo', + 'flipsidecrypto_near_ft_balances_daily' + ) }} diff --git a/models/gold/core/core__ez_native_daily_balances.sql b/models/gold/core/core__ez_native_daily_balances.sql new file mode 100644 index 0000000..50cbd01 --- /dev/null +++ b/models/gold/core/core__ez_native_daily_balances.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = 'incremental', + unique_key = "ez_near_daily_balances_id", + incremental_strategy = 'merge', + incremental_predicates = ['DBT_INTERNAL_DEST.epoch_date::DATE >= (select min(epoch_date::DATE) from ' ~ generate_tmp_view_name(this) ~ ')'], + cluster_by = ['epoch_date::DATE'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address);", + tags = ['scheduled_non_core'] +) }} + +SELECT + address, + epoch_block_id, + epoch_date, + liquid, + lockup_address, + lockup_liquid, + lockup_reward, + lockup_staked, + lockup_unstaked_not_liquid, + reward, + staked, + storage_usage, + unstaked_not_liquid, + {{ dbt_utils.generate_surrogate_key( + ['address','epoch_block_id'] + ) }} AS ez_near_daily_balances_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('silver__native_daily_balances') }} + +{% if is_incremental() %} +WHERE + epoch_date >= ( + SELECT + MAX(epoch_date) + FROM + {{ this }} + ) +{% endif %} + +--handle potential duplicates introduced by the hevo +qualify(ROW_NUMBER() over (PARTITION BY address, epoch_date +ORDER BY + modified_timestamp DESC) = 1) diff --git a/models/gold/core/core__ez_native_daily_balances.yml b/models/gold/core/core__ez_native_daily_balances.yml new file mode 100644 index 0000000..ebe0270 --- /dev/null +++ b/models/gold/core/core__ez_native_daily_balances.yml @@ -0,0 +1,78 @@ +version: 2 + +models: + - name: core__ez_native_daily_balances + description: This table records native near balances for accounts with abcdef + tests: + - dbt_utils.recency: + datepart: hour + field: epoch_date + interval: 48 + + columns: + - name: address + description: The address of the balance is for + tests: + - not_null: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + - name: epoch_block_id + description: The block id of the start of the epoch + tests: + - not_null: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + + - name: epoch_date + description: The date and time at which the epoch began + tests: + - not_null: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + + - name: liquid + description: The liquid balance of the address at this block height + tests: + - not_null: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + + - name: lockup_address + description: The address of the lockup + + - name: lockup_liquid + description: "" + + - name: lockup_reward + description: "" + + - name: lockup_staked + description: "" + + - name: lockup_unstaked_not_liquid + description: "" + + - name: reward + description: The reward balance of the address at this block height + + - name: staked + description: The staked balance of the address at this block height + tests: + - not_null: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + + - name: storage_usage + description: "" + + - name: storage_usage + description: "" + + - name: ez_near_daily_balances_id + description: "{{doc('id')}}" + tests: + - unique: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + - not_null: + where: epoch_date > SYSDATE() - INTERVAL '7 days' + + - name: INSERTED_TIMESTAMP + description: "{{doc('inserted_timestamp')}}" + + - name: MODIFIED_TIMESTAMP + description: "{{doc('modified_timestamp')}}" diff --git a/models/silver/balances/silver__native_daily_balances.sql b/models/silver/balances/silver__native_daily_balances.sql new file mode 100644 index 0000000..bab04f4 --- /dev/null +++ b/models/silver/balances/silver__native_daily_balances.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'view' +) }} + +SELECT + account_id AS address, + epoch_block_height :: INT AS epoch_block_id, + epoch_date :: DATE AS epoch_date, + liquid, + lockup_account_id AS lockup_address, + lockup_liquid, + lockup_reward, + lockup_staked, + lockup_unstaked_not_liquid, + reward, + staked, + storage_usage, + unstaked_not_liquid, + __HEVO_ID, + __HEVO__INGESTED_AT, + __HEVO__LOADED_AT, + {{ dbt_utils.generate_surrogate_key( + ['account_id','epoch_block_height'] + ) }} AS near_balances_daily_id, + DATEADD( + ms, + __HEVO__LOADED_AT, + '1970-01-01' + ) AS inserted_timestamp, + inserted_timestamp AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref( + 'bronze__ft_balances_daily' + ) }} diff --git a/models/sources.yml b/models/sources.yml index ca6149a..11c789e 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -27,3 +27,9 @@ sources: tables: - name: blocks - name: shards + + - name: hevo + schema: near + database: hevo + tables: + - name: flipsidecrypto_near_ft_balances_daily \ No newline at end of file