diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 2577c70..f18ef9c 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -44,6 +44,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | + dbt run-operation run_sp_refresh_external_tables_aptos dbt run -m external_models,tag:defillama external_models,tag:deepnftvalue external_models,tag:core external_models,tag:blast external_models,tag:polymarket external_models,tag:bitquery external_models,tag:oklink external_models,tag:artemis external_models,tag:aptos_gas --exclude models/defillama/bronze/bronze__defillama_stablecoin_supply.sql+ notify-failure: diff --git a/macros/streamline/run_sp_refresh_external_tables_aptos.sql b/macros/streamline/run_sp_refresh_external_tables_aptos.sql new file mode 100644 index 0000000..a2cc45a --- /dev/null +++ b/macros/streamline/run_sp_refresh_external_tables_aptos.sql @@ -0,0 +1,7 @@ +{% macro run_sp_refresh_external_tables_aptos() %} +{% set sql %} +call bronze.sp_refresh_external_tables_aptos(); +{% endset %} + +{% do run_query(sql) %} +{% endmacro %} \ No newline at end of file diff --git a/macros/streamline/sp_refresh_external_tables_aptos.sql b/macros/streamline/sp_refresh_external_tables_aptos.sql new file mode 100644 index 0000000..7aad1ef --- /dev/null +++ b/macros/streamline/sp_refresh_external_tables_aptos.sql @@ -0,0 +1,15 @@ +{% macro sp_refresh_external_tables_aptos() %} +{% set sql %} +create or replace procedure bronze.sp_refresh_external_tables_aptos() +returns boolean +language sql +execute as caller +as +$$ + begin + alter external table EXTERNAL.BRONZE.aptos_shinam refresh; + return TRUE; + end; +$${% endset %} +{% do run_query(sql) %} +{% endmacro %} \ No newline at end of file diff --git a/models/aptos_gas/lync/silver_aptos__lync.sql b/models/aptos_gas/lync/silver_aptos__lync.sql index a9d9736..05954d5 100644 --- a/models/aptos_gas/lync/silver_aptos__lync.sql +++ b/models/aptos_gas/lync/silver_aptos__lync.sql @@ -29,7 +29,7 @@ SELECT b.value :count :: bigint AS metric_count, b.value :amount :: FLOAT AS metric_amount, b.value :amountInUSD :: FLOAT AS metric_amount_in_usd, - {{ dbt_utils.generate_surrogate_key([ 'metric', 'metric_date']) }} AS artemis_id, + {{ dbt_utils.generate_surrogate_key([ 'metric', 'metric_date']) }} AS lync_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/aptos_gas/shinam/bronze_aptos__shinam_data.sql b/models/aptos_gas/shinam/bronze_aptos__shinam_data.sql new file mode 100644 index 0000000..b99df89 --- /dev/null +++ b/models/aptos_gas/shinam/bronze_aptos__shinam_data.sql @@ -0,0 +1,21 @@ +{{ config ( + materialized = "view", + tags = ['aptos_gas'], +) }} + +SELECT + TO_DATE ( + file_date :: STRING, + 'YYYYMMDD' + ) AS metric_date, + fund_name, + workspace_name, + contact_email, + fund_balance_remaining :: bigint AS fund_balance_remaining, + total_sponsorships :: bigint AS total_sponsorships, + total_fees :: bigint AS total_fees +FROM + {{ source( + 'external_bronze', + 'aptos_shinam' + ) }} diff --git a/models/aptos_gas/shinam/silver_aptos__shinam.sql b/models/aptos_gas/shinam/silver_aptos__shinam.sql new file mode 100644 index 0000000..03d91a8 --- /dev/null +++ b/models/aptos_gas/shinam/silver_aptos__shinam.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = "incremental", + incremental_strategy = 'delete+insert', + unique_key = ['metric_date'], + tags = ['aptos_gas'], +) }} + +SELECT + metric_date, + fund_name, + workspace_name, + contact_email, + fund_balance_remaining, + total_sponsorships, + total_fees, + {{ dbt_utils.generate_surrogate_key([ 'metric_date', 'fund_name','workspace_name']) }} AS shinam_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('bronze_aptos__shinam_data') }} + +{% if is_incremental() %} +WHERE + metric_date >= ( + SELECT + MAX(metric_date) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/aptos_gas/shinam/silver_aptos__shinam.yml b/models/aptos_gas/shinam/silver_aptos__shinam.yml new file mode 100644 index 0000000..c9b5cd7 --- /dev/null +++ b/models/aptos_gas/shinam/silver_aptos__shinam.yml @@ -0,0 +1,35 @@ +version: 2 + +models: + - name: silver_aptos__shinam + description: "Shinam data" + columns: + + - name: metric_date + description: "The date of the metric measurement" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + + - name: fund_name + description: "Then the of the fund" + + - name: workspace_name + description: "The name of the workspace" + + + - name: contact_email + description: "The contact email of the fund" + + - name: fund_balance_remaining + description: "The fund balance remaining" + + - name: total_sponsorships + description: "The total sponsorships" + + - name: total_fees + description: "The total fees paid by the fund" + + \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 37c47f3..2cca6ea 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -85,6 +85,7 @@ sources: - name: farcaster_channel_follows - name: farcaster_profile_with_addresses - name: farcaster_warpcast_power_users + - name: aptos_shinam - name: avalanche_share database: ava_labs_share schema: primary