diff --git a/models/defillama/bronze/bronze__defillama_historical_yields_100.sql b/models/defillama/bronze/bronze__defillama_historical_yields_100.sql index 766fdb4..0d8de34 100644 --- a/models/defillama/bronze/bronze__defillama_historical_yields_100.sql +++ b/models/defillama/bronze/bronze__defillama_historical_yields_100.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'defillama_historical_yields_id', - tags = ['defillama'] + tags = ['100'] ) }} WITH historical_yield AS ( diff --git a/models/defillama/bronze/bronze__defillama_historical_yields_101_200.sql b/models/defillama/bronze/bronze__defillama_historical_yields_101_200.sql index 26bef40..382c7bd 100644 --- a/models/defillama/bronze/bronze__defillama_historical_yields_101_200.sql +++ b/models/defillama/bronze/bronze__defillama_historical_yields_101_200.sql @@ -1,17 +1,10 @@ {{ config( materialized = 'incremental', unique_key = 'defillama_historical_yields_id', - tags = ['defillama','200'] + tags = ['200'] ) }} WITH -{# upstream_complete_check as ( - select - 'check' - FROM - {{ ref('bronze__defillama_historical_yields_101_200') }} -), #} - historical_yield AS ( {% for item in range(100) %} diff --git a/models/defillama/bronze/bronze__defillama_historical_yields_201_300.sql b/models/defillama/bronze/bronze__defillama_historical_yields_201_300.sql index ac16402..7d59dce 100644 --- a/models/defillama/bronze/bronze__defillama_historical_yields_201_300.sql +++ b/models/defillama/bronze/bronze__defillama_historical_yields_201_300.sql @@ -1,17 +1,10 @@ {{ config( materialized = 'incremental', unique_key = 'defillama_historical_yields_id', - tags = ['defillama','300'] + tags = ['300'] ) }} WITH -{# upstream_complete_check as ( - select - 'check' - FROM - {{ ref('bronze__defillama_historical_yields_101_200') }} -), #} - historical_yield AS ( {% for item in range(100) %} diff --git a/models/defillama/bronze/bronze__defillama_historical_yields_301_400.sql b/models/defillama/bronze/bronze__defillama_historical_yields_301_400.sql index 215047c..94509fa 100644 --- a/models/defillama/bronze/bronze__defillama_historical_yields_301_400.sql +++ b/models/defillama/bronze/bronze__defillama_historical_yields_301_400.sql @@ -1,17 +1,10 @@ {{ config( materialized = 'incremental', unique_key = 'defillama_historical_yields_id', - tags = ['defillama','400'] + tags = ['400'] ) }} WITH -{# upstream_complete_check as ( - select - 'check' - FROM - {{ ref('bronze__defillama_historical_yields_201_300') }} -), #} - historical_yield AS ( {% for item in range(100) %} diff --git a/models/defillama/bronze/bronze__defillama_historical_yields_401_500.sql b/models/defillama/bronze/bronze__defillama_historical_yields_401_500.sql index 27d8e6a..538bf76 100644 --- a/models/defillama/bronze/bronze__defillama_historical_yields_401_500.sql +++ b/models/defillama/bronze/bronze__defillama_historical_yields_401_500.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'incremental', unique_key = 'defillama_historical_yields_id', - tags = ['defillama','500'] + tags = ['500'] ) }} WITH diff --git a/models/defillama/silver/silver__defillama_historical_yields.sql b/models/defillama/silver/silver__defillama_historical_yields.sql index 263f31b..1100365 100644 --- a/models/defillama/silver/silver__defillama_historical_yields.sql +++ b/models/defillama/silver/silver__defillama_historical_yields.sql @@ -1,5 +1,5 @@ {{ config( - materialized = 'table', + materialized = 'incremental', unique_key = 'defillama_historical_yields_id', tags = ['defillama'] ) }} @@ -10,21 +10,61 @@ WITH yield_union AS( * FROM {{ ref('bronze__defillama_historical_yields_100') }} +{% if is_incremental() %} +WHERE _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} UNION ALL SELECT * FROM {{ ref('bronze__defillama_historical_yields_101_200') }} +{% if is_incremental() %} +WHERE _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} UNION ALL SELECT * FROM {{ ref('bronze__defillama_historical_yields_201_300') }} +{% if is_incremental() %} +WHERE _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} UNION ALL SELECT * FROM {{ ref('bronze__defillama_historical_yields_301_400') }} +{% if is_incremental() %} +WHERE _inserted_timestamp >= ( + SELECT + MAX( + _inserted_timestamp + ) + FROM + {{ this }} +) +{% endif %} UNION ALL SELECT * @@ -34,4 +74,7 @@ WITH yield_union AS( SELECT * FROM - yield_union + yield_union qualify(ROW_NUMBER() over(PARTITION BY defillama_historical_yields_id +ORDER BY + _inserted_timestamp DESC)) = 1 + diff --git a/models/defillama/silver/silver__defillama_historical_yields.yml b/models/defillama/silver/silver__defillama_historical_yields.yml new file mode 100644 index 0000000..9cf4dfe --- /dev/null +++ b/models/defillama/silver/silver__defillama_historical_yields.yml @@ -0,0 +1,7 @@ +version: 2 +models: + - name: silver__defillama_historical_yields + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - defillama_historical_yields_id \ No newline at end of file diff --git a/snowflake_call.sh b/snowflake_call.sh index c7a4dca..30e445f 100755 --- a/snowflake_call.sh +++ b/snowflake_call.sh @@ -1,11 +1,24 @@ -# Execute the query and capture the output -q_result=$(dbt run-operation execute_and_print_query) +while true; do + # Execute the query and capture the output + q_result=$(dbt run-operation execute_and_print_query) -# Use grep and sed to extract numbers, assuming they represent tag identifiers -numbers=($(echo "$q_result" | grep '.*/\1/")) + # Use grep and sed to extract numbers, assuming they represent tag identifiers + numbers=($(echo "$q_result" | grep '.*/\1/")) -# Format the extracted numbers as dbt tag selectors -tag_selectors=$(printf "tag:%s " "${numbers[@]}") + # Format the extracted numbers as dbt tag selectors + tag_selectors=$(printf "tag:%s " "${numbers[@]}") + tag_selectors=$(echo "$tag_selectors" | sed 's/ $//') # Remove trailing space + echo $tag_selectors -# Trim any trailing whitespace and output -echo "$tag_selectors" | sed 's/ $//' \ No newline at end of file + # Check if tag_selectors is not empty and matches the pattern + if [[ -n "$tag_selectors" && "$tag_selectors" =~ ^tag:[0-9]+ ]]; then + echo "Valid tags received: $tag_selectors" + dbt run --select "$tag_selectors" + + # Run the dependent models + dbt run -m models/defillama/silver/silver__defillama_historical_yields.sql models/defillama/bronze/bronze__defillama_historical_backfill_list.sql + else + echo "No valid tags to run, exiting loop" + break + fi +done