Merge pull request #33 from FlipsideCrypto/curated_models_macro

macro to curated models
2026-02-06 11:06:45 +00:00 · 2023-01-25 12:48:42 -07:00 · 2023-01-25 12:48:42 -07:00 · 992f84275f
commit 992f84275f
parent 0acfadbcce 26f91269ef
10 changed files with 152 additions and 39 deletions
--- a/models/silver/silver__actions_events_addkey_s3.sql
+++ b/models/silver/silver__actions_events_addkey_s3.sql
@ -13,8 +13,12 @@ WITH action_events AS (
  FROM
    {{ ref('silver__actions_events_s3') }}
  WHERE
-    action_name = 'AddKey'
-    AND {{ incremental_load_filter('_load_timestamp') }}
+    action_name = 'AddKey' 
+    {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+      AND {{ partition_load_manual('no_buffer') }}
+    {% else %}
+      AND {{ incremental_load_filter('_load_timestamp') }}
+    {% endif %}
 ),
 addkey_events AS (
  SELECT
--- a/models/silver/silver__actions_events_function_call_s3.sql
+++ b/models/silver/silver__actions_events_function_call_s3.sql
@ -13,8 +13,12 @@ WITH action_events AS (
  FROM
    {{ ref('silver__actions_events_s3') }}
  WHERE
-    {{ incremental_load_filter('_load_timestamp') }}
-    AND action_name = 'FunctionCall'
+    action_name = 'FunctionCall' 
+    {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+      AND {{ partition_load_manual('no_buffer') }}
+    {% else %}
+      AND {{ incremental_load_filter('_load_timestamp') }}
+    {% endif %}
 ),
 decoding AS (
  SELECT
--- a/models/silver/silver__actions_events_s3.sql
+++ b/models/silver/silver__actions_events_s3.sql
@ -25,8 +25,14 @@ WITH txs AS (
    _partition_by_block_number
  FROM
    {{ ref('silver__streamline_transactions_final') }}
-  WHERE
-    {{ incremental_load_filter('_load_timestamp') }}
+
+    {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+    WHERE
+      {{ partition_load_manual('no_buffer') }}
+    {% else %}
+    WHERE
+      {{ incremental_load_filter('_load_timestamp') }}
+    {% endif %}
 ),
 actions AS (
  SELECT
--- a/models/silver/silver__dex_swaps_s3.sql
+++ b/models/silver/silver__dex_swaps_s3.sql
@ -22,8 +22,12 @@ WITH base_swap_calls AS (
        method_name IN (
            'swap',
            'ft_transfer_call'
-        )
-        AND {{ incremental_load_filter('_load_timestamp') }}
+        ) 
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            AND {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            AND {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 base_swaps AS (
    SELECT
--- a/models/silver/silver__nft_mints_s3.sql
+++ b/models/silver/silver__nft_mints_s3.sql
@ -41,8 +41,11 @@ WITH function_call AS (
        method_name IN (
            'nft_mint',
            'nft_mint_batch'
-        )
-        AND {{ incremental_load_filter("_load_timestamp") }}
+        ) {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            AND {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            AND {{ incremental_load_filter("_load_timestamp") }}
+        {% endif %}
 ),
 --Data Pulled from Transaction
 mint_transactions AS (
@ -64,8 +67,11 @@ mint_transactions AS (
            FROM
                function_call
        )
-        AND tx_status = 'Success'
-        AND {{ incremental_load_filter("_load_timestamp") }}
+        AND tx_status = 'Success' {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            AND {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            AND {{ incremental_load_filter("_load_timestamp") }}
+        {% endif %}
 ),
 --Data pulled from Receipts Table
 receipts_data AS (
@ -84,8 +90,12 @@ receipts_data AS (
                DISTINCT tx_hash
            FROM
                function_call
-        )
-        AND {{ incremental_load_filter("_load_timestamp") }}
+        ) 
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            AND {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            AND {{ incremental_load_filter("_load_timestamp") }}
+        {% endif %}
 )
 SELECT
    DISTINCT action_id,
--- a/models/silver/silver__prices_oracle_s3.sql
+++ b/models/silver/silver__prices_oracle_s3.sql
@ -3,9 +3,7 @@
    unique_key = 'block_id',
    incremental_strategy = 'delete+insert',
    cluster_by = ['block_timestamp::DATE', '_load_timestamp::DATE'],
-  tags = ['curated', 's3_curated']
-
-
+    tags = ['curated', 's3_curated']
 ) }}

 WITH txs AS (
@ -14,8 +12,14 @@ WITH txs AS (
        *
    FROM
        {{ ref('silver__streamline_transactions_final') }}
-    WHERE
-        {{ incremental_load_filter('_load_timestamp') }}
+
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+        WHERE
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+        WHERE
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 token_labels AS (
    SELECT
--- a/models/silver/silver__staking_actions_s3.sql
+++ b/models/silver/silver__staking_actions_s3.sql
@ -15,21 +15,31 @@ WITH actions_events_function_call AS (
    FROM
        {{ ref('silver__actions_events_function_call_s3') }}
    WHERE
-        {{ incremental_load_filter('_load_timestamp') }}
-        AND method_name IN (
+        method_name IN (
            'deposit_and_stake',
            'stake',
            'unstake',
            'unstake_all'
-        )
+        ) 
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            AND {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            AND {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 base_txs AS (
    SELECT
        *
    FROM
        {{ ref('silver__streamline_transactions_final') }}
-    WHERE
-        {{ incremental_load_filter('_load_timestamp') }}
+
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+        WHERE
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+        WHERE
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 txs AS (
    SELECT
@ -62,7 +72,16 @@ deposit_and_stake_txs AS (
        block_timestamp,
        tx_receiver AS pool_address,
        tx_signer,
-        REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'staking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount,
+        REGEXP_SUBSTR(
+            ARRAY_TO_STRING(
+                tx :receipt [0] :outcome :logs,
+                ','
+            ),
+            'staking (\\d+)',
+            1,
+            1,
+            'e'
+        ) :: NUMBER AS stake_amount,
        'Stake' AS action,
        _load_timestamp
    FROM
@ -77,7 +96,16 @@ stake_txs AS (
        block_timestamp,
        tx_receiver AS pool_address,
        tx_signer,
-        REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'staking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount,
+        REGEXP_SUBSTR(
+            ARRAY_TO_STRING(
+                tx :receipt [0] :outcome :logs,
+                ','
+            ),
+            'staking (\\d+)',
+            1,
+            1,
+            'e'
+        ) :: NUMBER AS stake_amount,
        'Stake' AS action,
        _load_timestamp
    FROM
@ -92,7 +120,16 @@ stake_all_txs AS (
        block_timestamp,
        tx_receiver AS pool_address,
        tx_signer,
-        REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'staking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount,
+        REGEXP_SUBSTR(
+            ARRAY_TO_STRING(
+                tx :receipt [0] :outcome :logs,
+                ','
+            ),
+            'staking (\\d+)',
+            1,
+            1,
+            'e'
+        ) :: NUMBER AS stake_amount,
        'Stake' AS action,
        _load_timestamp
    FROM
@ -107,7 +144,16 @@ unstake_txs AS (
        block_timestamp,
        tx_receiver AS pool_address,
        tx_signer,
-        REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'unstaking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount,
+        REGEXP_SUBSTR(
+            ARRAY_TO_STRING(
+                tx :receipt [0] :outcome :logs,
+                ','
+            ),
+            'unstaking (\\d+)',
+            1,
+            1,
+            'e'
+        ) :: NUMBER AS stake_amount,
        'Unstake' AS action,
        _load_timestamp
    FROM
@ -122,7 +168,16 @@ unstake_all_txs AS (
        block_timestamp,
        tx_receiver AS pool_address,
        tx_signer,
-        REGEXP_SUBSTR(ARRAY_TO_STRING(tx :receipt [0] :outcome :logs, ','), 'unstaking (\\d+)', 1, 1, 'e') :: NUMBER AS stake_amount,
+        REGEXP_SUBSTR(
+            ARRAY_TO_STRING(
+                tx :receipt [0] :outcome :logs,
+                ','
+            ),
+            'unstaking (\\d+)',
+            1,
+            1,
+            'e'
+        ) :: NUMBER AS stake_amount,
        'Unstake' AS action,
        _load_timestamp
    FROM
--- a/models/silver/silver__staking_pools_s3.sql
+++ b/models/silver/silver__staking_pools_s3.sql
@ -17,8 +17,14 @@ WITH txs AS (
        _load_timestamp
    FROM
        {{ ref('silver__streamline_transactions_final') }}
-    WHERE
-        {{ incremental_load_filter('_load_timestamp') }}
+
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+        WHERE
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+        WHERE
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 function_calls AS (
    SELECT
@ -32,8 +38,12 @@ function_calls AS (
        method_name IN (
            'create_staking_pool',
            'update_reward_fee_fraction'
-        )
-        AND {{ incremental_load_filter('_load_timestamp') }}
+        ) 
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+            AND {{ partition_load_manual('no_buffer') }}
+        {% else %}
+            AND {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 pool_txs AS (
    SELECT
--- a/models/silver/silver__transfers_s3.sql
+++ b/models/silver/silver__transfers_s3.sql
@ -15,8 +15,12 @@ WITH action_events AS(
  FROM
    {{ ref('silver__actions_events_s3') }}
  WHERE
-    action_name = 'Transfer'
-    AND {{ incremental_load_filter("_load_timestamp") }}
+    action_name = 'Transfer' 
+    {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+      AND {{ partition_load_manual('no_buffer') }}
+    {% else %}
+      AND {{ incremental_load_filter("_load_timestamp") }}
+    {% endif %}
 ),
 txs AS (
  SELECT
@ -35,8 +39,14 @@ txs AS (
    _load_timestamp
  FROM
    {{ ref('silver__streamline_transactions_final') }}
-  WHERE
-    {{ incremental_load_filter("_load_timestamp") }}
+
+    {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+    WHERE
+      {{ partition_load_manual('no_buffer') }}
+    {% else %}
+    WHERE
+      {{ incremental_load_filter("_load_timestamp") }}
+    {% endif %}
 ),
 receipts AS (
  SELECT
--- a/models/silver/silver__usn_supply_s3.sql
+++ b/models/silver/silver__usn_supply_s3.sql
@ -12,8 +12,14 @@ WITH txs AS (
        *
    FROM
        {{ ref('silver__streamline_receipts_final') }}
-    WHERE
-        {{ incremental_load_filter('_load_timestamp') }}
+
+        {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %}
+        WHERE
+            {{ partition_load_manual('no_buffer') }}
+        {% else %}
+        WHERE
+            {{ incremental_load_filter('_load_timestamp') }}
+        {% endif %}
 ),
 logs AS (
    SELECT