From 8e89c670badddedff3eeec1cc642d88d651cf5ad Mon Sep 17 00:00:00 2001 From: Jack Forgash Date: Tue, 27 Jun 2023 12:55:22 -0600 Subject: [PATCH] upd readme and change to vars --- README.md | 79 +++++++++++++------ dbt_project.yml | 5 +- .../silver__actions_events_addkey_s3.sql | 2 +- ...ilver__actions_events_function_call_s3.sql | 2 +- .../actions/silver__actions_events_s3.sql | 2 +- .../silver/curated/silver__dex_swaps_s3.sql | 2 +- .../silver/curated/silver__lockup_actions.sql | 6 +- models/silver/curated/silver__logs_s3.sql | 2 +- .../silver/curated/silver__nft_mints_s3.sql | 8 +- .../silver/curated/silver__pool_balances.sql | 2 +- models/silver/curated/silver__pool_events.sql | 2 +- .../curated/silver__prices_oracle_s3.sql | 2 +- .../curated/silver__staking_actions_s3.sql | 4 +- .../curated/silver__staking_actions_v2.sql | 2 +- .../curated/silver__staking_pools_s3.sql | 5 +- .../curated/silver__standard_nft_mint_s3.sql | 6 +- .../silver/curated/silver__transfers_s3.sql | 4 +- .../silver/curated/silver__usn_supply_s3.sql | 2 +- .../silver_horizon__decoded_actions.sql | 4 +- .../horizon/silver_horizon__receipts.sql | 2 +- .../silver/social/silver_social__addkey.sql | 6 +- .../social/silver_social__decoded_actions.sql | 4 +- models/silver/social/silver_social__posts.sql | 2 +- .../social/silver_social__profile_changes.sql | 2 +- .../silver/social/silver_social__receipts.sql | 2 +- .../silver/social/silver_social__widgets.sql | 2 +- .../helpers/silver__flatten_receipts.sql | 2 +- .../silver__receipt_tx_hash_mapping.sql | 2 +- .../silver/streamline/silver__load_blocks.sql | 2 +- .../silver/streamline/silver__load_shards.sql | 2 +- .../silver__streamline_receipts_final.sql | 2 +- .../silver__streamline_transactions_final.sql | 4 +- 32 files changed, 106 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index f1970b5..08ef925 100644 --- a/README.md +++ b/README.md @@ -85,17 +85,17 @@ When creating a PR please include the following details in the PR description: ### Manual Batch Refresh -If data needs to be re-run for some reason, partitions of data can be re-reun through the models by utilizing the column `_partition_by_block_number` and the dbt profile name. +If data needs to be re-run for some reason, partitions of data can be re-reun through the models by utilizing the column `_partition_by_block_number` and passing environment variables. -Any data refresh will need to be done in a batch due to the nature of the receipt x tx hash mapping. The view `silver__receipt_tx_hash_mapping` is a recursive AncestryTree that follows linked receipt outcome ids to map all receipts generated in a transaction back to the primary hash. Receipts can be generated many blocks after the transaction occurs, so a generous buffer is required to ensure all receipts are captured. +Any data refresh will need to be done in a batch due to the nature of the receipt <> tx hash mapping. The view `silver__receipt_tx_hash_mapping` is a recursive AncestryTree that follows linked receipt outcome ids to map all receipts generated in a transaction back to the primary hash. Receipts can be generated many blocks after the transaction occurs, so a generous buffer is required to ensure all receipts are captured. + +The fix makes use of [project variables](https://docs.getdbt.com/docs/build/project-variables#defining-variables-on-the-command-line) to pass the following parameters: + - manual_fix (required): This will run the models with the specified range, rather than the standard incremental logic. `False` by default. + - range_start (required): The start of the block partition range (nearest 10,000) to run. + - range_end (required): The end of the block partition range (nearest 10,000) to run. + - front_buffer (optional): The number of partitions to add to the front of the range. 1 by default, not likely to need changing. + - end_buffer (optional): The number of partitions to add to the end of the range. 1 by default, not likely to need changing. -Models in the `streamline` folder can be run with standard incremental logic up until the 2 final receipt and transaction tables (tagged as such, see below). The next step, mapping receipts to tx hash over a range, can be run with the following command: - -``` -dbt run -s tag:load_shards tag:receipt_map --vars '{"range_start": X, "range_end": Y, "front_buffer": 1, "end_buffer": 1}' -t manual_fix_dev -``` - -The target name will determine how the model operates, calling a macro `partition_load_manual()` which takes the variables input in the command to set the range. `front_buffer` and `end_buffer` are set to 1 by default and indicate how many partitions (10k blocks) should be added on to the front or end of the range. - Flatten receipts is set to look back 1 partition to grab receipts that may have occurred prior to the start of the range. @@ -103,35 +103,68 @@ The target name will determine how the model operates, calling a macro `partitio - Receipts final does not buffer the range to only map receipts that occurred within the range to a tx hash (but the lookback is necessary in case the tree is truncated by the partition break). - Transactions final does not add a buffer when grabbing transactions from the int txs model, but does add an end buffer when selecting from receipts final to include mapped receipts that may have occurred after the end of the range. - A range is necessary for the mapping view as it consumes a significant amount of memory and will otherwise run out. - - Actions and curated models include the conditional based on target name so the tags `s3_actions` and `s3_curated` can be included to re-run the fixed data in downstream silver models. - - if missing data is loaded in new, this is not necessary as `_load_timestamp` will be set to when the data hits snowflake and will flow through the standard incremental logic in the curated models. +``` +dbt run -s [tags] --vars [variables] +``` #### dbt Model Tags To help with targeted refreshes, a number of tags have been applied to the models. These are defined below: -| Tag | Description | -| --- | --- | -| load | Runs models that load data into Snowflake from S3. The 2 `load_X` models are staging tables for data, which is then parsed and transformed up until the final txs/receipts models. | -| load_shards | Just the `load` models that touch shards | -| load_blocks | Just the `load` models that touch blocks | -| receipt_map | Runs the receipt-mapping models that must use a partition. This set of models cannot simply run with incremental logic due to the recursive tree used to map receipt IDs to Tx Hashes. | -| actions | Just the 3 action events models, an important set of intermediary models before curated activity. Note: These are also tagged with `s3_curated`. | -| curated | Models that are used to generate the curated tables | -| core | All public views are tagged with core, regardless of schema. At this time, that includes `core` and `social`. | +| Tag | Description | View Models | +| --- | --- | --- | +| load | Runs models that load data into Snowflake from S3. The 2 `load_X` models are staging tables for data, which is then parsed and transformed up until the final txs/receipts models. | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:load) | +| load_shards | Just the `load` models that touch shards | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:load_shards) | +| load_blocks | Just the `load` models that touch blocks | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:load_blocks) | +| receipt_map | Runs the receipt-mapping models that must use a partition. This set of models cannot simply run with incremental logic due to the recursive tree used to map receipt IDs to Tx Hashes. | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:receipt_map) | +| actions | Just the 3 action events models, an important set of intermediary models before curated activity. Note: These are also tagged with `s3_curated`. | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:actions) | +| curated | Models that are used to generate the curated tables | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:curated) | +| core | All public views are tagged with core, regardless of schema. At this time, that includes `core` and `social`. | [link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:core) | Note: there are other tags that are currently not used. All legacy models are tagged with something that includes `rpc`, but all are presently disabled to avoid an accidental run. You can visualize these tags by using the DAG Explorer in the [docs](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:load). + +#### Load Missing Blocks or Shards +[Blocks](models/silver/streamline/silver__load_blocks.sql) and [shards](models/silver/streamline/silver__load_shards.sql) can be re-loaded using the load tag(s). + - `load` will run both blocks and shards models, landing data in `silver__streamline_blocks`, `silver__streamline_receipts`, `silver__streamline_receipt_chunks`, and `silver__streamline_transactions`. + - `load_blocks` will run just the blocks models, landing data in `silver__streamline_blocks`. + - `load_shards` will run just the shards models, landing data in `silver__streamline_receipts`, `silver__streamline_receipt_chunks`, and `silver__streamline_transactions`. + +The logic in the load_x models will only check the external table for blocks and shards known to be missing. It will query the sequence gap test table(s). An accurate partition range is required. + +``` +dbt run -s tag:load --vars '{"manual_fix": True, "range_start": X, "range_end": Y}' +``` + +#### Map Tx Hash <> Receipt Hash +The middle step is to map receipt IDs to transaction hashes. This is done in 3 models, which are tagged with `receipt_map`. 2 of these models are helper views that recursively map out the receipt->parent receipt->...->transaction, thus linking all receipts to a transaction. This step is computationally intensive, and requires a tight partition range. For present blocks with more activity, <250k is recommended. + +If the range being mapped is the same range as the block/shard re-walk, then the tag can simply be appended to the same job. +``` +dbt run -s tag:load tag:receipt_map --vars '{"manual_fix": True, "range_start": X, "range_end": Y}' +``` + +The end result of this run will be `streamline__receipts_final` and `streamline__transactions_final` ([link](https://flipsidecrypto.github.io/near-models/#!/overview?g_v=1&g_i=tag:load%20tag:receipt_map)). + +#### Update curated models + Actions and curated models include the conditional based on target name so the tags `actions` and `curated` can be included to re-run the fixed data in downstream silver models. If missing data is loaded in new, this should not be necessary as `_load_timestamp` will be set to when the data hits snowflake and will flow through the standard incremental logic in the curated models. However, the range can be run with the curated tag: + + ``` +dbt run -s tag:curated --vars '{"manual_fix": True, "range_start": X, "range_end": Y}' +``` +Or +``` +dbt run -s tag:load tag:receipt_map tag:curated --vars '{"manual_fix": True, "range_start": X, "range_end": Y}' +``` + ### Incremental Load Strategy Because data must be run in batches to properly map receipts to a transaction hash, a conditional is added to curated models using jinja. This should be present on everything after the mapping process. Most data will have no issue running with a standard incremental load. This filter is required for the above commands in the **Manual Batch Refresh** section. Include the following conditional, as targeted runs of block partitions may be required: ``` - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/dbt_project.yml b/dbt_project.yml index 039e788..f6ec710 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -50,7 +50,7 @@ tests: near: silver: +severity: warn # default to warn for int tests - +error_if: ">25" + +error_if: ">25" # TODO - threshold not working gold: +severity: error # default to warn for int tests +store_failures: true # all tests @@ -60,4 +60,5 @@ vars: STREAMLINE_INVOKE_STREAMS: False STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False - UPDATE_SNOWFLAKE_TAGS: True \ No newline at end of file + UPDATE_SNOWFLAKE_TAGS: True + MANUAL_FIX: False diff --git a/models/silver/actions/silver__actions_events_addkey_s3.sql b/models/silver/actions/silver__actions_events_addkey_s3.sql index a399332..3784000 100644 --- a/models/silver/actions/silver__actions_events_addkey_s3.sql +++ b/models/silver/actions/silver__actions_events_addkey_s3.sql @@ -14,7 +14,7 @@ WITH action_events AS ( {{ ref('silver__actions_events_s3') }} WHERE action_name = 'AddKey' - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/actions/silver__actions_events_function_call_s3.sql b/models/silver/actions/silver__actions_events_function_call_s3.sql index 1d091e6..441923f 100644 --- a/models/silver/actions/silver__actions_events_function_call_s3.sql +++ b/models/silver/actions/silver__actions_events_function_call_s3.sql @@ -15,7 +15,7 @@ WITH action_events AS ( WHERE action_name = 'FunctionCall' - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/actions/silver__actions_events_s3.sql b/models/silver/actions/silver__actions_events_s3.sql index 02584c3..9d4915c 100644 --- a/models/silver/actions/silver__actions_events_s3.sql +++ b/models/silver/actions/silver__actions_events_s3.sql @@ -13,7 +13,7 @@ WITH receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/silver__dex_swaps_s3.sql b/models/silver/curated/silver__dex_swaps_s3.sql index 3ab9ade..c574eb9 100644 --- a/models/silver/curated/silver__dex_swaps_s3.sql +++ b/models/silver/curated/silver__dex_swaps_s3.sql @@ -23,7 +23,7 @@ WITH base_swap_calls AS ( 'swap', 'ft_transfer_call' ) - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/curated/silver__lockup_actions.sql b/models/silver/curated/silver__lockup_actions.sql index 30ac355..37b4651 100644 --- a/models/silver/curated/silver__lockup_actions.sql +++ b/models/silver/curated/silver__lockup_actions.sql @@ -11,7 +11,7 @@ WITH txs AS ( FROM {{ ref('silver__streamline_transactions_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} @@ -25,7 +25,7 @@ function_calls AS ( FROM {{ ref('silver__actions_events_function_call_s3') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} @@ -39,7 +39,7 @@ xfers AS ( FROM {{ ref('silver__transfers_s3') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/silver__logs_s3.sql b/models/silver/curated/silver__logs_s3.sql index 08f57fa..0c5807c 100644 --- a/models/silver/curated/silver__logs_s3.sql +++ b/models/silver/curated/silver__logs_s3.sql @@ -13,7 +13,7 @@ WITH receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/silver__nft_mints_s3.sql b/models/silver/curated/silver__nft_mints_s3.sql index 5b8cbc1..d0313a8 100644 --- a/models/silver/curated/silver__nft_mints_s3.sql +++ b/models/silver/curated/silver__nft_mints_s3.sql @@ -41,7 +41,8 @@ WITH function_call AS ( method_name IN ( 'nft_mint', 'nft_mint_batch' - ) {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + ) + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter("_load_timestamp") }} @@ -67,7 +68,8 @@ mint_transactions AS ( FROM function_call ) - AND tx_status = 'Success' {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + AND tx_status = 'Success' + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter("_load_timestamp") }} @@ -91,7 +93,7 @@ receipts_data AS ( FROM function_call ) - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter("_load_timestamp") }} diff --git a/models/silver/curated/silver__pool_balances.sql b/models/silver/curated/silver__pool_balances.sql index 3e3c472..d059a7e 100644 --- a/models/silver/curated/silver__pool_balances.sql +++ b/models/silver/curated/silver__pool_balances.sql @@ -13,7 +13,7 @@ WITH pool_events AS ( FROM {{ ref('silver__pool_events') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/curated/silver__pool_events.sql b/models/silver/curated/silver__pool_events.sql index 874b88e..2d1b36d 100644 --- a/models/silver/curated/silver__pool_events.sql +++ b/models/silver/curated/silver__pool_events.sql @@ -13,7 +13,7 @@ WITH receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/curated/silver__prices_oracle_s3.sql b/models/silver/curated/silver__prices_oracle_s3.sql index 7c4497a..6fd90b2 100644 --- a/models/silver/curated/silver__prices_oracle_s3.sql +++ b/models/silver/curated/silver__prices_oracle_s3.sql @@ -19,7 +19,7 @@ events_function_call AS ( FROM {{ ref('silver__actions_events_function_call_s3') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/silver__staking_actions_s3.sql b/models/silver/curated/silver__staking_actions_s3.sql index 4c89068..b3719fe 100644 --- a/models/silver/curated/silver__staking_actions_s3.sql +++ b/models/silver/curated/silver__staking_actions_s3.sql @@ -21,7 +21,7 @@ WITH actions_events_function_call AS ( 'unstake', 'unstake_all' ) - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter('_load_timestamp') }} @@ -33,7 +33,7 @@ base_txs AS ( FROM {{ ref('silver__streamline_transactions_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/silver__staking_actions_v2.sql b/models/silver/curated/silver__staking_actions_v2.sql index ee97d04..69b6e5e 100644 --- a/models/silver/curated/silver__staking_actions_v2.sql +++ b/models/silver/curated/silver__staking_actions_v2.sql @@ -13,7 +13,7 @@ WITH pool_events AS ( FROM {{ ref('silver__pool_events') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/curated/silver__staking_pools_s3.sql b/models/silver/curated/silver__staking_pools_s3.sql index 87d0f0e..ba2aca5 100644 --- a/models/silver/curated/silver__staking_pools_s3.sql +++ b/models/silver/curated/silver__staking_pools_s3.sql @@ -20,7 +20,7 @@ WITH txs AS ( FROM {{ ref('silver__streamline_transactions_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} @@ -49,7 +49,8 @@ function_calls AS ( 'create_staking_pool', 'update_reward_fee_fraction', 'new' - ) {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + ) + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/curated/silver__standard_nft_mint_s3.sql b/models/silver/curated/silver__standard_nft_mint_s3.sql index ee5de1b..9c02fa4 100644 --- a/models/silver/curated/silver__standard_nft_mint_s3.sql +++ b/models/silver/curated/silver__standard_nft_mint_s3.sql @@ -13,7 +13,7 @@ WITH logs AS ( FROM {{ ref('silver__logs_s3') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} @@ -25,7 +25,7 @@ tx AS ( FROM {{ ref('silver__streamline_transactions_final') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} @@ -41,7 +41,7 @@ function_call AS ( FROM {{ ref("silver__actions_events_function_call_s3") }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/curated/silver__transfers_s3.sql b/models/silver/curated/silver__transfers_s3.sql index 8610caf..77049bd 100644 --- a/models/silver/curated/silver__transfers_s3.sql +++ b/models/silver/curated/silver__transfers_s3.sql @@ -18,7 +18,7 @@ WITH action_events AS( {{ ref('silver__actions_events_s3') }} WHERE action_name = 'Transfer' - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} AND {{ incremental_load_filter("_load_timestamp") }} @@ -43,7 +43,7 @@ txs AS ( FROM {{ ref('silver__streamline_transactions_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/silver__usn_supply_s3.sql b/models/silver/curated/silver__usn_supply_s3.sql index c516791..e0f1c1f 100644 --- a/models/silver/curated/silver__usn_supply_s3.sql +++ b/models/silver/curated/silver__usn_supply_s3.sql @@ -13,7 +13,7 @@ WITH txs AS ( FROM {{ ref('silver__streamline_receipts_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/horizon/silver_horizon__decoded_actions.sql b/models/silver/horizon/silver_horizon__decoded_actions.sql index ad8f0b1..a86516d 100644 --- a/models/silver/horizon/silver_horizon__decoded_actions.sql +++ b/models/silver/horizon/silver_horizon__decoded_actions.sql @@ -12,7 +12,7 @@ WITH all_horizon_receipts AS ( FROM {{ ref('silver_horizon__receipts') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} @@ -28,7 +28,7 @@ decoded_function_calls AS ( FROM {{ ref('silver__actions_events_function_call_s3') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/horizon/silver_horizon__receipts.sql b/models/silver/horizon/silver_horizon__receipts.sql index 5e2fbd0..9dc0a1b 100644 --- a/models/silver/horizon/silver_horizon__receipts.sql +++ b/models/silver/horizon/silver_horizon__receipts.sql @@ -12,7 +12,7 @@ WITH all_horizon_receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/social/silver_social__addkey.sql b/models/silver/social/silver_social__addkey.sql index 074bc05..424fa7c 100644 --- a/models/silver/social/silver_social__addkey.sql +++ b/models/silver/social/silver_social__addkey.sql @@ -12,7 +12,7 @@ WITH receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} @@ -34,7 +34,7 @@ from_addkey_event AS ( FROM {{ ref('silver__actions_events_addkey_s3') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} @@ -58,7 +58,7 @@ nested_in_functioncall AS ( FROM {{ ref('silver__actions_events_function_call_s3') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/social/silver_social__decoded_actions.sql b/models/silver/social/silver_social__decoded_actions.sql index 706aa24..3c122b9 100644 --- a/models/silver/social/silver_social__decoded_actions.sql +++ b/models/silver/social/silver_social__decoded_actions.sql @@ -12,7 +12,7 @@ WITH all_social_receipts AS ( FROM {{ ref('silver_social__receipts') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} @@ -31,7 +31,7 @@ decoded_function_calls AS ( FROM {{ ref('silver__actions_events_function_call_s3') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/social/silver_social__posts.sql b/models/silver/social/silver_social__posts.sql index a6b95e7..369a821 100644 --- a/models/silver/social/silver_social__posts.sql +++ b/models/silver/social/silver_social__posts.sql @@ -12,7 +12,7 @@ WITH decoded_actions AS ( FROM {{ ref('silver_social__decoded_actions') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/social/silver_social__profile_changes.sql b/models/silver/social/silver_social__profile_changes.sql index 615c7b0..d43d4ce 100644 --- a/models/silver/social/silver_social__profile_changes.sql +++ b/models/silver/social/silver_social__profile_changes.sql @@ -12,7 +12,7 @@ WITH decoded_actions AS ( FROM {{ ref('silver_social__decoded_actions') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/social/silver_social__receipts.sql b/models/silver/social/silver_social__receipts.sql index 7971c2b..7c2b0f1 100644 --- a/models/silver/social/silver_social__receipts.sql +++ b/models/silver/social/silver_social__receipts.sql @@ -12,7 +12,7 @@ WITH all_social_receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/social/silver_social__widgets.sql b/models/silver/social/silver_social__widgets.sql index d0de272..2deeace 100644 --- a/models/silver/social/silver_social__widgets.sql +++ b/models/silver/social/silver_social__widgets.sql @@ -12,7 +12,7 @@ WITH decoded_actions AS ( FROM {{ ref('silver_social__decoded_actions') }} WHERE - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} {{ partition_load_manual('no_buffer') }} {% else %} {{ incremental_load_filter('_load_timestamp') }} diff --git a/models/silver/streamline/helpers/silver__flatten_receipts.sql b/models/silver/streamline/helpers/silver__flatten_receipts.sql index 0a26dad..6bce36c 100644 --- a/models/silver/streamline/helpers/silver__flatten_receipts.sql +++ b/models/silver/streamline/helpers/silver__flatten_receipts.sql @@ -18,7 +18,7 @@ WITH receipts AS ( outer => TRUE ) b - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('front') }} {% else %} diff --git a/models/silver/streamline/helpers/silver__receipt_tx_hash_mapping.sql b/models/silver/streamline/helpers/silver__receipt_tx_hash_mapping.sql index e677edc..208a584 100644 --- a/models/silver/streamline/helpers/silver__receipt_tx_hash_mapping.sql +++ b/models/silver/streamline/helpers/silver__receipt_tx_hash_mapping.sql @@ -29,7 +29,7 @@ txs AS ( FROM {{ ref('silver__streamline_transactions') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('front') }} {% else %} diff --git a/models/silver/streamline/silver__load_blocks.sql b/models/silver/streamline/silver__load_blocks.sql index 9cf1452..9e90c40 100644 --- a/models/silver/streamline/silver__load_blocks.sql +++ b/models/silver/streamline/silver__load_blocks.sql @@ -18,7 +18,7 @@ WITH blocks_json AS ( FROM {{ ref('bronze__streamline_blocks') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} AND block_id IN ( diff --git a/models/silver/streamline/silver__load_shards.sql b/models/silver/streamline/silver__load_shards.sql index 7dabe93..ba0b48f 100644 --- a/models/silver/streamline/silver__load_shards.sql +++ b/models/silver/streamline/silver__load_shards.sql @@ -24,7 +24,7 @@ WITH shards_json AS ( FROM {{ ref('bronze__streamline_shards') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} AND block_id IN ( diff --git a/models/silver/streamline/silver__streamline_receipts_final.sql b/models/silver/streamline/silver__streamline_receipts_final.sql index 460d343..86c3201 100644 --- a/models/silver/streamline/silver__streamline_receipts_final.sql +++ b/models/silver/streamline/silver__streamline_receipts_final.sql @@ -13,7 +13,7 @@ WITH base_receipts AS ( FROM {{ ref('silver__streamline_receipts') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/streamline/silver__streamline_transactions_final.sql b/models/silver/streamline/silver__streamline_transactions_final.sql index ee4c0e2..0037353 100644 --- a/models/silver/streamline/silver__streamline_transactions_final.sql +++ b/models/silver/streamline/silver__streamline_transactions_final.sql @@ -13,7 +13,7 @@ WITH int_txs AS ( FROM {{ ref('silver__streamline_transactions') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} {% else %} @@ -31,7 +31,7 @@ int_receipts AS ( FROM {{ ref('silver__streamline_receipts_final') }} - {% if target.name == 'manual_fix' or target.name == 'manual_fix_dev' %} + {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('end') }} {% else %}