From becaae2b2e7c82b6d7659ada448605ac668d4b77 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 11 Nov 2024 10:36:30 -0700 Subject: [PATCH 01/15] params --- dbt_project.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 6345f41..f176649 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -146,27 +146,27 @@ vars: # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True - BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 600000 - BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 120000 - BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 24000 + BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 300000 + BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 12000 # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 4 RECEIPTS_REALTIME_NEW_BUILD: True # RECEIPTS_REALTIME_TESTING_LIMIT: 3 RECEIPTS_HISTORY_NEW_BUILD: True - RECEIPTS_HISTORY_SQL_LIMIT: 600000 - RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 120000 - RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 24000 + RECEIPTS_HISTORY_SQL_LIMIT: 300000 + RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000 # RECEIPTS_HISTORY_TESTING_LIMIT: 4 TRACES_REALTIME_NEW_BUILD: True # TRACES_REALTIME_TESTING_LIMIT: 3 TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 300000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 37500 - TRACES_HISTORY_WORKER_BATCH_SIZE: 7500 + TRACES_HISTORY_SQL_LIMIT: 150000 + TRACES_HISTORY_PRODUCER_BATCH_SIZE: 18750 + TRACES_HISTORY_WORKER_BATCH_SIZE: 3750 # TRACES_HISTORY_TESTING_LIMIT: 4 # GOLD_FULL_REFRESH: True From a142009c9dcce138af5e95ad5651867636330d7b Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 11 Nov 2024 10:36:58 -0700 Subject: [PATCH 02/15] params --- dbt_project.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index f176649..b2fe449 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -164,9 +164,9 @@ vars: # TRACES_REALTIME_TESTING_LIMIT: 3 TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 150000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 18750 - TRACES_HISTORY_WORKER_BATCH_SIZE: 3750 + TRACES_HISTORY_SQL_LIMIT: 300000 + TRACES_HISTORY_PRODUCER_BATCH_SIZE: 37500 + TRACES_HISTORY_WORKER_BATCH_SIZE: 7500 # TRACES_HISTORY_TESTING_LIMIT: 4 # GOLD_FULL_REFRESH: True From 3a60a7302a5cb9f9ef97fe8c7ab0ad4018fab1b6 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 11 Nov 2024 14:42:11 -0700 Subject: [PATCH 03/15] new build --- dbt_project.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index b2fe449..e38ca85 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -142,22 +142,22 @@ vars: ## REQUIRED ## OPTIONAL - BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True + # BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 - BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True - BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 300000 - BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 60000 - BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 12000 + # BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True + # BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 300000 + # BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + # BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 12000 # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 4 - RECEIPTS_REALTIME_NEW_BUILD: True + # RECEIPTS_REALTIME_NEW_BUILD: True # RECEIPTS_REALTIME_TESTING_LIMIT: 3 - RECEIPTS_HISTORY_NEW_BUILD: True - RECEIPTS_HISTORY_SQL_LIMIT: 300000 - RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 60000 - RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000 + # RECEIPTS_HISTORY_NEW_BUILD: True + # RECEIPTS_HISTORY_SQL_LIMIT: 300000 + # RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + # RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000 # RECEIPTS_HISTORY_TESTING_LIMIT: 4 TRACES_REALTIME_NEW_BUILD: True From 5418da31ab15a4d498273004afc8fa499f8c01d6 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 11 Nov 2024 14:45:24 -0700 Subject: [PATCH 04/15] workflow --- .github/workflows/dbt_run_streamline_history.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index bf57ca8..c6b7297 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -42,4 +42,4 @@ jobs: - name: Run History Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history From 75e4ae0c6c864225042a75d176bfd779e9187a71 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 11 Nov 2024 15:14:06 -0700 Subject: [PATCH 05/15] add confirm blocks params --- dbt_project.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index e38ca85..a7d1e1c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -169,6 +169,15 @@ vars: TRACES_HISTORY_WORKER_BATCH_SIZE: 7500 # TRACES_HISTORY_TESTING_LIMIT: 4 + CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True + CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 + + CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True + CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 600000 + CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 120000 + CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 24000 + CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 4 + # GOLD_FULL_REFRESH: True # SILVER_FULL_REFRESH: True From 6f2b569222eac287acd0c044789383548b8c152b Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 11 Nov 2024 15:33:38 -0700 Subject: [PATCH 06/15] update workflows --- .github/workflows/dbt_run_streamline_chainhead.yml | 2 +- .github/workflows/dbt_run_streamline_history.yml | 2 +- dbt_project.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index ed7781e..a82bab9 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -42,7 +42,7 @@ jobs: - name: Run Chainhead Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_realtime_confirm_blocks" - name: Run Chainhead Tests run: | diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index c6b7297..5631ec2 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -42,4 +42,4 @@ jobs: - name: Run History Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_history_confirm_blocks" diff --git a/dbt_project.yml b/dbt_project.yml index a7d1e1c..56e875b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -170,13 +170,13 @@ vars: # TRACES_HISTORY_TESTING_LIMIT: 4 CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True - CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 + # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 600000 CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 120000 CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 24000 - CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 4 + # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 4 # GOLD_FULL_REFRESH: True # SILVER_FULL_REFRESH: True From 25cbcbe0b12c7e52e8f0ac61daed80b21dd3dfd5 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 12 Nov 2024 09:20:57 -0700 Subject: [PATCH 07/15] add workflow --- .github/workflows/dbt_run_scheduled_main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 5326481..6db3576 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -42,4 +42,4 @@ jobs: - name: Run Main Models run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" \ No newline at end of file + dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" "fsc_evm,tag:silver_confirm_blocks" \ No newline at end of file From e79120f8f9d2392e3ec91c6d9b7779aaab157146 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 12 Nov 2024 11:54:04 -0700 Subject: [PATCH 08/15] order by clause and sl params --- dbt_project.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 56e875b..d31061e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -164,18 +164,20 @@ vars: # TRACES_REALTIME_TESTING_LIMIT: 3 TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 300000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 37500 - TRACES_HISTORY_WORKER_BATCH_SIZE: 7500 + TRACES_HISTORY_SQL_LIMIT: 600000 + TRACES_HISTORY_PRODUCER_BATCH_SIZE: 75000 + TRACES_HISTORY_WORKER_BATCH_SIZE: 15000 + TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC, block_number DESC' # TRACES_HISTORY_TESTING_LIMIT: 4 CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True - CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 600000 - CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 120000 - CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 24000 + CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 900000 + CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 180000 + CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 36000 + CONFIRM_BLOCKS_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC, block_number DESC' # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 4 # GOLD_FULL_REFRESH: True From 226df84cf699dc7c585a4f6680229fa90015e3a4 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 12 Nov 2024 16:33:56 -0500 Subject: [PATCH 09/15] use backfill and alter params --- .github/workflows/dbt_run_scheduled_main.yml | 2 +- dbt_project.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 6db3576..8dcf6e5 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -15,7 +15,7 @@ env: PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ vars.REGION }}" DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" + WAREHOUSE: BACKFILL SCHEMA: "${{ vars.SCHEMA }}" concurrency: diff --git a/dbt_project.yml b/dbt_project.yml index d31061e..e9baa96 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -164,20 +164,20 @@ vars: # TRACES_REALTIME_TESTING_LIMIT: 3 TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 600000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 75000 - TRACES_HISTORY_WORKER_BATCH_SIZE: 15000 - TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC, block_number DESC' + TRACES_HISTORY_SQL_LIMIT: 1000000 + TRACES_HISTORY_PRODUCER_BATCH_SIZE: 20000 + TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 + TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' # TRACES_HISTORY_TESTING_LIMIT: 4 CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True - CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 900000 - CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 180000 + CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 1000000 + CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 360000 CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 36000 - CONFIRM_BLOCKS_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC, block_number DESC' + CONFIRM_BLOCKS_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 4 # GOLD_FULL_REFRESH: True From ab6344d68252af5e6e47c398df9dffac37ac5fc4 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 12 Nov 2024 16:40:01 -0500 Subject: [PATCH 10/15] remove confirm blocks --- .github/workflows/dbt_run_streamline_history.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 5631ec2..d4b4d02 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -42,4 +42,4 @@ jobs: - name: Run History Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_history_confirm_blocks" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history "fsc_evm,tag:streamline_core_complete_confirm_blocks" From 68ffc3f8a5e89e420ff277a53737f5f3c2486d60 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 12 Nov 2024 21:16:42 -0500 Subject: [PATCH 11/15] 1.2m --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index e9baa96..e543ffd 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -164,7 +164,7 @@ vars: # TRACES_REALTIME_TESTING_LIMIT: 3 TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 1000000 + TRACES_HISTORY_SQL_LIMIT: 1200000 TRACES_HISTORY_PRODUCER_BATCH_SIZE: 20000 TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' From 7c80b58f5ea26df1c01570b5de150c407a73e458 Mon Sep 17 00:00:00 2001 From: Austin Date: Tue, 12 Nov 2024 22:26:58 -0500 Subject: [PATCH 12/15] limit --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index e543ffd..e79644e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -164,7 +164,7 @@ vars: # TRACES_REALTIME_TESTING_LIMIT: 3 TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 1200000 + TRACES_HISTORY_SQL_LIMIT: 1500000 TRACES_HISTORY_PRODUCER_BATCH_SIZE: 20000 TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' From c874ed7e3964cf000aeba083bb10d1507d2f5ab2 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 13 Nov 2024 08:00:37 -0500 Subject: [PATCH 13/15] 30 mins --- data/github_actions__workflows.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 2940a92..4ba8342 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,4 +1,4 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"24,54 * * * *" dbt_run_scheduled_main,"2,32 * * * *" -dbt_run_streamline_history,"12 * * * *" +dbt_run_streamline_history,"12,42 * * * *" From 6feb7cb89a2c99dcbe4c8e873ac0e83f7e97642e Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 13 Nov 2024 11:00:33 -0500 Subject: [PATCH 14/15] updates --- .github/workflows/dbt_run_streamline_history.yml | 2 +- dbt_project.yml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index d4b4d02..5631ec2 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -42,4 +42,4 @@ jobs: - name: Run History Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history "fsc_evm,tag:streamline_core_complete_confirm_blocks" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_history_confirm_blocks" diff --git a/dbt_project.yml b/dbt_project.yml index e79644e..2ccbd00 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -160,14 +160,14 @@ vars: # RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000 # RECEIPTS_HISTORY_TESTING_LIMIT: 4 - TRACES_REALTIME_NEW_BUILD: True + # TRACES_REALTIME_NEW_BUILD: True # TRACES_REALTIME_TESTING_LIMIT: 3 - TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 1500000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 20000 - TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 - TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' + # TRACES_HISTORY_NEW_BUILD: True + # TRACES_HISTORY_SQL_LIMIT: 1500000 + # TRACES_HISTORY_PRODUCER_BATCH_SIZE: 20000 + # TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 + # TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' # TRACES_HISTORY_TESTING_LIMIT: 4 CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True From 6abb715097b30b04ab6367adc8e496a4eedefee7 Mon Sep 17 00:00:00 2001 From: Austin Date: Wed, 13 Nov 2024 11:55:40 -0500 Subject: [PATCH 15/15] hammer --- data/github_actions__workflows.csv | 2 +- dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 4ba8342..44aed71 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,4 +1,4 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"24,54 * * * *" dbt_run_scheduled_main,"2,32 * * * *" -dbt_run_streamline_history,"12,42 * * * *" +dbt_run_streamline_history,"*/20 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 2ccbd00..e906071 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -174,7 +174,7 @@ vars: # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True - CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 1000000 + CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 1500000 CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 360000 CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 36000 CONFIRM_BLOCKS_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC'