diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 619db88..44718b2 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -15,7 +15,7 @@ env: PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ vars.REGION }}" DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" + WAREHOUSE: BACKFILL SCHEMA: "${{ vars.SCHEMA }}" concurrency: @@ -46,4 +46,4 @@ jobs: - name: Send new logs for decoding run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index ed7781e..a82bab9 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -42,7 +42,7 @@ jobs: - name: Run Chainhead Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_realtime_confirm_blocks" - name: Run Chainhead Tests run: | diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index bf57ca8..5631ec2 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -42,4 +42,4 @@ jobs: - name: Run History Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" main_package.core.streamline.history.streamline__traces_history "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_history_confirm_blocks" diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 089cbfc..8edc3cf 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,7 +1,6 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"24,54 * * * *" dbt_run_scheduled_main,"2,32 * * * *" -dbt_run_streamline_history,"12 * * * *" dbt_run_scheduled_decoded_logs,"9,39 * * * *" dbt_test_intraday,"40 */4 * * *" dbt_test_daily,"23 8 * * *" @@ -9,3 +8,4 @@ dbt_test_monthly,"6 7 28 * *" dbt_run_scheduled_abis,"12 */12 * * *" dbt_run_streamline_decoded_logs_history,"6 22 * * 6" dbt_run_scheduled_decoded_logs_history_user_abis,"39 23 * * *" +dbt_run_streamline_history,"*/20 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 6012ea2..00d6922 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -148,33 +148,44 @@ vars: ## REQUIRED ## OPTIONAL - BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True + # BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 - BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True - BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 600000 - BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 120000 - BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 24000 + # BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True + # BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 300000 + # BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + # BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 12000 # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 4 - RECEIPTS_REALTIME_NEW_BUILD: True + # RECEIPTS_REALTIME_NEW_BUILD: True # RECEIPTS_REALTIME_TESTING_LIMIT: 3 - RECEIPTS_HISTORY_NEW_BUILD: True - RECEIPTS_HISTORY_SQL_LIMIT: 600000 - RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 120000 - RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 24000 + # RECEIPTS_HISTORY_NEW_BUILD: True + # RECEIPTS_HISTORY_SQL_LIMIT: 300000 + # RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 60000 + # RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 12000 # RECEIPTS_HISTORY_TESTING_LIMIT: 4 - TRACES_REALTIME_NEW_BUILD: True + # TRACES_REALTIME_NEW_BUILD: True # TRACES_REALTIME_TESTING_LIMIT: 3 - TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_SQL_LIMIT: 300000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 37500 - TRACES_HISTORY_WORKER_BATCH_SIZE: 7500 + # TRACES_HISTORY_NEW_BUILD: True + # TRACES_HISTORY_SQL_LIMIT: 1500000 + # TRACES_HISTORY_PRODUCER_BATCH_SIZE: 20000 + # TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 + # TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' # TRACES_HISTORY_TESTING_LIMIT: 4 + CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True + # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 + + CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True + CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 1500000 + CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 360000 + CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 36000 + CONFIRM_BLOCKS_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC' + # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 4 + # GOLD_FULL_REFRESH: True # SILVER_FULL_REFRESH: True