diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 08c4864..30a3481 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -22,6 +22,7 @@ on: - DBT - DBT_CLOUD - DBT_EMERGENCY + - DBT_MEGA default: DBT dbt_command: type: string diff --git a/.github/workflows/dbt_run_overflowed_traces.yml b/.github/workflows/dbt_run_overflowed_traces.yml new file mode 100644 index 0000000..5510953 --- /dev/null +++ b/.github/workflows/dbt_run_overflowed_traces.yml @@ -0,0 +1,19 @@ +name: dbt_run_overflowed_traces +run-name: dbt_run_overflowed_traces + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Overflow Models + command: | + dbt run -m "fsc_evm,tag:overflow,tag:traces" --vars '{"MAIN_CORE_TRACES_OVERFLOW_ENABLED":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml index 19f9e44..2eb8581 100644 --- a/.github/workflows/dbt_run_scheduled_abis.yml +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -16,4 +16,7 @@ jobs: command_name: Run ABI Models command: | dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:gold,tag:abis" + command_name_2: Kick off decoded logs history, if there are new user submitted ABIs + command_2: | + dbt run-operation fsc_evm.run_decoded_logs_history secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 933c23a..71b7487 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -18,5 +18,5 @@ jobs: dbt run -m "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" command_name_2: Run Streamline Models command_2: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index ed56c1b..eaa9635 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -17,7 +17,7 @@ jobs: command_name: Build Daily Testing Views command: | dbt run -m "fsc_evm,tag:daily_test" - command_name_2: Run Daily Tests + command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others) command_2: | - dbt test -m "fsc_evm,tag:daily_test" + dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index dba54f3..ad7568f 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -17,7 +17,8 @@ jobs: command_name: Build Recent Testing Views command: | dbt run -m "fsc_evm,tag:recent_test" - command_name_2: Run Recent Tests + command_name_2: Run Observability & Recent Tests command_2: | + dbt run -m "fsc_evm,tag:observability" dbt test -m "fsc_evm,tag:recent_test" secrets: inherit \ No newline at end of file diff --git a/README.md b/README.md index 18f8134..129bc6f 100644 --- a/README.md +++ b/README.md @@ -83,9 +83,9 @@ The following variables can be used to control various aspects of the dbt run. U dbt run --vars '{"UPDATE_SNOWFLAKE_TAGS":true}' -s models/silver/utilities/silver__number_sequence.sql ``` -5. Start GHA tasks: +5. Start all GHA tasks: ``` - dbt seed -s github_actions__workflows && dbt run -m models/github_actions --full-refresh && dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' + dbt seed -s github_actions__workflows && dbt run -m models/github_actions --full-refresh && dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":True}' ``` 6. Using two or more variables: diff --git a/makefile b/makefile index 1fa5ac4..69c29ae 100644 --- a/makefile +++ b/makefile @@ -4,7 +4,7 @@ cleanup_time: rm -f package-lock.yml && dbt clean && dbt deps deploy_github_actions: - dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) + dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) @@ -13,23 +13,29 @@ deploy_new_github_action: dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) +deploy_livequery: + dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + deploy_chain_phase_1: - dbt run -m livequery_models.deploy.core --vars '{UPDATE_UDFS_AND_SPS: true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.livequery_grants -t $(DBT_TARGET) - dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{UPDATE_UDFS_AND_SPS: true}' -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET) + dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET) # kick chainhead workflow # wait ~10 minutes # run deploy_chain_phase_2 deploy_chain_phase_2: - dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) make deploy_github_actions -t $(DBT_TARGET) # tasks set to SUSPEND by default # kick alter_gha_task workflow to RESUME individual tasks, as needed deploy_chain_phase_3: - dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) # kick alter_gha_task workflow to RESUME individual tasks, as needed - -.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 \ No newline at end of file + +.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery \ No newline at end of file diff --git a/packages.yml b/packages.yml index 52e2031..3563446 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: + revision: