From ed1554e228a9fce6eabb36f5b89a02ec9a1a97bd Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 28 Apr 2025 11:42:02 -0600 Subject: [PATCH] updates for fsc-evm tweaks --- .github/workflows/dbt_alter_all_gha_tasks.yml | 3 +- .github/workflows/dbt_alter_gha_task.yml | 6 +- .../workflows/dbt_deploy_new_workflows.yml | 2 +- .github/workflows/dbt_integration_test.yml | 5 + .github/workflows/dbt_run_scheduled_main.yml | 5 +- .../dbt_run_streamline_chainhead.yml | 4 +- data/github_actions__workflows_old.csv | 12 -- dbt_project.yml | 2 +- makefile | 118 ++++++++++++++---- models/sources.yml | 11 +- packages.yml | 2 +- 11 files changed, 113 insertions(+), 57 deletions(-) delete mode 100644 data/github_actions__workflows_old.csv diff --git a/.github/workflows/dbt_alter_all_gha_tasks.yml b/.github/workflows/dbt_alter_all_gha_tasks.yml index a868731..086327d 100644 --- a/.github/workflows/dbt_alter_all_gha_tasks.yml +++ b/.github/workflows/dbt_alter_all_gha_tasks.yml @@ -22,7 +22,6 @@ jobs: called_workflow_template: uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta with: - task_action: | - ${{ inputs.task_action }} + task_action: ${{ inputs.task_action }} target: prod secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index 4f0500c..1cb60fc 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -26,9 +26,7 @@ jobs: called_workflow_template: uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_task.yml@pre-release/v4-beta with: - workflow_name: | - ${{ inputs.workflow_name }} - task_action: | - ${{ inputs.task_action }} + workflow_name: ${{ inputs.workflow_name }} + task_action: ${{ inputs.task_action }} target: prod secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_deploy_new_workflows.yml b/.github/workflows/dbt_deploy_new_workflows.yml index 3ead6df..a9a877d 100644 --- a/.github/workflows/dbt_deploy_new_workflows.yml +++ b/.github/workflows/dbt_deploy_new_workflows.yml @@ -15,5 +15,5 @@ jobs: with: command_name: Deploy New Github Actions command: | - make deploy_new_github_action DBT_TARGET=prod + make deploy_new_gha_tasks DBT_TARGET=prod secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml index d994154..d0d5385 100644 --- a/.github/workflows/dbt_integration_test.yml +++ b/.github/workflows/dbt_integration_test.yml @@ -5,6 +5,10 @@ on: workflow_dispatch: branches: - "main" + inputs: + environment: + type: string + required: true concurrency: ${{ github.workflow }} @@ -12,6 +16,7 @@ jobs: called_workflow_template: uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta with: + target: ${{ inputs.environment }} command: | dbt test --selector 'integration_tests' secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 71b7487..23c70be 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -15,8 +15,5 @@ jobs: with: command_name: Run Main Models command: | - dbt run -m "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" - command_name_2: Run Streamline Models - command_2: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" + dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index c602cf4..efa0176 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -15,8 +15,6 @@ jobs: with: command_name: Run Chainhead Models command: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" - command_name_2: Run Chainhead Tests - command_2: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" dbt test -m "fsc_evm,tag:chainhead" secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows_old.csv b/data/github_actions__workflows_old.csv deleted file mode 100644 index dff8f36..0000000 --- a/data/github_actions__workflows_old.csv +++ /dev/null @@ -1,12 +0,0 @@ -workflow_name,workflow_schedule -dbt_run_streamline_chainhead,"24,54 * * * *" -dbt_run_scheduled_main,"2 * * * *" -dbt_run_scheduled_curated,"15 */4 * * *" -dbt_run_scheduled_decoded_logs,"10 * * * *" -dbt_test_intraday,"40 */4 * * *" -dbt_test_daily,"23 8 * * *" -dbt_test_monthly,"6 7 28 * *" -dbt_run_scheduled_abis,"32 23 * * *" -dbt_run_streamline_decoded_logs_history,"6 22 * * 6" -dbt_run_full_observability,"0 20 1 * *" -dbt_run_scheduled_scores,"15 5 * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index f4f378f..573024b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -60,7 +60,7 @@ models: relation: true columns: true +on_schema_change: "append_new_columns" - livequery_base: + livequery_models: +materialized: ephemeral fsc_evm: +enabled: false # disable fsc_evm package by default diff --git a/makefile b/makefile index 69c29ae..52ca153 100644 --- a/makefile +++ b/makefile @@ -1,41 +1,109 @@ DBT_TARGET ?= dev cleanup_time: + @set -e; \ rm -f package-lock.yml && dbt clean && dbt deps -deploy_github_actions: - dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) - dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) +deploy_gha_workflows_table: + @set -e; \ + echo "Collecting workflow names..." ; \ + WORKFLOW_VALUES="" ; \ + for file in $$(find .github/workflows -name "*.yml" -type f); do \ + filename=$$(basename "$$file" .yml) ; \ + if [ -z "$$WORKFLOW_VALUES" ]; then \ + WORKFLOW_VALUES="('$$filename')" ; \ + else \ + WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \ + fi ; \ + done ; \ + echo "Found workflows: $$WORKFLOW_VALUES" ; \ + dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET) -deploy_new_github_action: - dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) - dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) +deploy_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) + +deploy_new_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) deploy_livequery: - dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + @set -e; \ + dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) deploy_chain_phase_1: - dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET) - # kick chainhead workflow - # wait ~10 minutes - # run deploy_chain_phase_2 + @set -e; \ + read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \ + dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + if [ "$$receipts_by_hash" = "n" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + fi; \ + else \ + if [ "$$receipts_by_hash" = "n" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_2" deploy_chain_phase_2: - dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) - make deploy_github_actions -t $(DBT_TARGET) - # tasks set to SUSPEND by default - # kick alter_gha_task workflow to RESUME individual tasks, as needed + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_3" deploy_chain_phase_3: - dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) - # kick alter_gha_task workflow to RESUME individual tasks, as needed + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_4" -.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery \ No newline at end of file +deploy_chain_phase_4: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + fi; \ + +.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4 \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index e04977e..18ae91c 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -6,12 +6,13 @@ sources: schema: | {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }} tables: - - name: receipts - name: blocks - - name: traces - name: transactions - - name: decoded_logs + - name: receipts + - name: receipts_by_hash + - name: traces - name: confirm_blocks + - name: decoded_logs - name: contract_abis - name: crosschain_silver database: >- @@ -45,6 +46,7 @@ sources: schema: silver tables: - name: verified_abis + - name: complete_event_abis - name: fsc_evm_admin database: >- {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }} @@ -56,4 +58,5 @@ sources: database: "{{ target.database }}" schema: silver tables: - - name: logs \ No newline at end of file + - name: logs + - name: decoded_logs diff --git a/packages.yml b/packages.yml index 42ae788..b04ff92 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: "update/misc-macro-changes" + revision: v4.0.0-beta.25