updates for fsc-evm tweaks

2026-02-06 09:26:47 +00:00 · 2025-04-28 11:42:02 -06:00 · 2025-04-28 11:42:02 -06:00 · ed1554e228
commit ed1554e228
parent 74e30e349f
11 changed files with 113 additions and 57 deletions
--- a/.github/workflows/dbt_alter_all_gha_tasks.yml
+++ b/.github/workflows/dbt_alter_all_gha_tasks.yml
@ -22,7 +22,6 @@ jobs:
  called_workflow_template:
    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta
    with:
-      task_action: |
-         ${{ inputs.task_action }}
+      task_action: ${{ inputs.task_action }}
      target: prod
    secrets: inherit
--- a/.github/workflows/dbt_alter_gha_task.yml
+++ b/.github/workflows/dbt_alter_gha_task.yml
@ -26,9 +26,7 @@ jobs:
  called_workflow_template:
    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_task.yml@pre-release/v4-beta
    with:
-      workflow_name: |
-         ${{ inputs.workflow_name }}
-      task_action: |
-         ${{ inputs.task_action }}
+      workflow_name: ${{ inputs.workflow_name }}
+      task_action: ${{ inputs.task_action }}
      target: prod
    secrets: inherit
--- a/.github/workflows/dbt_deploy_new_workflows.yml
+++ b/.github/workflows/dbt_deploy_new_workflows.yml
@ -15,5 +15,5 @@ jobs:
    with:
      command_name: Deploy New Github Actions
      command: |
-        make deploy_new_github_action DBT_TARGET=prod
+        make deploy_new_gha_tasks DBT_TARGET=prod
    secrets: inherit
--- a/.github/workflows/dbt_integration_test.yml
+++ b/.github/workflows/dbt_integration_test.yml
@ -5,6 +5,10 @@ on:
  workflow_dispatch:
    branches:
      - "main"
+    inputs:
+      environment:
+        type: string
+        required: true

 concurrency: ${{ github.workflow }}

@ -12,6 +16,7 @@ jobs:
  called_workflow_template:
    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta
    with:
+      target: ${{ inputs.environment }}
      command: |
        dbt test --selector 'integration_tests'
    secrets: inherit
--- a/.github/workflows/dbt_run_scheduled_main.yml
+++ b/.github/workflows/dbt_run_scheduled_main.yml
@ -15,8 +15,5 @@ jobs:
    with:
      command_name: Run Main Models
      command: |
-        dbt run -m "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft"
-      command_name_2: Run Streamline Models
-      command_2: |
-        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete"
+        dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
    secrets: inherit
--- a/.github/workflows/dbt_run_streamline_chainhead.yml
+++ b/.github/workflows/dbt_run_streamline_chainhead.yml
@ -15,8 +15,6 @@ jobs:
    with:
      command_name: Run Chainhead Models
      command: |
-        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash"
-      command_name_2: Run Chainhead Tests
-      command_2: |
+        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash"
        dbt test -m "fsc_evm,tag:chainhead"
    secrets: inherit
--- a/data/github_actions__workflows_old.csv
+++ b/data/github_actions__workflows_old.csv
@ -1,12 +0,0 @@
-workflow_name,workflow_schedule
-dbt_run_streamline_chainhead,"24,54 * * * *"
-dbt_run_scheduled_main,"2 * * * *"
-dbt_run_scheduled_curated,"15 */4 * * *"
-dbt_run_scheduled_decoded_logs,"10 * * * *"
-dbt_test_intraday,"40 */4 * * *"
-dbt_test_daily,"23 8 * * *"
-dbt_test_monthly,"6 7 28 * *"
-dbt_run_scheduled_abis,"32 23 * * *"
-dbt_run_streamline_decoded_logs_history,"6 22 * * 6"
-dbt_run_full_observability,"0 20 1 * *"
-dbt_run_scheduled_scores,"15 5 * * *"
--- a/dbt_project.yml
+++ b/dbt_project.yml
@ -60,7 +60,7 @@ models:
      relation: true
      columns: true
    +on_schema_change: "append_new_columns"
-  livequery_base:
+  livequery_models:
    +materialized: ephemeral
  fsc_evm:
    +enabled: false # disable fsc_evm package by default
--- a/118
+++ b/118
@ -1,41 +1,109 @@
 DBT_TARGET ?= dev

 cleanup_time:
+	@set -e; \
 	rm -f package-lock.yml && dbt clean && dbt deps

-deploy_github_actions:
-	dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
-	dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+deploy_gha_workflows_table:
+	@set -e; \
+	echo "Collecting workflow names..." ; \
+	WORKFLOW_VALUES="" ; \
+	for file in $$(find .github/workflows -name "*.yml" -type f); do \
+		filename=$$(basename "$$file" .yml) ; \
+		if [ -z "$$WORKFLOW_VALUES" ]; then \
+			WORKFLOW_VALUES="('$$filename')" ; \
+		else \
+			WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \
+		fi ; \
+	done ; \
+	echo "Found workflows: $$WORKFLOW_VALUES" ; \
+	dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET)

-deploy_new_github_action:
-	dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET)
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
-	dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+deploy_gha_tasks:
+	@set -e; \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
+
+deploy_new_gha_tasks:
+	@set -e; \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)

 deploy_livequery:
-	dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
-	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
+	@set -e; \
+	dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
 	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)

 deploy_chain_phase_1:
-	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
-	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
-	dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
-	dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET)
-	dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET)
-	# kick chainhead workflow
-	# wait ~10 minutes
-	# run deploy_chain_phase_2
+	@set -e; \
+	read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \
+	dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		if [ "$$receipts_by_hash" = "n" ]; then \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+		else \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+		fi; \
+	else \
+		if [ "$$receipts_by_hash" = "n" ]; then \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+		else \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+		fi; \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_2"

 deploy_chain_phase_2:
-	dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET)
-	make deploy_github_actions -t $(DBT_TARGET)
-	# tasks set to SUSPEND by default
-	# kick alter_gha_task workflow to RESUME individual tasks, as needed
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_3"

 deploy_chain_phase_3:
-	dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET)
-	# kick alter_gha_task workflow to RESUME individual tasks, as needed
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_4"

-.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery
+deploy_chain_phase_4:
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
+	fi; \
+
+.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4
--- a/models/sources.yml
+++ b/models/sources.yml
@ -6,12 +6,13 @@ sources:
    schema: |
      {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }}
    tables:
-      - name: receipts
      - name: blocks
-      - name: traces
      - name: transactions
-      - name: decoded_logs
+      - name: receipts
+      - name: receipts_by_hash
+      - name: traces
      - name: confirm_blocks
+      - name: decoded_logs
      - name: contract_abis
  - name: crosschain_silver
    database: >-
@ -45,6 +46,7 @@ sources:
    schema: silver
    tables:
      - name: verified_abis
+      - name: complete_event_abis
  - name: fsc_evm_admin
    database: >-
      {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }}
@ -56,4 +58,5 @@ sources:
    database: "{{ target.database }}"
    schema: silver
    tables:
-      - name: logs
+      - name: logs
+      - name: decoded_logs
--- a/packages.yml
+++ b/packages.yml
@ -1,3 +1,3 @@
 packages:
  - git: https://github.com/FlipsideCrypto/fsc-evm.git
-    revision: "update/misc-macro-changes"
+    revision: v4.0.0-beta.25