updates for fsc-evm tweaks

.github/workflows/dbt_alter_all_gha_tasks.yml

@@ -22,7 +22,6 @@ jobs:
   called_workflow_template:
     uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta
     with:
-      task_action: |
+      task_action: ${{ inputs.task_action }}
-         ${{ inputs.task_action }}
       target: prod
     secrets: inherit

.github/workflows/dbt_alter_gha_task.yml

@@ -26,9 +26,7 @@ jobs:
   called_workflow_template:
     uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_task.yml@pre-release/v4-beta
     with:
-      workflow_name: |
+      workflow_name: ${{ inputs.workflow_name }}
-         ${{ inputs.workflow_name }}
+      task_action: ${{ inputs.task_action }}
-      task_action: |
-         ${{ inputs.task_action }}
       target: prod
     secrets: inherit

.github/workflows/dbt_deploy_new_workflows.yml

@@ -15,5 +15,5 @@ jobs:
     with:
       command_name: Deploy New Github Actions
       command: |
-        make deploy_new_github_action DBT_TARGET=prod
+        make deploy_new_gha_tasks DBT_TARGET=prod
     secrets: inherit

.github/workflows/dbt_integration_test.yml

@@ -5,6 +5,10 @@ on:
   workflow_dispatch:
     branches:
       - "main"
+    inputs:
+      environment:
+        type: string
+        required: true
 
 concurrency: ${{ github.workflow }}
 
@@ -12,6 +16,7 @@ jobs:
   called_workflow_template:
     uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta
     with:
+      target: ${{ inputs.environment }}
       command: |
         dbt test --selector 'integration_tests'
     secrets: inherit

.github/workflows/dbt_run_scheduled_main.yml

@@ -15,8 +15,5 @@ jobs:
     with:
       command_name: Run Main Models
       command: |
-        dbt run -m "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft"
+        dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
-      command_name_2: Run Streamline Models
-      command_2: |
-        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete"
     secrets: inherit

.github/workflows/dbt_run_streamline_chainhead.yml

@@ -15,8 +15,6 @@ jobs:
     with:
       command_name: Run Chainhead Models
       command: |
-        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash"
+        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash"
-      command_name_2: Run Chainhead Tests
-      command_2: |
         dbt test -m "fsc_evm,tag:chainhead"
     secrets: inherit

data/github_actions__workflows_old.csv

@@ -1,12 +0,0 @@
-workflow_name,workflow_schedule
-dbt_run_streamline_chainhead,"24,54 * * * *"
-dbt_run_scheduled_main,"2 * * * *"
-dbt_run_scheduled_curated,"15 */4 * * *"
-dbt_run_scheduled_decoded_logs,"10 * * * *"
-dbt_test_intraday,"40 */4 * * *"
-dbt_test_daily,"23 8 * * *"
-dbt_test_monthly,"6 7 28 * *"
-dbt_run_scheduled_abis,"32 23 * * *"
-dbt_run_streamline_decoded_logs_history,"6 22 * * 6"
-dbt_run_full_observability,"0 20 1 * *"
-dbt_run_scheduled_scores,"15 5 * * *"

dbt_project.yml

@@ -60,7 +60,7 @@ models:
       relation: true
       columns: true
     +on_schema_change: "append_new_columns"
-  livequery_base:
+  livequery_models:
     +materialized: ephemeral
   fsc_evm:
     +enabled: false # disable fsc_evm package by default

makefile

@@ -1,41 +1,109 @@
 DBT_TARGET ?= dev
 
 cleanup_time:
+	@set -e; \
 	rm -f package-lock.yml && dbt clean && dbt deps
 
-deploy_github_actions:
+deploy_gha_workflows_table:
-	dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
+	@set -e; \
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
+	echo "Collecting workflow names..." ; \
-	dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+	WORKFLOW_VALUES="" ; \
+	for file in $$(find .github/workflows -name "*.yml" -type f); do \
+		filename=$$(basename "$$file" .yml) ; \
+		if [ -z "$$WORKFLOW_VALUES" ]; then \
+			WORKFLOW_VALUES="('$$filename')" ; \
+		else \
+			WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \
+		fi ; \
+	done ; \
+	echo "Found workflows: $$WORKFLOW_VALUES" ; \
+	dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET)
 
-deploy_new_github_action:
+deploy_gha_tasks:
-	dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET)
+	@set -e; \
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
-	dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+	dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
+
+deploy_new_gha_tasks:
+	@set -e; \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
 
 deploy_livequery:
-	dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
+	@set -e; \
-	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
+	dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
 	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
 
 deploy_chain_phase_1:
-	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
+	@set -e; \
-	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
+	read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \
-	dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
+	dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
-	dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET)
+	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
-	dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET)
+	dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
-	# kick chainhead workflow
+	dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \
-	# wait ~10 minutes
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
-	# run deploy_chain_phase_2
+		if [ "$$receipts_by_hash" = "n" ]; then \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+		else \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+		fi; \
+	else \
+		if [ "$$receipts_by_hash" = "n" ]; then \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+		else \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+		fi; \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_2"
 
 deploy_chain_phase_2:
-	dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET)
+	@set -e; \
-	make deploy_github_actions -t $(DBT_TARGET)
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
-	# tasks set to SUSPEND by default
+		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
-	# kick alter_gha_task workflow to RESUME individual tasks, as needed
+		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_3"
 
 deploy_chain_phase_3:
-	dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET)
+	@set -e; \
-	# kick alter_gha_task workflow to RESUME individual tasks, as needed
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_4"
 
-.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery
+deploy_chain_phase_4:
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
+	fi; \
+
+.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4

models/sources.yml

@@ -6,12 +6,13 @@ sources:
     schema: |
       {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }}
     tables:
-      - name: receipts
       - name: blocks
-      - name: traces
       - name: transactions
-      - name: decoded_logs
+      - name: receipts
+      - name: receipts_by_hash
+      - name: traces
       - name: confirm_blocks
+      - name: decoded_logs
       - name: contract_abis
   - name: crosschain_silver
     database: >-
@@ -45,6 +46,7 @@ sources:
     schema: silver
     tables:
       - name: verified_abis
+      - name: complete_event_abis
   - name: fsc_evm_admin
     database: >-
       {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }}
@@ -56,4 +58,5 @@ sources:
     database: "{{ target.database }}"
     schema: silver
     tables:
-      - name: logs
+      - name: logs
+      - name: decoded_logs

packages.yml

@@ -1,3 +1,3 @@
 packages:
   - git: https://github.com/FlipsideCrypto/fsc-evm.git
-    revision: "update/misc-macro-changes"
+    revision: v4.0.0-beta.25