diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml index ebcc2c3..19f9e44 100644 --- a/.github/workflows/dbt_run_scheduled_abis.yml +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -15,5 +15,5 @@ jobs: with: command_name: Run ABI Models command: | - dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:gold,tag:abis" secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_decoder.yml b/.github/workflows/dbt_run_scheduled_decoder.yml index aaa8a3e..a0b7a2b 100644 --- a/.github/workflows/dbt_run_scheduled_decoder.yml +++ b/.github/workflows/dbt_run_scheduled_decoder.yml @@ -15,5 +15,5 @@ jobs: with: command_name: Run Decoder Models command: | - dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs" + dbt run -m "fsc_evm,tag:bronze,tag:decoded_logs" "fsc_evm,tag:silver,tag:decoded_logs" "fsc_evm,tag:gold,tag:decoded_logs" secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 5cf8a86..933c23a 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -15,8 +15,8 @@ jobs: with: command_name: Run Main Models command: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" + dbt run -m "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" command_name_2: Run Streamline Models command_2: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 07317cb..c602cf4 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -15,7 +15,7 @@ jobs: with: command_name: Run Chainhead Models command: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" command_name_2: Run Chainhead Tests command_2: | dbt test -m "fsc_evm,tag:chainhead" diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml index a17dc7a..b736c9c 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history.yml @@ -13,7 +13,10 @@ jobs: called_workflow_template: uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta with: - command_name: Run Decoder Models + command_name: Run Decoder Complete command: | - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m "fsc_evm,tag:streamline_decoded_logs_complete" "fsc_evm,tag:streamline_decoded_logs_history" + dbt run -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" + command_name_2: Run Streamline Decoder History + command_2: | + dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 09a34de..08dfbc6 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -15,5 +15,5 @@ jobs: with: command_name: Run History Models command: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" --exclude "fsc_evm,tag:receipts_by_hash" secrets: inherit \ No newline at end of file diff --git a/makefile b/makefile index 3886866..1fa5ac4 100644 --- a/makefile +++ b/makefile @@ -1,50 +1,35 @@ DBT_TARGET ?= dev -deploy_streamline_functions: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - cleanup_time: rm -f package-lock.yml && dbt clean && dbt deps -deploy_streamline_tables: - rm -f package-lock.yml && dbt clean && dbt deps -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) -else - dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET) -endif - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) - -deploy_streamline_requests: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) - deploy_github_actions: dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif deploy_new_github_action: dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET) dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif -release_main_package: - dbt run-operation fsc_evm.release_chain --args '{"schema_name": "core", "role_name": "internal_dev"}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.release_chain --args '{"schema_name": "price", "role_name": "internal_dev"}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.release_chain --args '{"schema_name": "nft", "role_name": "internal_dev"}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.release_chain --args '{"schema_name": "core", "role_name": "velocity_ethereum"}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.release_chain --args '{"schema_name": "price", "role_name": "velocity_ethereum"}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.release_chain --args '{"schema_name": "nft", "role_name": "velocity_ethereum"}' -t $(DBT_TARGET) +deploy_chain_phase_1: + dbt run -m livequery_models.deploy.core --vars '{UPDATE_UDFS_AND_SPS: true}' -t $(DBT_TARGET) + dbt run-operation fsc_evm.livequery_grants -t $(DBT_TARGET) + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{UPDATE_UDFS_AND_SPS: true}' -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET) + # kick chainhead workflow + # wait ~10 minutes + # run deploy_chain_phase_2 + +deploy_chain_phase_2: + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) + make deploy_github_actions -t $(DBT_TARGET) + # tasks set to SUSPEND by default + # kick alter_gha_task workflow to RESUME individual tasks, as needed + +deploy_chain_phase_3: + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) + # kick alter_gha_task workflow to RESUME individual tasks, as needed -.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action release_main_package \ No newline at end of file +.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 \ No newline at end of file