fsc-evm v4 compatibility

.github/workflows/dbt_alter_all_gha_tasks.yml

@@ -0,0 +1,27 @@
+name: dbt_alter_all_gha_tasks
+run-name: dbt_alter_all_gha_tasks
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      task_action:
+        type: choice 
+        description: Action to perform on all tasks
+        required: true
+        options:
+          - RESUME
+          - SUSPEND
+        default: RESUME
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta
+    with:
+      task_action: ${{ inputs.task_action }}
+      target: prod
+    secrets: inherit

.github/workflows/dbt_alter_gha_task.yml

@@ -0,0 +1,32 @@
+name: dbt_alter_gha_task
+run-name: dbt_alter_gha_task
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      workflow_name:
+        type: string
+        description: Name of the workflow to perform the action on, no .yml extension
+        required: true
+      task_action:
+        type: choice 
+        description: Action to perform
+        required: true
+        options:
+          - SUSPEND
+          - RESUME
+        default: SUSPEND
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_task.yml@pre-release/v4-beta
+    with:
+      workflow_name: ${{ inputs.workflow_name }}
+      task_action: ${{ inputs.task_action }}
+      target: prod
+    secrets: inherit

.github/workflows/dbt_deploy_new_workflows.yml

@@ -6,47 +6,14 @@ on:
     branches:
       - "main"
 
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Deploy new workflows as Snowflake tasks
-        run: |
-          make deploy_new_github_action DBT_TARGET=prod
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Deploy New Github Actions
+      command: |
+        make deploy_new_gha_tasks DBT_TARGET=prod
+    secrets: inherit

.github/workflows/dbt_docs_update.yml

@@ -5,79 +5,10 @@ on:
     branches:
       - "main"
 
-env:
-  USE_VARS: "${{ vars.USE_VARS }}"
-  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
-  DBT_VERSION: "${{ vars.DBT_VERSION }}"
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: refresh ddl for datashare
-        run: |
-          cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi;
-
-      - name: checkout docs branch
-        run: |
-          git checkout -B docs origin/main
-
-      - name: generate dbt docs
-        run: dbt docs generate -t prod
-
-      - name: move files to docs directory
-        run: |
-          mkdir -p ./docs
-          cp target/{catalog.json,manifest.json,index.html} docs/
-
-      - name: clean up target directory
-        run: dbt clean
-
-      - name: check for changes
-        run: git status
-
-      - name: stage changed files
-        run: git add .
-
-      - name: commit changed files
-        run: |
-          git config user.email "abc@xyz"
-          git config user.name "github-actions"
-          git commit -am "Auto-update docs"
-
-      - name: push changes to docs
-        run: |
-          git push -f --set-upstream origin docs
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_docs_update.yml@pre-release/v4-beta
+    secrets: inherit

.github/workflows/dbt_integration_test.yml

@@ -3,22 +3,20 @@ run-name: ${{ github.event.inputs.branch }}
 
 on:
   workflow_dispatch:
+    branches:
+      - "main"
+    inputs:
+      environment:
+        type: string
+        required: true
 
 concurrency: ${{ github.workflow }}
 
 jobs:
   called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta
     with:
-      command: >
+      target: ${{ inputs.environment }}
+      command: |
         dbt test --selector 'integration_tests'
-      environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }}
-      warehouse: ${{ vars.WAREHOUSE }}
-    secrets: inherit
-
-  notify-failure:
-    needs: [called_workflow_template]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+    secrets: inherit

.github/workflows/dbt_run_adhoc.yml

@@ -6,8 +6,8 @@ on:
     branches:
       - "main"
     inputs:
-      environment:
-        type: choice
+      target:
+        type: choice 
         description: DBT Run Environment
         required: true
         options:
@@ -15,59 +15,29 @@ on:
           - prod
         default: dev
       warehouse:
-        type: choice
+        type: choice 
         description: Snowflake warehouse
-        required: true
+        required: true 
         options:
           - DBT
           - DBT_CLOUD
           - DBT_EMERGENCY
+          - DBT_MEGA
         default: DBT
       dbt_command:
         type: string
-        description: "DBT Run Command"
+        description: 'DBT Run Command'
         required: true
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ inputs.warehouse }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
+    
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_${{ inputs.environment }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-      - name: Run DBT Jobs
-        run: |
-          ${{ inputs.dbt_command }}
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      warehouse: ${{ inputs.warehouse }}
+      target: ${{ inputs.target }}
+      command_name: Run DBT Command
+      command: ${{ inputs.dbt_command }}
+    secrets: inherit

.github/workflows/dbt_run_dev_refresh.yml

@@ -3,80 +3,13 @@ run-name: dbt_run_dev_refresh
 
 on:
   workflow_dispatch:
-  schedule:
-    # Runs “At 1:20 on Monday.” (see https://crontab.guru)
-    - cron: "20 13 * * 1"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
+    branches:
+      - "main"
 
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs_refresh:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-      - name: Run Dev Refresh
-        run: |
-          dbt run-operation fsc_evm.run_sp_create_prod_clone
-
-  notify-failure:
-    needs: [run_dbt_jobs_refresh]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-
-  run_dbt_jobs_udfs:
-    runs-on: ubuntu-latest
-    needs: run_dbt_jobs_refresh
-    environment:
-      name: workflow_dev
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-      - name: Run Recreate UDFs
-        run: |
-          dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
-          dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
-
-  notify-failure2:
-    needs: [run_dbt_jobs_udfs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_dev_refresh.yml@pre-release/v4-beta
+    secrets: inherit

.github/workflows/dbt_run_full_observability.yml

@@ -5,47 +5,18 @@ on:
   workflow_dispatch:
     branches:
       - "main"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
+    
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod_2xl
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Observability Models
+      command: |
+        dbt run --threads 2 --vars '{"MAIN_OBSERV_FULL_TEST_ENABLED":True}' -m "fsc_evm,tag:observability"
+    secrets: inherit
+  
 
-    steps:
-      - uses: actions/checkout@v3
 
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-      - name: Run DBT Jobs
-        run: |
-          dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_heal_models.yml

@@ -0,0 +1,19 @@
+name: dbt_run_heal_models
+run-name: dbt_run_heal_models
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Heal Models
+      command: |
+        dbt run -m "$PROJECT_NAME,tag:heal" --vars '{"HEAL_MODEL":True}'
+    secrets: inherit

.github/workflows/dbt_run_scheduled_abis.yml

@@ -6,51 +6,17 @@ on:
     branches:
       - "main"
 
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Update ABI models
-        run: |
-          dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis"
-
-      - name: Kick off decoded logs history, if there are new ABIs from users
-        run: |
-          dbt run-operation fsc_evm.run_decoded_logs_history
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run ABI Models
+      command: |
+        dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:gold,tag:abis"
+      command_name_2: Kick off decoded logs history, if there are new user submitted ABIs
+      command_2: |
+        dbt run-operation fsc_evm.run_decoded_logs_history
+    secrets: inherit

.github/workflows/dbt_run_scheduled_curated.yml

@@ -0,0 +1,19 @@
+name: dbt_run_scheduled_curated
+run-name: dbt_run_scheduled_curated
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Curated Models
+      command: |
+        dbt run -m "$PROJECT_NAME,tag:curated" "fsc_evm,tag:curated"
+    secrets: inherit

.github/workflows/dbt_run_scheduled_decoded_logs.yml

@@ -1,52 +0,0 @@
-name: dbt_run_scheduled_decoded_logs
-run-name: dbt_run_scheduled_decoded_logs
-
-on:
-  workflow_dispatch:
-    branches:
-      - "main"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
-
-jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Update the silver and gold tables with the latest decoded logs
-        run: |
-          dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_scheduled_decoder.yml

@@ -0,0 +1,19 @@
+name: dbt_run_scheduled_decoder
+run-name: dbt_run_scheduled_decoder
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Decoder Models
+      command: |
+        dbt run -m "fsc_evm,tag:bronze,tag:decoded_logs" "fsc_evm,tag:silver,tag:decoded_logs" "fsc_evm,tag:gold,tag:decoded_logs"
+    secrets: inherit

.github/workflows/dbt_run_scheduled_main.yml

@@ -6,55 +6,14 @@ on:
     branches:
       - "main"
 
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Update main models
-        run: |
-          dbt run -m "fsc_evm,tag:bronze_core" "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" "fsc_evm,tag:silver_confirm_blocks" "fsc_evm,tag:nft_core"
-
-      - name: Send new logs for decoding
-        run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete"
-
-      - name: Get new contract ABIs
-        run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_abis" "fsc_evm,tag:streamline_abis_realtime" "fsc_evm,tag:streamline_abis_complete"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Main Models
+      command: |
+        dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --exclude "fsc_evm,tag:receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
+    secrets: inherit

.github/workflows/dbt_run_scheduled_scores.yml

@@ -1,3 +1,4 @@
+
 name: dbt_run_scheduled_scores
 run-name: dbt_run_scheduled_scores
 
@@ -6,47 +7,14 @@ on:
     branches:
       - "main"
 
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Update scores
-        run: |
-          dbt run -m "fsc_evm,tag:scores"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Scores Models
+      command: |
+        dbt run -m "fsc_evm,tag:scores"
+    secrets: inherit

.github/workflows/dbt_run_streamline_chainhead.yml

@@ -6,51 +6,15 @@ on:
     branches:
       - "main"
 
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Get new block data
-        run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_realtime_confirm_blocks"
-
-      - name: Test chainhead
-        run: |
-          dbt test -m "fsc_evm,tag:chainhead"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Chainhead Models
+      command: |
+        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts"
+        dbt test -m "fsc_evm,tag:chainhead"
+    secrets: inherit

.github/workflows/dbt_run_streamline_decoded_logs_history.yml

@@ -1,56 +0,0 @@
-name: dbt_run_streamline_decoded_logs_history
-run-name: dbt_run_streamline_decoded_logs_history
-
-on:
-  workflow_dispatch:
-    branches:
-      - "main"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
-
-jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Update complete table
-        run: |
-          dbt run -m "fsc_evm,tag:streamline_decoded_logs_complete"
-
-      - name: Decode historical logs
-        run: |
-          dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_streamline_decoder_history.yml

@@ -0,0 +1,22 @@
+name: dbt_run_streamline_decoder_history
+run-name: dbt_run_streamline_decoder_history
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run Decoder Complete
+      command: |
+        dbt run -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete"
+      command_name_2: Run Streamline Decoder History
+      command_2: |
+        dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
+    secrets: inherit

.github/workflows/dbt_run_streamline_history.yml

@@ -6,47 +6,14 @@ on:
     branches:
       - "main"
 
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Get historical data
-        run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_history_confirm_blocks" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      command_name: Run History Models
+      command: |
+        dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" --exclude "fsc_evm,tag:receipts"
+    secrets: inherit

.github/workflows/dbt_test_daily.yml

@@ -1,56 +1,23 @@
-name: dbt_test_daily
+name: dbt_test_daily 
 run-name: dbt_test_daily
 
 on:
   workflow_dispatch:
     branches:
       - "main"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
+    
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_test
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Build daily testing views
-        run: |
-          dbt run -m "fsc_evm,tag:daily_test"
-
-      - name: Run daily tests
-        run: |
-          dbt test -m "fsc_evm,tag:daily_test"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      target: test
+      command_name: Build Daily Testing Views
+      command: |
+        dbt run -m "fsc_evm,tag:daily_test"
+      command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others)
+      command_2: |
+        dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models
+    secrets: inherit

.github/workflows/dbt_test_intraday.yml

@@ -5,53 +5,17 @@ on:
   workflow_dispatch:
     branches:
       - "main"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
+    
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_test
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Build testing views
-        run: |
-          dbt run -m "fsc_evm,tag:recent_test"
-
-      - name: Run intraday tests
-        run: |
-          dbt run -m "fsc_evm,tag:observability"
-          dbt test -m "fsc_evm,tag:recent_test" "fsc_evm,tag:recent_test_confirm_blocks"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      target: test
+      command_name: Run Observability & Recent Tests
+      command: |
+        dbt run -m "fsc_evm,tag:observability"
+        dbt test -m "fsc_evm,tag:recent_test"
+    secrets: inherit

.github/workflows/dbt_test_monthly.yml

@@ -5,52 +5,19 @@ on:
   workflow_dispatch:
     branches:
       - "main"
-
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
+    
 concurrency:
   group: ${{ github.workflow }}
 
 jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment:
-      name: workflow_test
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-
-      - name: Build full testing views
-        run: |
-          dbt run -m "fsc_evm,tag:full_test"
-
-      - name: Test all data
-        run: |
-          dbt test -m "fsc_evm,tag:full_test"
-
-  notify-failure:
-    needs: [run_dbt_jobs]
-    if: failure()
-    uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
-    secrets:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  called_workflow_template:
+    uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta
+    with:
+      target: test
+      command_name: Build Full Testing Views
+      command: |
+        dbt run -m "fsc_evm,tag:full_test"
+      command_name_2: Run Full Tests
+      command_2: |
+        dbt test -m "fsc_evm,tag:full_test"
+    secrets: inherit

data/github_actions__workflows.csv

@@ -1,11 +0,0 @@
-workflow_name,workflow_schedule
-dbt_run_streamline_chainhead,"23,53 * * * *"
-dbt_run_scheduled_main,"1,31 * * * *"
-dbt_run_scheduled_decoded_logs,"8,38 * * * *"
-dbt_test_intraday,"39 */4 * * *"
-dbt_test_daily,"22 8 * * *"
-dbt_test_monthly,"5 7 28 * *"
-dbt_run_scheduled_abis,"11 23 * * *"
-dbt_run_streamline_decoded_logs_history,"5 22 * * 6"
-dbt_run_full_observability,"0 19 1 * *"
-dbt_run_scheduled_scores,"1 5 * * *"

dbt_project.yml

@@ -70,9 +70,11 @@ models:
       columns: true
     +on_schema_change: "append_new_columns"
     main_package:
-      +enabled: false
-      core:
+      +enabled: false # disable main_package by default, enabled other packages as needed
+      admin:
         +enabled: true
+      core:
+        +enabled: true # enable subpackages, as needed
       github_actions:
         +enabled: true
       labels:
@@ -89,6 +91,10 @@ models:
         +enabled: true
       decoded_logs:
         +enabled: true
+    curated_package:
+      +enabled: false
+      stats:
+        +enabled: true
     scores_package:
       +enabled: true
 
@@ -102,7 +108,6 @@ vars:
   WAIT: 0
   HEAL_MODEL: False
   HEAL_MODELS: []
-  START_GHA_TASKS: False
 
   #### STREAMLINE 2.0 BEGIN ####
 
@@ -129,73 +134,3 @@ vars:
         - DBT_CLOUD_CORE
 
   #### STREAMLINE 2.0 END ####
-
-  #### FSC_EVM BEGIN ####
-
-  ### GLOBAL VARIABLES BEGIN ###
-  ## REQUIRED
-  GLOBAL_PROD_DB_NAME: "core"
-  GLOBAL_NODE_SECRET_PATH: "Vault/prod/core/drpc/mainnet"
-  GLOBAL_BLOCKS_PER_HOUR: 1200
-  GLOBAL_WRAPPED_ASSET_ADDRESS: "0x40375c92d9faf44d2f9db9bd9ba41a3317a2404f"
-
-  ## OPTIONAL
-  GLOBAL_USES_RECEIPTS_BY_HASH: True
-  GLOBAL_AVG_TXS_PER_BLOCK: 50
-
-  ### GLOBAL VARIABLES END ###
-
-  ### MAIN_PACKAGE VARIABLES BEGIN ###
-
-  ### CORE ###
-  ## REQUIRED
-  TRACES_FULL_RELOAD_START_BLOCK: 20000000
-
-  ## OPTIONAL
-  # GOLD_FULL_REFRESH: True
-  # SILVER_FULL_REFRESH: True
-
-  ### PRICES ###
-  ## REQUIRED
-  PRICES_NATIVE_SYMBOLS: "CORE"
-  PRICES_PROVIDER_PLATFORMS: ["Core", "core"]
-
-  ### LABELS ###
-
-  ### OBSERVABILITY ###
-  OBSERV_USES_EXCLUSION_LIST_BLOCKS: False
-  OBSERV_USES_EXCLUSION_LIST_TRANSACTIONS: False
-  OBSERV_USES_EXCLUSION_LIST_RECEIPTS: False
-  OBSERV_USES_EXCLUSION_LIST_LOGS: False
-  OBSERV_USES_EXCLUSION_LIST_TRACES: False
-
-  ### MAIN_PACKAGE VARIABLES END ###
-
-  ### DECODER_PACKAGE VARIABLES BEGIN ###
-
-  ## REQUIRED
-
-  DECODER_ABIS_BLOCK_EXPLORER_NAME: "CoreScan"
-  DECODER_ABIS_BLOCK_EXPLORER_URL: "https://openapi.coredao.org/api?module=contract&action=getabi&address="
-  DECODER_ABIS_BLOCK_EXPLORER_URL_SUFFIX: ""
-  DECODER_ABIS_BLOCK_EXPLORER_SECRET_PATH: "Vault/prod/block_explorers/core_scan"
-  DECODER_ABIS_RELEVANT_CONTRACT_LIMIT: 50
-  DECODER_ABIS_RELEVANT_CONTRACT_COUNT: 200
-  DECODER_ABIS_BRONZE_API_TABLE_ENABLED: True
-
-  ### DECODER_PACKAGE VARIABLES END ###
-
-  ### CURATED_PACKAGE VARIABLES BEGIN ###
-
-  ### STATS ###
-  # STATS_TOKEN_ADDRESS: '' ### REQUIRED
-
-  ### CURATED_PACKAGE VARIABLES END ###
-
-  ### SCORES_PACKAGE VARIABLES BEGIN ###
-
-  ### SCORES_FULL_RELOAD_MODE: True
-
-  ### SCORES_PACKAGE VARIABLES END ###
-#### FSC_EVM END ####
-

makefile

@@ -1,44 +1,109 @@
 DBT_TARGET ?= dev
 
-deploy_streamline_functions:
-	rm -f package-lock.yml && dbt clean && dbt deps
-	dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
-	dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
-
 cleanup_time:
+	@set -e; \
 	rm -f package-lock.yml && dbt clean && dbt deps
 
-deploy_streamline_tables:
-	rm -f package-lock.yml && dbt clean && dbt deps
-ifeq ($(findstring dev,$(DBT_TARGET)),dev)
-	dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET)
-else
-	dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET)
-endif
-	dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET)
+deploy_gha_workflows_table:
+	@set -e; \
+	echo "Collecting workflow names..." ; \
+	WORKFLOW_VALUES="" ; \
+	for file in $$(find .github/workflows -name "*.yml" -type f); do \
+		filename=$$(basename "$$file" .yml) ; \
+		if [ -z "$$WORKFLOW_VALUES" ]; then \
+			WORKFLOW_VALUES="('$$filename')" ; \
+		else \
+			WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \
+		fi ; \
+	done ; \
+	echo "Found workflows: $$WORKFLOW_VALUES" ; \
+	dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET)
 
-deploy_streamline_requests:
-	rm -f package-lock.yml && dbt clean && dbt deps
-	dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET)
+deploy_gha_tasks:
+	@set -e; \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
 
-deploy_github_actions:
-	dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
-	dbt seed -s github_actions__workflows -t $(DBT_TARGET)
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
-ifeq ($(findstring dev,$(DBT_TARGET)),dev)
-	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
-else
-	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET)
-endif
+deploy_new_gha_tasks:
+	@set -e; \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
 
-deploy_new_github_action:
-	dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET)
-	dbt seed -s github_actions__workflows -t $(DBT_TARGET)
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
-ifeq ($(findstring dev,$(DBT_TARGET)),dev)
-	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
-else
-	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET)
-endif
+deploy_livequery:
+	@set -e; \
+	dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
 
-.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action
+deploy_chain_phase_1:
+	@set -e; \
+	read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \
+	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		if [ "$$receipts_by_hash" = "n" ]; then \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+		else \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+		fi; \
+	else \
+		if [ "$$receipts_by_hash" = "n" ]; then \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+		else \
+			dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
+			dbt test -m "fsc_evm,tag:chainhead"; \
+			dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+		fi; \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_2"
+
+deploy_chain_phase_2:
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_3"
+
+deploy_chain_phase_3:
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
+	fi; \
+	echo "# wait ~10 minutes"; \
+	echo "# run deploy_chain_phase_4"
+
+deploy_chain_phase_4:
+	@set -e; \
+	if [ "$(DBT_TARGET)" != "prod" ]; then \
+		dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
+	else \
+		dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \
+		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
+	fi; \
+
+.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4

models/observability/silver_observability__exclusion_list.sql

@@ -1,13 +0,0 @@
-{{ config(
-    materialized = 'view',
-    tags = ['observability']
-) }}
-
-SELECT
-    column1 AS block_number
-FROM
-    (
-        VALUES
-            (0),
-            (1)
-    ) AS block_number(column1)

models/sources.yml

@@ -1,25 +1,22 @@
 version: 2
 
 sources:
-  - name: github_actions
-    database: "{{ target.database }}"
-    schema: github_actions
-    tables:
-      - name: workflows
   - name: bronze_streamline
     database: streamline
-    schema: >-
-      {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }}
+    schema: |
+      {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }}
     tables:
       - name: blocks
       - name: transactions
-      - name: >-
-          {{ 'receipts_by_hash' if var("GLOBAL_USES_RECEIPTS_BY_HASH", False) else 'receipts' }}
+      - name: receipts
+      - name: receipts_by_hash
       - name: traces
       - name: confirm_blocks
       - name: decoded_logs
+      - name: contract_abis
   - name: crosschain_silver
-    database: "{{ 'crosschain' if target.database.upper() == var('GLOBAL_PROD_DB_NAME').upper() else 'crosschain_dev' }}"
+    database: >-
+      {{ 'CROSSCHAIN_DEV' if '_DEV' in target.database.upper() else 'CROSSCHAIN' }}
     schema: silver
     tables:
       - name: labels_combined
@@ -29,24 +26,37 @@ sources:
       - name: complete_provider_prices
       - name: complete_token_asset_metadata
       - name: complete_token_prices
-  - name: bronze_api
-    database: "{{ target.database }}"
-    schema: bronze_api
-    tables:
-      - name: contract_abis
   - name: crosschain_public
     database: crosschain
     schema: bronze_public
     tables:
       - name: user_abis
-  - name: silver
+  - name: complete_streamline
+    database: "{{ target.database }}"
+    schema: streamline
+    tables:
+      - name: complete_contract_abis
+  - name: github_actions
+    database: "{{ target.database }}"
+    schema: github_actions
+    tables:
+      - name: workflows
+  - name: abis_silver
     database: "{{ target.database }}"
     schema: silver
     tables:
       - name: verified_abis
-  - name: bronze_abi
+      - name: complete_event_abis
+  - name: fsc_evm_admin
     database: >-
-      {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }}
-    schema: streamline
+      {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }}
+    schema: admin
     tables:
-      - name: complete_contract_abis
+      - name: _master_keys
+      - name: rpc_node_logs
+  - name: logs_temp
+    database: "{{ target.database }}"
+    schema: silver
+    tables:
+      - name: logs
+      - name: decoded_logs

packages.yml

@@ -1,3 +1,3 @@
 packages:
   - git: https://github.com/FlipsideCrypto/fsc-evm.git
-    revision: v3.30.0
+    revision: "updates-for-migrations2"