This commit is contained in:
sam 2025-04-23 13:28:37 +08:00
parent 587d26f660
commit 07dcdfc129
24 changed files with 235 additions and 73 deletions

View File

@ -11,14 +11,14 @@ on:
description: Name of the workflow to perform the action on, no .yml extension
required: true
task_action:
type: choice
type: choice
description: Action to perform
required: true
options:
- SUSPEND
- RESUME
default: SUSPEND
env:
DBT_PROFILES_DIR: ./
@ -39,8 +39,15 @@ jobs:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7
with:
workflow_name: |
${{ inputs.workflow_name }}
${{ inputs.workflow_name }}
task_action: |
${{ inputs.task_action }}
${{ inputs.task_action }}
environment: workflow_prod
secrets: inherit
secrets: inherit
notify-failure:
needs: [called_workflow_template]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_test
steps:
@ -39,7 +39,14 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Deploy New Github Actions
run: |
make deploy_new_github_action DBT_TARGET=prod
make deploy_new_github_action DBT_TARGET=prod
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -39,10 +39,10 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: refresh ddl for datashare
run: |
cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi;
cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi;
- name: checkout docs branch
run: |
@ -73,4 +73,11 @@ jobs:
- name: push changes to docs
run: |
git push -f --set-upstream origin docs
git push -f --set-upstream origin docs
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -12,6 +12,13 @@ jobs:
with:
command: >
dbt test --selector 'integration_tests'
environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }}
environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }}
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
secrets: inherit
notify-failure:
needs: [called_workflow_template]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -7,7 +7,7 @@ on:
- "main"
inputs:
environment:
type: choice
type: choice
description: DBT Run Environment
required: true
options:
@ -15,9 +15,9 @@ on:
- prod
default: dev
warehouse:
type: choice
type: choice
description: Snowflake warehouse
required: true
required: true
options:
- DBT
- DBT_CLOUD
@ -25,9 +25,9 @@ on:
default: DBT
dbt_command:
type: string
description: 'DBT Run Command'
description: "DBT Run Command"
required: true
env:
DBT_PROFILES_DIR: ./
@ -46,7 +46,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_${{ inputs.environment }}
steps:
@ -63,4 +63,11 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
${{ inputs.dbt_command }}
${{ inputs.dbt_command }}
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -7,9 +7,9 @@ on:
- "main"
inputs:
warehouse:
type: choice
type: choice
description: Snowflake warehouse
required: true
required: true
options:
- DBT
- DBT_CLOUD
@ -17,9 +17,9 @@ on:
default: DBT
dbt_command:
type: string
description: 'DBT Run Command'
description: "DBT Run Command"
required: true
env:
DBT_PROFILES_DIR: ./
DBT_VERSION: "${{ vars.DBT_VERSION }}"
@ -64,4 +64,11 @@ jobs:
name: dbt-logs
path: |
logs
target
target
notify-failure:
needs: [dbt]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -43,6 +43,13 @@ jobs:
run: |
dbt run-operation fsc_evm.run_sp_create_prod_clone
notify-failure:
needs: [run_dbt_jobs_refresh]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
run_dbt_jobs_udfs:
runs-on: ubuntu-latest
needs: run_dbt_jobs_refresh
@ -65,4 +72,11 @@ jobs:
- name: Run Recreate UDFs
run: |
dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
notify-failure:
needs: [run_dbt_jobs_udfs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod_2xl
steps:
@ -43,6 +43,10 @@ jobs:
- name: Run Observability Models
run: |
dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -42,4 +42,11 @@ jobs:
- name: Run Heal Models
run: |
dbt run -m "boba_models,tag:heal" --vars '{"HEAL_MODEL":True}'
dbt run -m "boba_models,tag:heal" --vars '{"HEAL_MODEL":True}'
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -45,7 +45,14 @@ jobs:
run: |
reorg_model_list=$(dbt list --select "boba_models,tag:reorg" --resource-type model --output name | grep '__' | awk -F'.' '{print $NF}' | tr '\n' ',' | sed 's/,$//')
echo "model_list=$reorg_model_list" >> $GITHUB_OUTPUT
- name: Execute block_reorg macro
run: |
dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && awk '/SQL status/ {print; next} /DELETE FROM/{getline; print} /\/\* {/ {print}' logs/dbt.log
dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && awk '/SQL status/ {print; next} /DELETE FROM/{getline; print} /\/\* {/ {print}' logs/dbt.log
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -47,3 +47,10 @@ jobs:
- name: Kick off decoded logs history, if there are new ABIs from users
run: |
dbt run-operation fsc_evm.run_decoded_logs_history
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -42,4 +42,11 @@ jobs:
- name: Run Curated Models
run: |
dbt run -m "boba_models,tag:curated" "fsc_evm,tag:curated"
dbt run -m "boba_models,tag:curated" "fsc_evm,tag:curated"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -42,4 +42,11 @@ jobs:
- name: Update the silver and gold tables with the latest decoded logs
run: |
dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs"
dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -51,3 +51,10 @@ jobs:
- name: Get new contract ABIs
run: |
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_abis" "fsc_evm,tag:streamline_abis_realtime" "fsc_evm,tag:streamline_abis_complete"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -1,4 +1,3 @@
name: dbt_run_scheduled_scores
run-name: dbt_run_scheduled_scores
@ -43,4 +42,11 @@ jobs:
- name: Update scores
run: |
dbt run -m "fsc_evm,tag:scores"
dbt run -m "fsc_evm,tag:scores"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -39,11 +39,18 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Run Chainhead Models
run: |
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts"
- name: Run Chainhead Tests
run: |
dbt test -m "fsc_evm,tag:chainhead"
dbt test -m "fsc_evm,tag:chainhead"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -41,4 +41,11 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt run -m "fsc_evm,tag:decoded_logs"
dbt run -m "fsc_evm,tag:decoded_logs"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -45,4 +45,11 @@ jobs:
- name: Decode historical logs
run: |
dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -39,7 +39,14 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Run History Models
run: |
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts"
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -1,11 +1,11 @@
name: dbt_test_daily
name: dbt_test_daily
run-name: dbt_test_daily
on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_test
steps:
@ -34,16 +34,23 @@ jobs:
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Build Daily Testing Views
run: |
dbt run -m "fsc_evm,tag:daily_test"
- name: Run Daily Tests
run: |
dbt test -m "fsc_evm,tag:daily_test"
dbt test -m "fsc_evm,tag:daily_test"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,7 +24,7 @@ concurrency:
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_test
steps:
@ -39,7 +39,7 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Build Recent Testing Views
run: |
dbt run -m "fsc_evm,tag:recent_test"
@ -47,5 +47,10 @@ jobs:
- name: Run Recent Tests
run: >
dbt test -m "fsc_evm,tag:recent_test" --exclude "fsc_evm.main_package.core.nft.tests.dbt_expectations_expect_row_values_to_have_recent_data_test_nft__ez_nft_transfers_recent_BLOCK_TIMESTAMP__hour__3 fsc_evm.main_package.core.nft.tests.dbt_expectations_expect_row_values_to_have_recent_data_test_nft__ez_nft_transfers_recent_INSERTED_TIMESTAMP__hour__3 fsc_evm.main_package.core.bronze.token_reads.dbt_expectations_expect_row_values_to_have_recent_data_bronze_api__token_reads__INSERTED_TIMESTAMP__day__1"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -20,11 +20,11 @@ env:
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_test
steps:
@ -46,4 +46,11 @@ jobs:
- name: Run Full Tests
run: |
dbt test -m "fsc_evm,tag:full_test"
dbt test -m "fsc_evm,tag:full_test"
notify-failure:
needs: [run_dbt_jobs]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
@ -24,4 +24,11 @@ concurrency:
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7
secrets: inherit
secrets: inherit
notify-failure:
needs: [called_workflow_template]
if: failure()
uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main
secrets:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@ -1,3 +1,3 @@
packages:
- git: https://github.com/FlipsideCrypto/fsc-evm.git
revision: v3.28.1
revision: v3.30.0