diff --git a/.github/workflows/dbt_alter_all_gha_tasks.yml b/.github/workflows/dbt_alter_all_gha_tasks.yml new file mode 100644 index 0000000..29930c5 --- /dev/null +++ b/.github/workflows/dbt_alter_all_gha_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_alter_all_gha_tasks +run-name: dbt_alter_all_gha_tasks + +on: + workflow_dispatch: + branches: + - "main" + inputs: + task_action: + type: choice + description: Action to perform on all tasks + required: true + options: + - RESUME + - SUSPEND + default: RESUME + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@main + with: + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_alter_gha_tasks.yml b/.github/workflows/dbt_alter_gha_tasks.yml new file mode 100644 index 0000000..731486b --- /dev/null +++ b/.github/workflows/dbt_alter_gha_tasks.yml @@ -0,0 +1,32 @@ +name: dbt_alter_gha_tasks +run-name: dbt_alter_gha_tasks + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_tasks.yml@main + with: + workflow_name: ${{ inputs.workflow_name }} + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_deploy_new_workflows.yml b/.github/workflows/dbt_deploy_new_workflows.yml index 1373a88..3975f8b 100644 --- a/.github/workflows/dbt_deploy_new_workflows.yml +++ b/.github/workflows/dbt_deploy_new_workflows.yml @@ -6,65 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Deploy New Github Actions - run: | - make deploy_new_github_action DBT_TARGET=prod - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + command_name: Deploy New Github Actions + command: | + make deploy_new_gha_tasks DBT_TARGET=prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 1512045..b9d7113 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -5,97 +5,10 @@ on: branches: - "main" -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: refresh ddl for datashare - run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - - - name: checkout docs branch - run: | - git checkout -B docs origin/main - - - name: generate dbt docs - run: dbt docs generate -t prod - - - name: move files to docs directory - run: | - mkdir -p ./docs - cp target/{catalog.json,manifest.json,index.html} docs/ - - - name: clean up target directory - run: dbt clean - - - name: check for changes - run: git status - - - name: stage changed files - run: git add . - - - name: commit changed files - run: | - git config user.email "abc@xyz" - git config user.name "github-actions" - git commit -am "Auto-update docs" - - - name: push changes to docs - run: | - git push -f --set-upstream origin docs - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_docs_update.yml@main + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml index 01d6850..f607211 100644 --- a/.github/workflows/dbt_integration_test.yml +++ b/.github/workflows/dbt_integration_test.yml @@ -3,40 +3,20 @@ run-name: ${{ github.event.inputs.branch }} on: workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: string + required: true concurrency: ${{ github.workflow }} jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@main with: - command: > + target: ${{ inputs.environment }} + command: | dbt test --selector 'integration_tests' - environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }} - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index d344324..10c1928 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -6,7 +6,7 @@ on: branches: - "main" inputs: - environment: + target: type: choice description: DBT Run Environment required: true @@ -22,70 +22,22 @@ on: - DBT - DBT_CLOUD - DBT_EMERGENCY + - DBT_MEGA default: DBT dbt_command: type: string description: 'DBT Run Command' required: true -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.warehouse }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_${{ inputs.environment }} - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - ${{ inputs.dbt_command }} - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + warehouse: ${{ inputs.warehouse }} + target: ${{ inputs.target }} + command_name: Run DBT Command + command: ${{ inputs.dbt_command }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index b3e195d..47c1a78 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -6,113 +6,10 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs_refresh: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run Dev Refresh - run: | - dbt run-operation fsc_evm.run_sp_create_prod_clone - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs_refresh - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - - run_dbt_jobs_udfs: - runs-on: ubuntu-latest - needs: run_dbt_jobs_refresh - environment: - name: workflow_dev - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Recreate UDFs - run: | - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs_udfs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_dev_refresh.yml@main + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml new file mode 100644 index 0000000..f6656cb --- /dev/null +++ b/.github/workflows/dbt_run_full_observability.yml @@ -0,0 +1,23 @@ +name: dbt_run_full_observability +run-name: dbt_run_full_observability + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + warehouse: DBT_EMERGENCY + command_name: Run Observability Models + command: | + dbt run --threads 2 --vars '{"MAIN_OBSERV_FULL_TEST_ENABLED":True}' -m "fsc_evm,tag:observability" + secrets: inherit + + + diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index ccbaf96..76d319f 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -6,65 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Main Models - run: | - dbt run -m "monad_models,tag:silver_testnet" "monad_models,tag:gold_testnet" - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + command_name: Run Main Models + command: | + dbt run -m "monad_models,tag:silver_testnet" "monad_models,tag:gold_testnet" "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:streamline,tag:token_reads,tag:realtime" "fsc_evm,tag:streamline,tag:token_reads,tag:complete" --exclude "fsc_evm,tag:receipts_by_hash" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 5c25ab9..e4ee4c3 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -6,69 +6,15 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Chainhead Models - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "monad_models,tag:streamline_testnet_complete" "monad_models,tag:streamline_testnet_realtime" - - - name: Run Chainhead Tests - run: | - dbt test -m "monad_models,tag:chainhead" - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + command_name: Run Chainhead Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "monad_models,tag:streamline_testnet_complete" "monad_models,tag:streamline_testnet_realtime" "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" + dbt test -m "monad_models,tag:chainhead" "fsc_evm,tag:chainhead" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index c9e870d..0587ad7 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -6,65 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run History Models - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "monad_models,tag:streamline_testnet_complete" "monad_models,tag:streamline_testnet_history" - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + command_name: Run History Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" --exclude "fsc_evm,tag:receipts_by_hash" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_temp_backfill.yml b/.github/workflows/dbt_temp_backfill.yml deleted file mode 100644 index b1a2e27..0000000 --- a/.github/workflows/dbt_temp_backfill.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: dbt_temp_backfill -run-name: dbt_temp_backfill - -on: - schedule: - - cron: '0 */3 * * *' - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Temp Backfill - run: | - dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" streamline__get_chainhead streamline__blocks --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t prod - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index d466879..2404285 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -6,69 +6,18 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Daily Testing Views - run: | - dbt run -m "fsc_evm,tag:daily_test" - - - name: Run Daily Tests - run: | - dbt test -m "fsc_evm,tag:daily_test" - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + target: test + command_name: Build Daily Testing Views + command: | + dbt run -m "fsc_evm,tag:daily_test" + command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others) + command_2: | + dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" "fsc_evm,tag:observability" "$PROJECT_NAME,tag:curated" "fsc_evm,tag:curated" livequery_models main_package.core.bronze.token_reads + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index a5f5dd2..912066f 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -6,69 +6,16 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Recent Testing Views - run: | - dbt run -m "fsc_evm,tag:recent_test" - - - name: Run Recent Tests - run: | - dbt test -m "fsc_evm,tag:recent_test" - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + target: test + command_name: Run Observability & Recent Tests + command: | + dbt run -m "fsc_evm,tag:observability" + dbt test -m "fsc_evm,tag:recent_test" "fsc_evm,tag:observability" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml index b8a2bb4..50a40ec 100644 --- a/.github/workflows/dbt_test_monthly.yml +++ b/.github/workflows/dbt_test_monthly.yml @@ -6,69 +6,18 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} - + jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Full Testing Views - run: | - dbt run -m "fsc_evm,tag:full_test" - - - name: Run Full Tests - run: | - dbt test -m "fsc_evm,tag:full_test" - - notify: - runs-on: ubuntu-latest - needs: run_dbt_jobs - if: failure() - environment: workflow_prod - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - pip install requests - pip install -r requirements.txt - dbt deps - - - name: Send Slack notification - run: python macros/python/slack_alert.py - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@main + with: + warehouse: DBT_EMERGENCY + command_name: Build Full Testing Views + command: | + dbt run -m "fsc_evm,tag:full_test" + command_name_2: Run Full Tests + command_2: | + dbt test -m "fsc_evm,tag:full_test" "$PROJECT_NAME,tag:curated" "fsc_evm,tag:curated" + secrets: inherit \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index dd22ecf..6966938 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -79,7 +79,7 @@ models: labels: +enabled: true observability: - +enabled: false + +enabled: true prices: +enabled: true token_reads: