diff --git a/.github/workflows/dbt_alter_all_gha_tasks.yml b/.github/workflows/dbt_alter_all_gha_tasks.yml new file mode 100644 index 0000000..086327d --- /dev/null +++ b/.github/workflows/dbt_alter_all_gha_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_alter_all_gha_tasks +run-name: dbt_alter_all_gha_tasks + +on: + workflow_dispatch: + branches: + - "main" + inputs: + task_action: + type: choice + description: Action to perform on all tasks + required: true + options: + - RESUME + - SUSPEND + default: RESUME + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta + with: + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index 8b5c13f..1cb60fc 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -11,7 +11,7 @@ on: description: Name of the workflow to perform the action on, no .yml extension required: true task_action: - type: choice + type: choice description: Action to perform required: true options: @@ -19,35 +19,14 @@ on: - RESUME default: SUSPEND -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_task.yml@pre-release/v4-beta with: - workflow_name: | - ${{ inputs.workflow_name }} - task_action: | - ${{ inputs.task_action }} - environment: workflow_prod - secrets: inherit - - notify-failure: - needs: [called_workflow_template] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + workflow_name: ${{ inputs.workflow_name }} + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_deploy_new_workflows.yml b/.github/workflows/dbt_deploy_new_workflows.yml index bf69a91..a9a877d 100644 --- a/.github/workflows/dbt_deploy_new_workflows.yml +++ b/.github/workflows/dbt_deploy_new_workflows.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Deploy New Github Actions - run: | - make deploy_new_github_action DBT_TARGET=prod - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Deploy New Github Actions + command: | + make deploy_new_gha_tasks DBT_TARGET=prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index e964899..10ec0ba 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -5,79 +5,10 @@ on: branches: - "main" -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: refresh ddl for datashare - run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - - - name: checkout docs branch - run: | - git checkout -B docs origin/main - - - name: generate dbt docs - run: dbt docs generate -t prod - - - name: move files to docs directory - run: | - mkdir -p ./docs - cp target/{catalog.json,manifest.json,index.html} docs/ - - - name: clean up target directory - run: dbt clean - - - name: check for changes - run: git status - - - name: stage changed files - run: git add . - - - name: commit changed files - run: | - git config user.email "abc@xyz" - git config user.name "github-actions" - git commit -am "Auto-update docs" - - - name: push changes to docs - run: | - git push -f --set-upstream origin docs - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_docs_update.yml@pre-release/v4-beta + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml index bfc2bf0..d0d5385 100644 --- a/.github/workflows/dbt_integration_test.yml +++ b/.github/workflows/dbt_integration_test.yml @@ -3,22 +3,20 @@ run-name: ${{ github.event.inputs.branch }} on: workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: string + required: true concurrency: ${{ github.workflow }} jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta with: - command: > + target: ${{ inputs.environment }} + command: | dbt test --selector 'integration_tests' - environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }} - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit - - notify-failure: - needs: [called_workflow_template] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 5ada2f7..30a3481 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -6,8 +6,8 @@ on: branches: - "main" inputs: - environment: - type: choice + target: + type: choice description: DBT Run Environment required: true options: @@ -15,59 +15,29 @@ on: - prod default: dev warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD - DBT_EMERGENCY + - DBT_MEGA default: DBT dbt_command: type: string - description: "DBT Run Command" + description: 'DBT Run Command' required: true - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.warehouse }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_${{ inputs.environment }} - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - ${{ inputs.dbt_command }} - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + warehouse: ${{ inputs.warehouse }} + target: ${{ inputs.target }} + command_name: Run DBT Command + command: ${{ inputs.dbt_command }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml deleted file mode 100644 index fca3c31..0000000 --- a/.github/workflows/dbt_run_deployment.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: dbt_run_deployment -run-name: ${{ inputs.dbt_command }} - -on: - workflow_dispatch: - branches: - - "main" - inputs: - warehouse: - type: choice - description: Snowflake warehouse - required: true - options: - - DBT - - DBT_CLOUD - - DBT_EMERGENCY - default: DBT - dbt_command: - type: string - description: "DBT Run Command" - required: true - -env: - DBT_PROFILES_DIR: ./ - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - dbt: - runs-on: ubuntu-latest - environment: - name: workflow_prod - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - ${{ inputs.dbt_command }} - - name: Run datashare model - run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" - - name: Store logs - uses: actions/upload-artifact@v3 - with: - name: dbt-logs - path: | - logs - target - - notify-failure: - needs: [dbt] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index d9e8521..938995c 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -6,77 +6,10 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs_refresh: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run Dev Refresh - run: | - dbt run-operation fsc_evm.run_sp_create_prod_clone - - notify-failure: - needs: [run_dbt_jobs_refresh] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - - run_dbt_jobs_udfs: - runs-on: ubuntu-latest - needs: run_dbt_jobs_refresh - environment: - name: workflow_dev - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Recreate UDFs - run: | - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - - notify-failure2: - needs: [run_dbt_jobs_udfs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_dev_refresh.yml@pre-release/v4-beta + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index e128c66..379ffd0 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -5,48 +5,18 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_2xl + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Observability Models + command: | + dbt run --threads 2 --vars '{"MAIN_OBSERV_FULL_TEST_ENABLED":True}' -m "fsc_evm,tag:observability" + secrets: inherit + - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Observability Models - run: | - dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml index 52883b7..6eaf50f 100644 --- a/.github/workflows/dbt_run_heal_models.yml +++ b/.github/workflows/dbt_run_heal_models.yml @@ -5,48 +5,15 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Heal Models - run: | - dbt run -m "boba_models,tag:heal" --vars '{"HEAL_MODEL":True}' - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Heal Models + command: | + dbt run -m "$PROJECT_NAME,tag:heal" --vars '{"HEAL_MODEL":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_operation_reorg.yml b/.github/workflows/dbt_run_operation_reorg.yml deleted file mode 100644 index e89172f..0000000 --- a/.github/workflows/dbt_run_operation_reorg.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: dbt_run_operation_reorg -run-name: dbt_run_operation_reorg - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: List reorg models - id: list_models - run: | - reorg_model_list=$(dbt list --select "boba_models,tag:reorg" --resource-type model --output name | grep '__' | awk -F'.' '{print $NF}' | tr '\n' ',' | sed 's/,$//') - echo "model_list=$reorg_model_list" >> $GITHUB_OUTPUT - - - name: Execute block_reorg macro - run: | - dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && awk '/SQL status/ {print; next} /DELETE FROM/{getline; print} /\/\* {/ {print}' logs/dbt.log - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml index 8683a0c..2eb8581 100644 --- a/.github/workflows/dbt_run_scheduled_abis.yml +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -6,51 +6,17 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run ABI Models - run: | - dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" - - - name: Kick off decoded logs history, if there are new ABIs from users - run: | - dbt run-operation fsc_evm.run_decoded_logs_history - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run ABI Models + command: | + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:gold,tag:abis" + command_name_2: Kick off decoded logs history, if there are new user submitted ABIs + command_2: | + dbt run-operation fsc_evm.run_decoded_logs_history + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml index 516aa5d..61d9d16 100644 --- a/.github/workflows/dbt_run_scheduled_curated.yml +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Curated Models - run: | - dbt run -m "boba_models,tag:curated" "fsc_evm,tag:curated" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Curated Models + command: | + dbt run -m "$PROJECT_NAME,tag:curated" "fsc_evm,tag:curated" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_decoded_logs.yml b/.github/workflows/dbt_run_scheduled_decoded_logs.yml deleted file mode 100644 index a6f8792..0000000 --- a/.github/workflows/dbt_run_scheduled_decoded_logs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: dbt_run_scheduled_decoded_logs -run-name: dbt_run_scheduled_decoded_logs - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update the silver and gold tables with the latest decoded logs - run: | - dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_scheduled_decoder.yml b/.github/workflows/dbt_run_scheduled_decoder.yml new file mode 100644 index 0000000..a0b7a2b --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_decoder.yml @@ -0,0 +1,19 @@ +name: dbt_run_scheduled_decoder +run-name: dbt_run_scheduled_decoder + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Decoder Models + command: | + dbt run -m "fsc_evm,tag:bronze,tag:decoded_logs" "fsc_evm,tag:silver,tag:decoded_logs" "fsc_evm,tag:gold,tag:decoded_logs" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 0f0247c..9390444 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -6,55 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Main Models - run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_confirm_blocks" "fsc_evm,tag:bronze_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" "fsc_evm,tag:nft_core" - - - name: Send new logs for decoding - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" - - - name: Get new contract ABIs - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_abis" "fsc_evm,tag:streamline_abis_realtime" "fsc_evm,tag:streamline_abis_complete" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Main Models + command: | + dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --exclude "fsc_evm,tag:receipts_by_hash" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_scores.yml b/.github/workflows/dbt_run_scheduled_scores.yml index 8944fbc..5c2f0f9 100644 --- a/.github/workflows/dbt_run_scheduled_scores.yml +++ b/.github/workflows/dbt_run_scheduled_scores.yml @@ -1,3 +1,4 @@ + name: dbt_run_scheduled_scores run-name: dbt_run_scheduled_scores @@ -6,47 +7,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update scores - run: | - dbt run -m "fsc_evm,tag:scores" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Scores Models + command: | + dbt run -m "fsc_evm,tag:scores" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 2135db9..efa0176 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -6,51 +6,15 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Chainhead Models - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" - - - name: Run Chainhead Tests - run: | - dbt test -m "fsc_evm,tag:chainhead" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Chainhead Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" + dbt test -m "fsc_evm,tag:chainhead" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml deleted file mode 100644 index 75be0ab..0000000 --- a/.github/workflows/dbt_run_streamline_decoder_history.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: dbt_run_streamline_decoder_history -run-name: dbt_run_streamline_decoder_history - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Update complete table - run: | - dbt run -m "fsc_evm,tag:streamline_decoded_logs_complete" - - - name: Decode historical logs - run: | - dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 1aeb4d5..08dfbc6 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run History Models - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run History Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" --exclude "fsc_evm,tag:receipts_by_hash" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index fc38409..eaa9635 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -1,56 +1,23 @@ -name: dbt_test_daily +name: dbt_test_daily run-name: dbt_test_daily on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Daily Testing Views - run: | - dbt run -m "fsc_evm,tag:daily_test" - - - name: Run Daily Tests - run: | - dbt test -m "fsc_evm,tag:daily_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Build Daily Testing Views + command: | + dbt run -m "fsc_evm,tag:daily_test" + command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others) + command_2: | + dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index 0996e99..4823fcd 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -5,52 +5,17 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Recent Testing Views - run: | - dbt run -m "fsc_evm,tag:recent_test" - - - name: Run Recent Tests - run: > - dbt test -m "fsc_evm,tag:recent_test" --exclude "fsc_evm.main_package.core.nft.tests.dbt_expectations_expect_row_values_to_have_recent_data_test_nft__ez_nft_transfers_recent_BLOCK_TIMESTAMP__hour__3 fsc_evm.main_package.core.nft.tests.dbt_expectations_expect_row_values_to_have_recent_data_test_nft__ez_nft_transfers_recent_INSERTED_TIMESTAMP__hour__3 fsc_evm.main_package.core.bronze.token_reads.dbt_expectations_expect_row_values_to_have_recent_data_bronze_api__token_reads__INSERTED_TIMESTAMP__day__1" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Run Observability & Recent Tests + command: | + dbt run -m "fsc_evm,tag:observability" + dbt test -m "fsc_evm,tag:recent_test" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml index 18aed0a..35db8f5 100644 --- a/.github/workflows/dbt_test_monthly.yml +++ b/.github/workflows/dbt_test_monthly.yml @@ -5,52 +5,19 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Full Testing Views - run: | - dbt run -m "fsc_evm,tag:full_test" - - - name: Run Full Tests - run: | - dbt test -m "fsc_evm,tag:full_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Build Full Testing Views + command: | + dbt run -m "fsc_evm,tag:full_test" + command_name_2: Run Full Tests + command_2: | + dbt test -m "fsc_evm,tag:full_test" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml deleted file mode 100644 index 2530fe5..0000000 --- a/.github/workflows/dbt_test_tasks.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: dbt_test_tasks -run-name: dbt_test_tasks - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit - - notify-failure: - needs: [called_workflow_template] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv deleted file mode 100644 index e10f8cb..0000000 --- a/data/github_actions__workflows.csv +++ /dev/null @@ -1,10 +0,0 @@ -workflow_name,workflow_schedule -dbt_run_dev_refresh,"29 4 * * 1" -dbt_run_streamline_chainhead,"3,33 * * * *" -dbt_run_scheduled_main,"43 * * * *" -dbt_test_intraday,"9 */4 * * *" -dbt_test_daily,"34 7 * * *" -dbt_run_scheduled_decoded_logs,"55 * * * *" -dbt_run_scheduled_abis,"33 23 * * *" -dbt_run_scheduled_decoded_logs_history,"31 22 * * 6" -dbt_run_scheduled_scores,"3 5 * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index abb2eae..4dd172e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -71,12 +71,16 @@ models: +on_schema_change: "append_new_columns" main_package: +enabled: false # disable main_package by default, enabled other packages as needed + admin: + +enabled: true core: +enabled: true # enable subpackages, as needed github_actions: +enabled: true labels: +enabled: true + observability: + +enabled: true prices: +enabled: true utils: @@ -87,6 +91,10 @@ models: +enabled: true decoded_logs: +enabled: true + curated_package: + +enabled: false + stats: + +enabled: true scores_package: +enabled: true @@ -96,11 +104,9 @@ vars: STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False UPDATE_SNOWFLAKE_TAGS: True - OBSERV_FULL_TEST: False WAIT: 0 HEAL_MODEL: False HEAL_MODELS: [] - START_GHA_TASKS: False #### STREAMLINE 2.0 BEGIN #### @@ -126,54 +132,4 @@ vars: - INTERNAL_DEV - DBT_CLOUD_BOBA # replace with the name of the chain - #### STREAMLINE 2.0 END #### - - #### FSC_EVM BEGIN #### - - GLOBAL_PROD_DB_NAME: "boba" - GLOBAL_NODE_SECRET_PATH: "Vault/prod/boba/drpc/mainnet" - GLOBAL_BLOCKS_PER_HOUR: 1800 - GLOBAL_WRAPPED_ASSET_ADDRESS: "0xdeaddeaddeaddeaddeaddeaddeaddeaddead0000" - - # Please visit https://github.com/FlipsideCrypto/fsc-evm/wiki for more information on required and optional variables - - BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True - RECEIPTS_REALTIME_NEW_BUILD: True - TRACES_REALTIME_NEW_BUILD: True - - BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True - BLOCKS_TRANSACTIONS_PRODUCER_BATCH_SIZE: 5000 - BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 1000 - BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 1000000 - - RECEIPTS_HISTORY_NEW_BUILD: True - RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 5000 - RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 1000 - RECEIPTS_HISTORY_SQL_LIMIT: 1000000 - - TRACES_HISTORY_NEW_BUILD: True - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 2000 - TRACES_HISTORY_WORKER_BATCH_SIZE: 1000 - TRACES_HISTORY_SQL_LIMIT: 1000000 - - TRACES_REQUEST_START_BLOCK: 1149019 - TRACES_FULL_RELOAD_START_BLOCK: 15000000 - - ### PRICES ### - ## REQUIRED - MAIN_PRICES_TOKEN_WETH_ENABLED: true - PRICES_NATIVE_SYMBOLS: "ETH" - PRICES_NATIVE_BLOCKCHAINS: "ethereum" - PRICES_PROVIDER_PLATFORMS: ["boba network", "boba"] - - ### DECODER_PACKAGE VARIABLES BEGIN ### - DECODER_ABIS_BLOCK_EXPLORER_NAME: "routescan" - DECODER_ABIS_BLOCK_EXPLORER_URL: "https://api.routescan.io/v2/network/mainnet/evm/288/etherscan/api?module=contract&action=getabi&address=" - DECODER_ABIS_BLOCK_EXPLORER_URL_SUFFIX: "" - DECODER_ABIS_BLOCK_EXPLORER_SECRET_PATH: "" - DECODER_ABIS_RELEVANT_CONTRACT_LIMIT: 50 - DECODER_ABIS_RELEVANT_CONTRACT_COUNT: 200 - DECODER_ABIS_BRONZE_API_TABLE_ENABLED: True - - ### DECODER_PACKAGE VARIABLES END ### -#### FSC_EVM END #### + #### STREAMLINE 2.0 END #### \ No newline at end of file diff --git a/makefile b/makefile index 041a8ec..92ccf34 100644 --- a/makefile +++ b/makefile @@ -1,53 +1,109 @@ DBT_TARGET ?= dev - -deploy_streamline_functions: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) +RECEIPTS_BY_HASH_ENABLED ?= false cleanup_time: + @set -e; \ rm -f package-lock.yml && dbt clean && dbt deps -deploy_streamline_tables: - rm -f package-lock.yml && dbt clean && dbt deps -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run -m "fsc_evm,tag:bronze_core" "fsc_evm,tag:bronze_receipts" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) -else - dbt run -m "fsc_evm,tag:bronze_core" "fsc_evm,tag:bronze_receipts" -t $(DBT_TARGET) -endif - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) +deploy_gha_workflows_table: + @set -e; \ + echo "Collecting workflow names..." ; \ + WORKFLOW_VALUES="" ; \ + for file in $$(find .github/workflows -name "*.yml" -type f); do \ + filename=$$(basename "$$file" .yml) ; \ + if [ -z "$$WORKFLOW_VALUES" ]; then \ + WORKFLOW_VALUES="('$$filename')" ; \ + else \ + WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \ + fi ; \ + done ; \ + echo "Found workflows: $$WORKFLOW_VALUES" ; \ + dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET) -deploy_streamline_requests: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_complete_receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) +deploy_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) -deploy_streamline_history: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_history_receipts" "fsc_evm,tag:streamline_core_complete_receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) +deploy_new_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) -deploy_github_actions: - dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt seed -s github_actions__workflows -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif +deploy_livequery: + @set -e; \ + dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) -deploy_new_github_action: - dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET) - dbt seed -s github_actions__workflows -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif +deploy_chain_phase_1: + @set -e; \ + dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + if [ "$(RECEIPTS_BY_HASH_ENABLED)" = "true" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + fi; \ + else \ + if [ "$(RECEIPTS_BY_HASH_ENABLED)" = "true" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_2" -deploy_phase_2: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:phase_2,tag:bronze_abis" --vars '{"BRONZE_CONTRACT_ABIS_FULL_REFRESH":"true"}' -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:phase_2" --exclude tag:bronze_abis -t $(DBT_TARGET) +deploy_chain_phase_2: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_3" -.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action deploy_streamline_history deploy_phase_2 \ No newline at end of file +deploy_chain_phase_3: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_4" + +deploy_chain_phase_4: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \ + fi; \ + +.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4 \ No newline at end of file diff --git a/models/__overview__.md b/models/__overview__.md index 3f1371c..83cf1db 100644 --- a/models/__overview__.md +++ b/models/__overview__.md @@ -19,6 +19,7 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables (boba.core) **Dimension Tables:** +- [dim_labels](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__dim_labels) - [dim_contracts](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__dim_contracts) - [dim_contract_abis](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__dim_contract_abis) @@ -28,11 +29,11 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__fact_transactions) - [fact_traces](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__fact_traces) -**EZ Tables:** +**Convenience Tables:** - [ez_decoded_event_logs](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__ez_decoded_event_logs) - [ez_native_transfers](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__ez_native_transfers) - [ez_token_transfers](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.core__ez_token_transfers) - + ### Price Tables (boba.price) - [dim_asset_metadata](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.price__dim_asset_metadata) - [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/boba-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) diff --git a/models/sources.yml b/models/sources.yml index a11ee43..79d7089 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,25 +1,22 @@ version: 2 sources: - - name: github_actions - database: "{{ target.database }}" - schema: github_actions - tables: - - name: workflows - name: bronze_streamline database: streamline - schema: >- - {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} + schema: | + {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }} tables: - name: blocks - name: transactions - - name: >- - {{ 'receipts_by_hash' if var("GLOBAL_USES_RECEIPTS_BY_HASH", False) else 'receipts' }} + - name: receipts + - name: receipts_by_hash - name: traces - name: confirm_blocks - name: decoded_logs + - name: contract_abis - name: crosschain_silver - database: "{{ 'crosschain' if target.database.upper() == var('GLOBAL_PROD_DB_NAME').upper() else 'crosschain_dev' }}" + database: >- + {{ 'CROSSCHAIN_DEV' if '_DEV' in target.database.upper() else 'CROSSCHAIN' }} schema: silver tables: - name: labels_combined @@ -29,24 +26,38 @@ sources: - name: complete_provider_prices - name: complete_token_asset_metadata - name: complete_token_prices - - name: bronze_api - database: "{{ target.database }}" - schema: bronze_api - tables: - - name: contract_abis - name: crosschain_public database: crosschain schema: bronze_public tables: - name: user_abis - - name: silver + - name: complete_streamline + database: "{{ target.database }}" + schema: streamline + tables: + - name: complete_contract_abis + - name: github_actions + database: "{{ target.database }}" + schema: github_actions + tables: + - name: workflows + - name: abis_silver database: "{{ target.database }}" schema: silver tables: - name: verified_abis - - name: bronze_abi + - name: complete_event_abis + - name: fsc_evm_admin database: >- - {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} - schema: streamline + {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }} + schema: admin tables: - - name: complete_contract_abis + - name: _master_keys + - name: rpc_node_logs + - name: logs_temp + database: "{{ target.database }}" + schema: silver + tables: + - name: logs + - name: decoded_logs + diff --git a/packages.yml b/packages.yml index 407fb51..656b7b9 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v3.30.0 + revision: v4.0.0-beta.37 \ No newline at end of file