diff --git a/.github/workflows/dbt_alter_all_gha_tasks.yml b/.github/workflows/dbt_alter_all_gha_tasks.yml new file mode 100644 index 0000000..086327d --- /dev/null +++ b/.github/workflows/dbt_alter_all_gha_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_alter_all_gha_tasks +run-name: dbt_alter_all_gha_tasks + +on: + workflow_dispatch: + branches: + - "main" + inputs: + task_action: + type: choice + description: Action to perform on all tasks + required: true + options: + - RESUME + - SUSPEND + default: RESUME + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta + with: + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..1cb60fc --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,32 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_task.yml@pre-release/v4-beta + with: + workflow_name: ${{ inputs.workflow_name }} + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_deploy_new_workflows.yml b/.github/workflows/dbt_deploy_new_workflows.yml index 7bbe6bf..a9a877d 100644 --- a/.github/workflows/dbt_deploy_new_workflows.yml +++ b/.github/workflows/dbt_deploy_new_workflows.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Deploy new workflows as Snowflake tasks - run: | - make deploy_new_github_action DBT_TARGET=prod - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Deploy New Github Actions + command: | + make deploy_new_gha_tasks DBT_TARGET=prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index e964899..10ec0ba 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -5,79 +5,10 @@ on: branches: - "main" -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: refresh ddl for datashare - run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - - - name: checkout docs branch - run: | - git checkout -B docs origin/main - - - name: generate dbt docs - run: dbt docs generate -t prod - - - name: move files to docs directory - run: | - mkdir -p ./docs - cp target/{catalog.json,manifest.json,index.html} docs/ - - - name: clean up target directory - run: dbt clean - - - name: check for changes - run: git status - - - name: stage changed files - run: git add . - - - name: commit changed files - run: | - git config user.email "abc@xyz" - git config user.name "github-actions" - git commit -am "Auto-update docs" - - - name: push changes to docs - run: | - git push -f --set-upstream origin docs - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_docs_update.yml@pre-release/v4-beta + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml index bfc2bf0..d0d5385 100644 --- a/.github/workflows/dbt_integration_test.yml +++ b/.github/workflows/dbt_integration_test.yml @@ -3,22 +3,20 @@ run-name: ${{ github.event.inputs.branch }} on: workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: string + required: true concurrency: ${{ github.workflow }} jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta with: - command: > + target: ${{ inputs.environment }} + command: | dbt test --selector 'integration_tests' - environment: ${{ github.ref == 'refs/heads/main' && 'workflow_prod' || 'workflow_dev' }} - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit - - notify-failure: - needs: [called_workflow_template] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 5ada2f7..30a3481 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -6,8 +6,8 @@ on: branches: - "main" inputs: - environment: - type: choice + target: + type: choice description: DBT Run Environment required: true options: @@ -15,59 +15,29 @@ on: - prod default: dev warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD - DBT_EMERGENCY + - DBT_MEGA default: DBT dbt_command: type: string - description: "DBT Run Command" + description: 'DBT Run Command' required: true - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.warehouse }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_${{ inputs.environment }} - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - ${{ inputs.dbt_command }} - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + warehouse: ${{ inputs.warehouse }} + target: ${{ inputs.target }} + command_name: Run DBT Command + command: ${{ inputs.dbt_command }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 7cfb934..938995c 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -3,80 +3,13 @@ run-name: dbt_run_dev_refresh on: workflow_dispatch: - schedule: - # Runs “At 1:20 on Monday.” (see https://crontab.guru) - - cron: "20 13 * * 1" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" + branches: + - "main" concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs_refresh: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run Dev Refresh - run: | - dbt run-operation fsc_evm.run_sp_create_prod_clone - - notify-failure: - needs: [run_dbt_jobs_refresh] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - - run_dbt_jobs_udfs: - runs-on: ubuntu-latest - needs: run_dbt_jobs_refresh - environment: - name: workflow_dev - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run Recreate UDFs - run: | - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - - notify-failure2: - needs: [run_dbt_jobs_udfs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_dev_refresh.yml@pre-release/v4-beta + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index 375dc54..379ffd0 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -5,47 +5,18 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_2xl + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Observability Models + command: | + dbt run --threads 2 --vars '{"MAIN_OBSERV_FULL_TEST_ENABLED":True}' -m "fsc_evm,tag:observability" + secrets: inherit + - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml new file mode 100644 index 0000000..6eaf50f --- /dev/null +++ b/.github/workflows/dbt_run_heal_models.yml @@ -0,0 +1,19 @@ +name: dbt_run_heal_models +run-name: dbt_run_heal_models + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Heal Models + command: | + dbt run -m "$PROJECT_NAME,tag:heal" --vars '{"HEAL_MODEL":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml index 6318626..2eb8581 100644 --- a/.github/workflows/dbt_run_scheduled_abis.yml +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -6,51 +6,17 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update ABI models - run: | - dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" - - - name: Kick off decoded logs history, if there are new ABIs from users - run: | - dbt run-operation fsc_evm.run_decoded_logs_history - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run ABI Models + command: | + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:gold,tag:abis" + command_name_2: Kick off decoded logs history, if there are new user submitted ABIs + command_2: | + dbt run-operation fsc_evm.run_decoded_logs_history + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml new file mode 100644 index 0000000..61d9d16 --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -0,0 +1,19 @@ +name: dbt_run_scheduled_curated +run-name: dbt_run_scheduled_curated + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Curated Models + command: | + dbt run -m "$PROJECT_NAME,tag:curated" "fsc_evm,tag:curated" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_decoded_logs.yml b/.github/workflows/dbt_run_scheduled_decoded_logs.yml deleted file mode 100644 index a6f8792..0000000 --- a/.github/workflows/dbt_run_scheduled_decoded_logs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: dbt_run_scheduled_decoded_logs -run-name: dbt_run_scheduled_decoded_logs - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update the silver and gold tables with the latest decoded logs - run: | - dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_scheduled_decoder.yml b/.github/workflows/dbt_run_scheduled_decoder.yml new file mode 100644 index 0000000..a0b7a2b --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_decoder.yml @@ -0,0 +1,19 @@ +name: dbt_run_scheduled_decoder +run-name: dbt_run_scheduled_decoder + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Decoder Models + command: | + dbt run -m "fsc_evm,tag:bronze,tag:decoded_logs" "fsc_evm,tag:silver,tag:decoded_logs" "fsc_evm,tag:gold,tag:decoded_logs" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 7a227f6..56cca41 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -6,55 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update main models - run: | - dbt run -m "fsc_evm,tag:bronze_core" "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" "fsc_evm,tag:silver_confirm_blocks" "fsc_evm,tag:nft_core" - - - name: Send new logs for decoding - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" - - - name: Get new contract ABIs - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_abis" "fsc_evm,tag:streamline_abis_realtime" "fsc_evm,tag:streamline_abis_complete" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Main Models + command: | + dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --exclude "fsc_evm,tag:receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_scores.yml b/.github/workflows/dbt_run_scheduled_scores.yml index 8944fbc..5c2f0f9 100644 --- a/.github/workflows/dbt_run_scheduled_scores.yml +++ b/.github/workflows/dbt_run_scheduled_scores.yml @@ -1,3 +1,4 @@ + name: dbt_run_scheduled_scores run-name: dbt_run_scheduled_scores @@ -6,47 +7,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update scores - run: | - dbt run -m "fsc_evm,tag:scores" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Scores Models + command: | + dbt run -m "fsc_evm,tag:scores" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 4e09b7b..d8745f3 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -6,51 +6,15 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Get new block data - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_realtime_confirm_blocks" - - - name: Test chainhead - run: | - dbt test -m "fsc_evm,tag:chainhead" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Chainhead Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" + dbt test -m "fsc_evm,tag:chainhead" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoded_logs_history.yml b/.github/workflows/dbt_run_streamline_decoded_logs_history.yml deleted file mode 100644 index e8293ce..0000000 --- a/.github/workflows/dbt_run_streamline_decoded_logs_history.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: dbt_run_streamline_decoded_logs_history -run-name: dbt_run_streamline_decoded_logs_history - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update complete table - run: | - dbt run -m "fsc_evm,tag:streamline_decoded_logs_complete" - - - name: Decode historical logs - run: | - dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml new file mode 100644 index 0000000..b736c9c --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history.yml @@ -0,0 +1,22 @@ +name: dbt_run_streamline_decoder_history +run-name: dbt_run_streamline_decoder_history + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Decoder Complete + command: | + dbt run -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" + command_name_2: Run Streamline Decoder History + command_2: | + dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 6d566e0..8e36769 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Get historical data - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_history_confirm_blocks" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run History Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" --exclude "fsc_evm,tag:receipts" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index acc2c9f..eaa9635 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -1,56 +1,23 @@ -name: dbt_test_daily +name: dbt_test_daily run-name: dbt_test_daily on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build daily testing views - run: | - dbt run -m "fsc_evm,tag:daily_test" - - - name: Run daily tests - run: | - dbt test -m "fsc_evm,tag:daily_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Build Daily Testing Views + command: | + dbt run -m "fsc_evm,tag:daily_test" + command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others) + command_2: | + dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index 069e49c..4823fcd 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -5,53 +5,17 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build testing views - run: | - dbt run -m "fsc_evm,tag:recent_test" - - - name: Run intraday tests - run: | - dbt run -m "fsc_evm,tag:observability" - dbt test -m "fsc_evm,tag:recent_test" "fsc_evm,tag:recent_test_confirm_blocks" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Run Observability & Recent Tests + command: | + dbt run -m "fsc_evm,tag:observability" + dbt test -m "fsc_evm,tag:recent_test" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml index 23d33a4..35db8f5 100644 --- a/.github/workflows/dbt_test_monthly.yml +++ b/.github/workflows/dbt_test_monthly.yml @@ -5,52 +5,19 @@ on: workflow_dispatch: branches: - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - + concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build full testing views - run: | - dbt run -m "fsc_evm,tag:full_test" - - - name: Test all data - run: | - dbt test -m "fsc_evm,tag:full_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Build Full Testing Views + command: | + dbt run -m "fsc_evm,tag:full_test" + command_name_2: Run Full Tests + command_2: | + dbt test -m "fsc_evm,tag:full_test" + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv deleted file mode 100644 index 07981be..0000000 --- a/data/github_actions__workflows.csv +++ /dev/null @@ -1,11 +0,0 @@ -workflow_name,workflow_schedule -dbt_run_streamline_chainhead,"23,53 * * * *" -dbt_run_scheduled_main,"1,31 * * * *" -dbt_run_scheduled_decoded_logs,"8,38 * * * *" -dbt_test_intraday,"39 */4 * * *" -dbt_test_daily,"22 8 * * *" -dbt_test_monthly,"5 7 28 * *" -dbt_run_scheduled_abis,"11 23 * * *" -dbt_run_streamline_decoded_logs_history,"5 22 * * 6" -dbt_run_full_observability,"0 19 1 * *" -dbt_run_scheduled_scores,"1 5 * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 5cad1ee..1fe50f3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -70,9 +70,11 @@ models: columns: true +on_schema_change: "append_new_columns" main_package: - +enabled: false - core: + +enabled: false # disable main_package by default, enabled other packages as needed + admin: +enabled: true + core: + +enabled: true # enable subpackages, as needed github_actions: +enabled: true labels: @@ -89,6 +91,10 @@ models: +enabled: true decoded_logs: +enabled: true + curated_package: + +enabled: false + stats: + +enabled: true scores_package: +enabled: true @@ -102,7 +108,6 @@ vars: WAIT: 0 HEAL_MODEL: False HEAL_MODELS: [] - START_GHA_TASKS: False #### STREAMLINE 2.0 BEGIN #### @@ -129,73 +134,3 @@ vars: - DBT_CLOUD_CORE #### STREAMLINE 2.0 END #### - - #### FSC_EVM BEGIN #### - - ### GLOBAL VARIABLES BEGIN ### - ## REQUIRED - GLOBAL_PROD_DB_NAME: "core" - GLOBAL_NODE_SECRET_PATH: "Vault/prod/core/drpc/mainnet" - GLOBAL_BLOCKS_PER_HOUR: 1200 - GLOBAL_WRAPPED_ASSET_ADDRESS: "0x40375c92d9faf44d2f9db9bd9ba41a3317a2404f" - - ## OPTIONAL - GLOBAL_USES_RECEIPTS_BY_HASH: True - GLOBAL_AVG_TXS_PER_BLOCK: 50 - - ### GLOBAL VARIABLES END ### - - ### MAIN_PACKAGE VARIABLES BEGIN ### - - ### CORE ### - ## REQUIRED - TRACES_FULL_RELOAD_START_BLOCK: 20000000 - - ## OPTIONAL - # GOLD_FULL_REFRESH: True - # SILVER_FULL_REFRESH: True - - ### PRICES ### - ## REQUIRED - PRICES_NATIVE_SYMBOLS: "CORE" - PRICES_PROVIDER_PLATFORMS: ["Core", "core"] - - ### LABELS ### - - ### OBSERVABILITY ### - OBSERV_USES_EXCLUSION_LIST_BLOCKS: False - OBSERV_USES_EXCLUSION_LIST_TRANSACTIONS: False - OBSERV_USES_EXCLUSION_LIST_RECEIPTS: False - OBSERV_USES_EXCLUSION_LIST_LOGS: False - OBSERV_USES_EXCLUSION_LIST_TRACES: False - - ### MAIN_PACKAGE VARIABLES END ### - - ### DECODER_PACKAGE VARIABLES BEGIN ### - - ## REQUIRED - - DECODER_ABIS_BLOCK_EXPLORER_NAME: "CoreScan" - DECODER_ABIS_BLOCK_EXPLORER_URL: "https://openapi.coredao.org/api?module=contract&action=getabi&address=" - DECODER_ABIS_BLOCK_EXPLORER_URL_SUFFIX: "" - DECODER_ABIS_BLOCK_EXPLORER_SECRET_PATH: "Vault/prod/block_explorers/core_scan" - DECODER_ABIS_RELEVANT_CONTRACT_LIMIT: 50 - DECODER_ABIS_RELEVANT_CONTRACT_COUNT: 200 - DECODER_ABIS_BRONZE_API_TABLE_ENABLED: True - - ### DECODER_PACKAGE VARIABLES END ### - - ### CURATED_PACKAGE VARIABLES BEGIN ### - - ### STATS ### - # STATS_TOKEN_ADDRESS: '' ### REQUIRED - - ### CURATED_PACKAGE VARIABLES END ### - - ### SCORES_PACKAGE VARIABLES BEGIN ### - - ### SCORES_FULL_RELOAD_MODE: True - - ### SCORES_PACKAGE VARIABLES END ### -#### FSC_EVM END #### - diff --git a/makefile b/makefile index 99710c1..6b21b7d 100644 --- a/makefile +++ b/makefile @@ -1,44 +1,109 @@ DBT_TARGET ?= dev -deploy_streamline_functions: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - cleanup_time: + @set -e; \ rm -f package-lock.yml && dbt clean && dbt deps -deploy_streamline_tables: - rm -f package-lock.yml && dbt clean && dbt deps -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) -else - dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET) -endif - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) +deploy_gha_workflows_table: + @set -e; \ + echo "Collecting workflow names..." ; \ + WORKFLOW_VALUES="" ; \ + for file in $$(find .github/workflows -name "*.yml" -type f); do \ + filename=$$(basename "$$file" .yml) ; \ + if [ -z "$$WORKFLOW_VALUES" ]; then \ + WORKFLOW_VALUES="('$$filename')" ; \ + else \ + WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \ + fi ; \ + done ; \ + echo "Found workflows: $$WORKFLOW_VALUES" ; \ + dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET) -deploy_streamline_requests: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) +deploy_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) -deploy_github_actions: - dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt seed -s github_actions__workflows -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif +deploy_new_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) -deploy_new_github_action: - dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET) - dbt seed -s github_actions__workflows -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif +deploy_livequery: + @set -e; \ + dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) -.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action \ No newline at end of file +deploy_chain_phase_1: + @set -e; \ + read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \ + dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + if [ "$$receipts_by_hash" = "n" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + fi; \ + else \ + if [ "$$receipts_by_hash" = "n" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_2" + +deploy_chain_phase_2: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_3" + +deploy_chain_phase_3: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_4" + +deploy_chain_phase_4: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + fi; \ + +.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4 \ No newline at end of file diff --git a/models/observability/silver_observability__exclusion_list.sql b/models/observability/silver_observability__exclusion_list.sql deleted file mode 100644 index 55fbada..0000000 --- a/models/observability/silver_observability__exclusion_list.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['observability'] -) }} - -SELECT - column1 AS block_number -FROM - ( - VALUES - (0), - (1) - ) AS block_number(column1) diff --git a/models/sources.yml b/models/sources.yml index a11ee43..18ae91c 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,25 +1,22 @@ version: 2 sources: - - name: github_actions - database: "{{ target.database }}" - schema: github_actions - tables: - - name: workflows - name: bronze_streamline database: streamline - schema: >- - {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} + schema: | + {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }} tables: - name: blocks - name: transactions - - name: >- - {{ 'receipts_by_hash' if var("GLOBAL_USES_RECEIPTS_BY_HASH", False) else 'receipts' }} + - name: receipts + - name: receipts_by_hash - name: traces - name: confirm_blocks - name: decoded_logs + - name: contract_abis - name: crosschain_silver - database: "{{ 'crosschain' if target.database.upper() == var('GLOBAL_PROD_DB_NAME').upper() else 'crosschain_dev' }}" + database: >- + {{ 'CROSSCHAIN_DEV' if '_DEV' in target.database.upper() else 'CROSSCHAIN' }} schema: silver tables: - name: labels_combined @@ -29,24 +26,37 @@ sources: - name: complete_provider_prices - name: complete_token_asset_metadata - name: complete_token_prices - - name: bronze_api - database: "{{ target.database }}" - schema: bronze_api - tables: - - name: contract_abis - name: crosschain_public database: crosschain schema: bronze_public tables: - name: user_abis - - name: silver + - name: complete_streamline + database: "{{ target.database }}" + schema: streamline + tables: + - name: complete_contract_abis + - name: github_actions + database: "{{ target.database }}" + schema: github_actions + tables: + - name: workflows + - name: abis_silver database: "{{ target.database }}" schema: silver tables: - name: verified_abis - - name: bronze_abi + - name: complete_event_abis + - name: fsc_evm_admin database: >- - {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} - schema: streamline + {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }} + schema: admin tables: - - name: complete_contract_abis + - name: _master_keys + - name: rpc_node_logs + - name: logs_temp + database: "{{ target.database }}" + schema: silver + tables: + - name: logs + - name: decoded_logs diff --git a/packages.yml b/packages.yml index 407fb51..3095fcd 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v3.30.0 + revision: "updates-for-migrations2"