From 69ca81f33f8915ec47afbb402debf7b979a12927 Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 1 May 2025 09:36:57 -0600 Subject: [PATCH] AN-5988/ronin-migration --- .github/workflows/dbt_alter_all_gha_tasks.yml | 27 ++++ .github/workflows/dbt_alter_gha_tasks.yml | 32 +++++ .../workflows/dbt_deploy_new_workflows.yml | 47 +----- .github/workflows/dbt_docs_update.yml | 75 +--------- .github/workflows/dbt_integration_test.yml | 33 +---- .github/workflows/dbt_run_adhoc.yml | 50 ++----- .github/workflows/dbt_run_dev_refresh.yml | 73 +--------- .../workflows/dbt_run_full_observability.yml | 48 ++----- .github/workflows/dbt_run_heal_models.yml | 19 +++ .github/workflows/dbt_run_scheduled_abis.yml | 54 ++----- .../workflows/dbt_run_scheduled_curated.yml | 19 +++ .../dbt_run_scheduled_decoded_logs.yml | 52 ------- .../workflows/dbt_run_scheduled_decoder.yml | 19 +++ .github/workflows/dbt_run_scheduled_main.yml | 55 +------ .../workflows/dbt_run_scheduled_scores.yml | 47 ++---- .../dbt_run_streamline_chainhead.yml | 52 ++----- ...bt_run_streamline_decoded_logs_history.yml | 56 -------- .../dbt_run_streamline_decoder_history.yml | 22 +++ .../workflows/dbt_run_streamline_history.yml | 47 +----- .github/workflows/dbt_test_daily.yml | 55 ++----- .github/workflows/dbt_test_intraday.yml | 53 ++----- .github/workflows/dbt_test_monthly.yml | 57 ++------ .github/workflows/dbt_test_tasks.yml | 34 ----- data/github_actions__workflows.csv | 10 -- dbt_project.yml | 105 ++------------ makefile | 135 +++++++++++++----- models/__overview__.md | 13 +- models/sources.yml | 50 ++++--- packages.yml | 2 +- 29 files changed, 400 insertions(+), 941 deletions(-) create mode 100644 .github/workflows/dbt_alter_all_gha_tasks.yml create mode 100644 .github/workflows/dbt_alter_gha_tasks.yml create mode 100644 .github/workflows/dbt_run_heal_models.yml create mode 100644 .github/workflows/dbt_run_scheduled_curated.yml delete mode 100644 .github/workflows/dbt_run_scheduled_decoded_logs.yml create mode 100644 .github/workflows/dbt_run_scheduled_decoder.yml delete mode 100644 .github/workflows/dbt_run_streamline_decoded_logs_history.yml create mode 100644 .github/workflows/dbt_run_streamline_decoder_history.yml delete mode 100644 .github/workflows/dbt_test_tasks.yml delete mode 100644 data/github_actions__workflows.csv diff --git a/.github/workflows/dbt_alter_all_gha_tasks.yml b/.github/workflows/dbt_alter_all_gha_tasks.yml new file mode 100644 index 0000000..086327d --- /dev/null +++ b/.github/workflows/dbt_alter_all_gha_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_alter_all_gha_tasks +run-name: dbt_alter_all_gha_tasks + +on: + workflow_dispatch: + branches: + - "main" + inputs: + task_action: + type: choice + description: Action to perform on all tasks + required: true + options: + - RESUME + - SUSPEND + default: RESUME + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_all_gha_tasks.yml@pre-release/v4-beta + with: + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_alter_gha_tasks.yml b/.github/workflows/dbt_alter_gha_tasks.yml new file mode 100644 index 0000000..be92ee5 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_tasks.yml @@ -0,0 +1,32 @@ +name: dbt_alter_gha_tasks +run-name: dbt_alter_gha_tasks + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_alter_gha_tasks.yml@pre-release/v4-beta + with: + workflow_name: ${{ inputs.workflow_name }} + task_action: ${{ inputs.task_action }} + target: prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_deploy_new_workflows.yml b/.github/workflows/dbt_deploy_new_workflows.yml index 6b6cd38..a9a877d 100644 --- a/.github/workflows/dbt_deploy_new_workflows.yml +++ b/.github/workflows/dbt_deploy_new_workflows.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Deploy New Github Actions - run: | - make deploy_new_github_action DBT_TARGET=prod - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Deploy New Github Actions + command: | + make deploy_new_gha_tasks DBT_TARGET=prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 7d4e3fd..10ec0ba 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -5,79 +5,10 @@ on: branches: - "main" -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: refresh ddl for datashare - run: | - cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; - - - name: checkout docs branch - run: | - git checkout -B docs origin/main - - - name: generate dbt docs - run: dbt docs generate -t prod - - - name: move files to docs directory - run: | - mkdir -p ./docs - cp target/{catalog.json,manifest.json,index.html} docs/ - - - name: clean up target directory - run: dbt clean - - - name: check for changes - run: git status - - - name: stage changed files - run: git add . - - - name: commit changed files - run: | - git config user.email "abc@xyz" - git config user.name "github-actions" - git commit -am "Auto-update docs" - - - name: push changes to docs - run: | - git push -f --set-upstream origin docs - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_docs_update.yml@pre-release/v4-beta + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml index 8d191e5..d0d5385 100644 --- a/.github/workflows/dbt_integration_test.yml +++ b/.github/workflows/dbt_integration_test.yml @@ -3,39 +3,20 @@ run-name: ${{ github.event.inputs.branch }} on: workflow_dispatch: + branches: + - "main" inputs: environment: - required: true type: string + required: true concurrency: ${{ github.workflow }} jobs: - prepare_vars: - runs-on: ubuntu-latest - environment: - name: ${{ inputs.environment }} - outputs: - warehouse: ${{ steps.set_outputs.outputs.warehouse }} - steps: - - name: Set warehouse output - id: set_outputs - run: | - echo "warehouse=${{ vars.WAREHOUSE }}" >> $GITHUB_OUTPUT - called_workflow_template: - needs: prepare_vars - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt.yml@main + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_integration_test.yml@pre-release/v4-beta with: - command: > + target: ${{ inputs.environment }} + command: | dbt test --selector 'integration_tests' - environment: ${{ inputs.environment }} - warehouse: ${{ needs.prepare_vars.outputs.warehouse }} - secrets: inherit - - notify-failure: - needs: [called_workflow_template] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 547eb05..30a3481 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -6,7 +6,7 @@ on: branches: - "main" inputs: - environment: + target: type: choice description: DBT Run Environment required: true @@ -22,52 +22,22 @@ on: - DBT - DBT_CLOUD - DBT_EMERGENCY + - DBT_MEGA default: DBT dbt_command: type: string description: 'DBT Run Command' required: true -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ inputs.warehouse }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_${{ inputs.environment }} - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - ${{ inputs.dbt_command }} - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + warehouse: ${{ inputs.warehouse }} + target: ${{ inputs.target }} + command_name: Run DBT Command + command: ${{ inputs.dbt_command }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index ed3b791..938995c 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -6,77 +6,10 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs_refresh: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run Dev Refresh - run: | - dbt run-operation fsc_evm.run_sp_create_prod_clone - - notify-failure: - needs: [run_dbt_jobs_refresh] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - - run_dbt_jobs_udfs: - runs-on: ubuntu-latest - needs: run_dbt_jobs_refresh - environment: - name: workflow_dev - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Recreate UDFs - run: | - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev - - notify-failure2: - needs: [run_dbt_jobs_udfs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template_dev_refresh.yml@pre-release/v4-beta + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index e17d401..379ffd0 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -6,47 +6,17 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_2xl - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Observability Models - run: | - dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability" + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Observability Models + command: | + dbt run --threads 2 --vars '{"MAIN_OBSERV_FULL_TEST_ENABLED":True}' -m "fsc_evm,tag:observability" + secrets: inherit - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + + diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml new file mode 100644 index 0000000..6eaf50f --- /dev/null +++ b/.github/workflows/dbt_run_heal_models.yml @@ -0,0 +1,19 @@ +name: dbt_run_heal_models +run-name: dbt_run_heal_models + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Heal Models + command: | + dbt run -m "$PROJECT_NAME,tag:heal" --vars '{"HEAL_MODEL":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml index f833bfe..2eb8581 100644 --- a/.github/workflows/dbt_run_scheduled_abis.yml +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -6,51 +6,17 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update ABI Models - run: | - dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" - - - name: Kick off decoded logs history, if there are new ABIs from users - run: | - dbt run-operation fsc_evm.run_decoded_logs_history - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run ABI Models + command: | + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:gold,tag:abis" + command_name_2: Kick off decoded logs history, if there are new user submitted ABIs + command_2: | + dbt run-operation fsc_evm.run_decoded_logs_history + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml new file mode 100644 index 0000000..61d9d16 --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -0,0 +1,19 @@ +name: dbt_run_scheduled_curated +run-name: dbt_run_scheduled_curated + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Curated Models + command: | + dbt run -m "$PROJECT_NAME,tag:curated" "fsc_evm,tag:curated" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_decoded_logs.yml b/.github/workflows/dbt_run_scheduled_decoded_logs.yml deleted file mode 100644 index 1eb8465..0000000 --- a/.github/workflows/dbt_run_scheduled_decoded_logs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: dbt_run_scheduled_decoded_logs -run-name: dbt_run_scheduled_decoded_logs - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update the silver and gold tables with the latest decoded logs - run: | - dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_decoder.yml b/.github/workflows/dbt_run_scheduled_decoder.yml new file mode 100644 index 0000000..a0b7a2b --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_decoder.yml @@ -0,0 +1,19 @@ +name: dbt_run_scheduled_decoder +run-name: dbt_run_scheduled_decoder + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Decoder Models + command: | + dbt run -m "fsc_evm,tag:bronze,tag:decoded_logs" "fsc_evm,tag:silver,tag:decoded_logs" "fsc_evm,tag:gold,tag:decoded_logs" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index 04ef203..56cca41 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -6,55 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Main Models - run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_confirm_blocks" "fsc_evm,tag:nft_core" "fsc_evm,tag:bronze_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" - - - name: Send new logs for decoding - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" - - - name: Get new contract ABIs - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_abis" "fsc_evm,tag:streamline_abis_realtime" "fsc_evm,tag:streamline_abis_complete" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Main Models + command: | + dbt run -m "fsc_evm,tag:bronze,tag:core" "fsc_evm,tag:silver,tag:core" "fsc_evm,tag:gold,tag:core" "fsc_evm,tag:silver,tag:prices" "fsc_evm,tag:gold,tag:prices" "fsc_evm,tag:silver,tag:labels" "fsc_evm,tag:gold,tag:labels" "fsc_evm,tag:gold,tag:nft" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --exclude "fsc_evm,tag:receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_scores.yml b/.github/workflows/dbt_run_scheduled_scores.yml index 844c122..5c2f0f9 100644 --- a/.github/workflows/dbt_run_scheduled_scores.yml +++ b/.github/workflows/dbt_run_scheduled_scores.yml @@ -1,3 +1,4 @@ + name: dbt_run_scheduled_scores run-name: dbt_run_scheduled_scores @@ -6,46 +7,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Update scores - run: | - dbt run -m "fsc_evm,tag:scores" --vars '{"INCLUDE_GAMING_METRICS": true}' - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Scores Models + command: | + dbt run -m "fsc_evm,tag:scores" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 6243b08..d8745f3 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -6,51 +6,15 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run Chainhead Models - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_realtime_confirm_blocks" - - - name: Run Chainhead Tests - run: | - dbt test -m "fsc_evm,tag:chainhead" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Chainhead Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:chainhead" "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" + dbt test -m "fsc_evm,tag:chainhead" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoded_logs_history.yml b/.github/workflows/dbt_run_streamline_decoded_logs_history.yml deleted file mode 100644 index eadf8c1..0000000 --- a/.github/workflows/dbt_run_streamline_decoded_logs_history.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: dbt_run_streamline_decoded_logs_history -run-name: dbt_run_streamline_decoded_logs_history - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Update complete table - run: | - dbt run -m "fsc_evm,tag:streamline_decoded_logs_complete" - - - name: Decode historical logs - run: | - dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml new file mode 100644 index 0000000..b736c9c --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history.yml @@ -0,0 +1,22 @@ +name: dbt_run_streamline_decoder_history +run-name: dbt_run_streamline_decoder_history + +on: + workflow_dispatch: + branches: + - "main" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run Decoder Complete + command: | + dbt run -m "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" + command_name_2: Run Streamline Decoder History + command_2: | + dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 625234f..8e36769 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -6,47 +6,14 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run History Models - run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_history_receipts_by_hash" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_history_confirm_blocks" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + command_name: Run History Models + command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:history" --exclude "fsc_evm,tag:receipts" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index d79b1e5..eaa9635 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -6,51 +6,18 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Daily Testing Views - run: | - dbt run -m "fsc_evm,tag:daily_test" - - - name: Run Daily Tests - run: | - dbt test -m "fsc_evm,tag:daily_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Build Daily Testing Views + command: | + dbt run -m "fsc_evm,tag:daily_test" + command_name_2: Run Daily Tests (all tests excluding full, recent and misc. others) + command_2: | + dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index e3d1a56..4823fcd 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -6,51 +6,16 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Recent Testing Views - run: | - dbt run -m "fsc_evm,tag:recent_test" - - - name: Run Recent Tests - run: | - dbt test -m "fsc_evm,tag:recent_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Run Observability & Recent Tests + command: | + dbt run -m "fsc_evm,tag:observability" + dbt test -m "fsc_evm,tag:recent_test" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml index 2761cef..35db8f5 100644 --- a/.github/workflows/dbt_test_monthly.yml +++ b/.github/workflows/dbt_test_monthly.yml @@ -6,51 +6,18 @@ on: branches: - "main" -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} - + jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_test - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Build Full Testing Views - run: | - dbt run -m "fsc_evm,tag:full_test" - - - name: Run Full Tests - run: | - dbt test -m "fsc_evm,tag:full_test" - - notify-failure: - needs: [run_dbt_jobs] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + called_workflow_template: + uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta + with: + target: test + command_name: Build Full Testing Views + command: | + dbt run -m "fsc_evm,tag:full_test" + command_name_2: Run Full Tests + command_2: | + dbt test -m "fsc_evm,tag:full_test" + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml deleted file mode 100644 index 04e910c..0000000 --- a/.github/workflows/dbt_test_tasks.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: dbt_test_tasks -run-name: dbt_test_tasks - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 - secrets: inherit - - notify-failure: - needs: [called_workflow_template] - if: failure() - uses: FlipsideCrypto/fsc-evm/.github/workflows/slack_notify.yml@main - secrets: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv deleted file mode 100644 index bd86b16..0000000 --- a/data/github_actions__workflows.csv +++ /dev/null @@ -1,10 +0,0 @@ -workflow_name,workflow_schedule -dbt_run_streamline_chainhead,"19,49 * * * *" -dbt_run_dev_refresh,"33 3 * * 1" -dbt_run_scheduled_main,"29 * * * *" -dbt_run_scheduled_decoded_logs,"42 * * * *" -dbt_test_intraday,"9 */4 * * *" -dbt_test_daily,"34 7 * * *" -dbt_run_scheduled_abis,"33 23 * * *" -dbt_run_scheduled_decoded_logs_history,"31 22 * * 6" -dbt_run_scheduled_scores,"15 5 * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 94f07c7..6ec9ee0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -70,23 +70,31 @@ models: columns: true +on_schema_change: "append_new_columns" main_package: - +enabled: false - core: + +enabled: false # disable main_package by default, enabled other packages as needed + admin: +enabled: true + core: + +enabled: true # enable subpackages, as needed github_actions: +enabled: true + labels: + +enabled: true + observability: + +enabled: true prices: +enabled: true utils: +enabled: true - labels: - +enabled: true decoder_package: +enabled: false abis: +enabled: true decoded_logs: +enabled: true + curated_package: + +enabled: false + stats: + +enabled: true scores_package: +enabled: true @@ -96,11 +104,9 @@ vars: STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False UPDATE_SNOWFLAKE_TAGS: True - OBSERV_FULL_TEST: False - WAIT: 0 HEAL_MODEL: False HEAL_MODELS: [] - START_GHA_TASKS: False + MAIN_CORE_RECEIPTS_BY_HASH_ENABLED: True #### STREAMLINE 2.0 BEGIN #### @@ -127,88 +133,3 @@ vars: - DBT_CLOUD_RONIN # replace with the name of the chain #### STREAMLINE 2.0 END #### - - #### FSC_EVM BEGIN #### - - # Please visit https://github.com/FlipsideCrypto/fsc-evm/wiki for more information on required and optional variables - - ### GLOBAL VARIABLES BEGIN ### - - ## REQUIRED - GLOBAL_PROD_DB_NAME: "ronin" - GLOBAL_NODE_URL: "{api_url}" - GLOBAL_NODE_SECRET_PATH: "Vault/prod/ronin/tatum/mainnet_new" - GLOBAL_BLOCKS_PER_HOUR: 1200 - GLOBAL_WRAPPED_ASSET_ADDRESS: "0xe514d9deb7966c8be0ca922de8a064264ea6bcd4" #WRON - - ## OPTIONAL - GLOBAL_USES_RECEIPTS_BY_HASH: True - GLOBAL_AVG_TXS_PER_BLOCK: 45 - GLOBAL_START_UP_BLOCK: 38633701 #October 1st, 2024 - GLOBAL_CHAINHEAD_DELAY: 6 - - ### GLOBAL VARIABLES END ### - - ### MAIN_PACKAGE VARIABLES BEGIN ### - - ### CORE ### - ## REQUIRED - - ## OPTIONAL - TRACES_FULL_RELOAD_START_BLOCK: 38633701 - - # GOLD_FULL_REFRESH: True #comment out after FR is complete / for prod - # SILVER_FULL_REFRESH: True #comment out after FR is complete / for prod - # BRONZE_FULL_REFRESH: True #comment out after FR is complete / for prod - - # BLOCKS_COMPLETE_FULL_REFRESH: True - # CONFIRM_BLOCKS_COMPLETE_FULL_REFRESH: True - # TRACES_COMPLETE_FULL_REFRESH: True - # RECEIPTS_BY_HASH_COMPLETE_FULL_REFRESH: True - # TRANSACTIONS_COMPLETE_FULL_REFRESH: True - - BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 1000000 - BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 3000 - BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 1000 - - RECEIPTS_BY_HASH_HISTORY_ASYNC_CONCURRENT_REQUESTS: 20 - RECEIPTS_BY_HASH_HISTORY_SQL_LIMIT: 1800000 - RECEIPTS_BY_HASH_HISTORY_PRODUCER_BATCH_SIZE: 10500 - RECEIPTS_BY_HASH_HISTORY_WORKER_BATCH_SIZE: 1500 - RECEIPTS_BY_HASH_HISTORY_ORDER_BY_CLAUSE: "ORDER BY partition_key DESC, block_number DESC" - - TRACES_HISTORY_ASYNC_CONCURRENT_REQUESTS: 1 - TRACES_HISTORY_SQL_LIMIT: 10000 - TRACES_HISTORY_PRODUCER_BATCH_SIZE: 1 - TRACES_HISTORY_WORKER_BATCH_SIZE: 1 - - CONFIRM_BLOCKS_HISTORY_SQL_LIMIT: 1000000 - CONFIRM_BLOCKS_HISTORY_PRODUCER_BATCH_SIZE: 3000 - CONFIRM_BLOCKS_HISTORY_WORKER_BATCH_SIZE: 1000 - - ### PRICES ### - ## REQUIRED - PRICES_NATIVE_SYMBOLS: "RON" - PRICES_PROVIDER_PLATFORMS: ["ronin"] - - ### LABELS ### - - ### MAIN_PACKAGE VARIABLES END ### - - ### DECODER_PACKAGE VARIABLES BEGIN ### - - ## ABIS - - DECODER_ABIS_BLOCK_EXPLORER_NAME: "RoninChain" - DECODER_ABIS_BLOCK_EXPLORER_URL: "https://explorer-kintsugi.roninchain.com/v2/2020/contract/" - DECODER_ABIS_BLOCK_EXPLORER_URL_SUFFIX: "/abi" - DECODER_ABIS_BLOCK_EXPLORER_SECRET_PATH: "" - DECODER_ABIS_RELEVANT_CONTRACT_LIMIT: 50 - DECODER_ABIS_RELEVANT_CONTRACT_COUNT: 200 - DECODER_ABIS_BRONZE_API_TABLE_ENABLED: True # this is for all existing non eth repos with a bronze_api__contract_abis model - - - # BRONZE_CONTRACT_ABIS_FULL_REFRESH: True #comment out after FR is complete / for prod - - ### DECODER_PACKAGE VARIABLES END ### -#### FSC_EVM END #### diff --git a/makefile b/makefile index 99710c1..92ccf34 100644 --- a/makefile +++ b/makefile @@ -1,44 +1,109 @@ DBT_TARGET ?= dev - -deploy_streamline_functions: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) +RECEIPTS_BY_HASH_ENABLED ?= false cleanup_time: + @set -e; \ rm -f package-lock.yml && dbt clean && dbt deps -deploy_streamline_tables: - rm -f package-lock.yml && dbt clean && dbt deps -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) -else - dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET) -endif - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) +deploy_gha_workflows_table: + @set -e; \ + echo "Collecting workflow names..." ; \ + WORKFLOW_VALUES="" ; \ + for file in $$(find .github/workflows -name "*.yml" -type f); do \ + filename=$$(basename "$$file" .yml) ; \ + if [ -z "$$WORKFLOW_VALUES" ]; then \ + WORKFLOW_VALUES="('$$filename')" ; \ + else \ + WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \ + fi ; \ + done ; \ + echo "Found workflows: $$WORKFLOW_VALUES" ; \ + dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET) -deploy_streamline_requests: - rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) +deploy_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) -deploy_github_actions: - dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt seed -s github_actions__workflows -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif +deploy_new_gha_tasks: + @set -e; \ + make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET) -deploy_new_github_action: - dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET) - dbt seed -s github_actions__workflows -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) -ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) -else - dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) -endif +deploy_livequery: + @set -e; \ + dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) -.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action \ No newline at end of file +deploy_chain_phase_1: + @set -e; \ + dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + if [ "$(RECEIPTS_BY_HASH_ENABLED)" = "true" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + fi; \ + else \ + if [ "$(RECEIPTS_BY_HASH_ENABLED)" = "true" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_2" + +deploy_chain_phase_2: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_3" + +deploy_chain_phase_3: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_4" + +deploy_chain_phase_4: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \ + fi; \ + +.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4 \ No newline at end of file diff --git a/models/__overview__.md b/models/__overview__.md index 705679c..6f41e4e 100644 --- a/models/__overview__.md +++ b/models/__overview__.md @@ -19,9 +19,9 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables (ronin.core) **Dimension Tables:** +- [dim_labels](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__dim_labels) - [dim_contracts](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__dim_contracts) - [dim_contract_abis](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__dim_contract_abis) -- [dim_labels](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__dim_labels) **Fact Tables:** - [fact_blocks](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_blocks) @@ -29,19 +29,20 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_transactions) - [fact_traces](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_traces) -**EZ Tables:** +**Convenience Tables:** +- [ez_decoded_event_logs](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__ez_decoded_event_logs) - [ez_native_transfers](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__ez_native_transfers) - [ez_token_transfers](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__ez_token_transfers) - -**NFT Tables:** -- [ez_nft_transfers](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.nft__ez_nft_transfers) - + ### Price Tables (ronin.price) - [dim_asset_metadata](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__dim_asset_metadata) - [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) - [ez_asset_metadata](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__ez_asset_metadata) - [ez_prices_hourly](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__ez_prices_hourly) +### NFT Tables (ronin.nft) +- [ez_nft_transfers](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.nft__ez_nft_transfers) + ## **Helpful User-Defined Functions (UDFs)** UDFs are custom functions built by the Flipside team that can be used in your queries to make your life easier. diff --git a/models/sources.yml b/models/sources.yml index a11ee43..18ae91c 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,25 +1,22 @@ version: 2 sources: - - name: github_actions - database: "{{ target.database }}" - schema: github_actions - tables: - - name: workflows - name: bronze_streamline database: streamline - schema: >- - {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} + schema: | + {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }} tables: - name: blocks - name: transactions - - name: >- - {{ 'receipts_by_hash' if var("GLOBAL_USES_RECEIPTS_BY_HASH", False) else 'receipts' }} + - name: receipts + - name: receipts_by_hash - name: traces - name: confirm_blocks - name: decoded_logs + - name: contract_abis - name: crosschain_silver - database: "{{ 'crosschain' if target.database.upper() == var('GLOBAL_PROD_DB_NAME').upper() else 'crosschain_dev' }}" + database: >- + {{ 'CROSSCHAIN_DEV' if '_DEV' in target.database.upper() else 'CROSSCHAIN' }} schema: silver tables: - name: labels_combined @@ -29,24 +26,37 @@ sources: - name: complete_provider_prices - name: complete_token_asset_metadata - name: complete_token_prices - - name: bronze_api - database: "{{ target.database }}" - schema: bronze_api - tables: - - name: contract_abis - name: crosschain_public database: crosschain schema: bronze_public tables: - name: user_abis - - name: silver + - name: complete_streamline + database: "{{ target.database }}" + schema: streamline + tables: + - name: complete_contract_abis + - name: github_actions + database: "{{ target.database }}" + schema: github_actions + tables: + - name: workflows + - name: abis_silver database: "{{ target.database }}" schema: silver tables: - name: verified_abis - - name: bronze_abi + - name: complete_event_abis + - name: fsc_evm_admin database: >- - {{ var('GLOBAL_PROD_DB_NAME') ~ ('_dev' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else '') }} - schema: streamline + {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }} + schema: admin tables: - - name: complete_contract_abis + - name: _master_keys + - name: rpc_node_logs + - name: logs_temp + database: "{{ target.database }}" + schema: silver + tables: + - name: logs + - name: decoded_logs diff --git a/packages.yml b/packages.yml index 407fb51..9eb19cd 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v3.30.0 + revision: v4.0.0-beta.38 \ No newline at end of file