From 8fbafe38ff371e75aa4b6c553749a6a65ac0a382 Mon Sep 17 00:00:00 2001 From: Austin Date: Mon, 9 Dec 2024 13:47:51 -0500 Subject: [PATCH] updates --- .github/workflows/dbt_run_abi_refresh.yml | 14 +++--- ...heduled_decoded_logs_history_user_abis.yml | 45 ------------------- .github/workflows/dbt_test_tasks.yml | 29 ------------ data/github_actions__workflows.csv | 7 ++- .../bronze_api_testnet__contract_abis.sql | 4 +- 5 files changed, 13 insertions(+), 86 deletions(-) delete mode 100644 .github/workflows/dbt_run_scheduled_decoded_logs_history_user_abis.yml delete mode 100644 .github/workflows/dbt_test_tasks.yml diff --git a/.github/workflows/dbt_run_abi_refresh.yml b/.github/workflows/dbt_run_abi_refresh.yml index 2af96d7..ea3e419 100644 --- a/.github/workflows/dbt_run_abi_refresh.yml +++ b/.github/workflows/dbt_run_abi_refresh.yml @@ -3,9 +3,9 @@ run-name: dbt_run_abi_refresh on: workflow_dispatch: - schedule: - # Runs “At minute 30 past every 12th hour.” (see https://crontab.guru) - - cron: '30 */12 * * *' + branches: + - "main" + env: DBT_PROFILES_DIR: ./ @@ -42,9 +42,11 @@ jobs: run: | pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + + - name: Update ABI models run: | dbt run -m "berachain_models,tag:abis" - - + - name: Kick off decoded logs history, if there are new ABIs from users + run: | + dbt run-operation run_decoded_logs_history \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_decoded_logs_history_user_abis.yml b/.github/workflows/dbt_run_scheduled_decoded_logs_history_user_abis.yml deleted file mode 100644 index 63bf5a2..0000000 --- a/.github/workflows/dbt_run_scheduled_decoded_logs_history_user_abis.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_scheduled_decoded_logs_history_user_abis -run-name: dbt_run_scheduled_decoded_logs_history_user_abis - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Kick off decoded logs history, if there are new ABIs from users - run: | - dbt run-operation run_decoded_logs_history \ No newline at end of file diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml deleted file mode 100644 index a1ffffa..0000000 --- a/.github/workflows/dbt_test_tasks.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: dbt_test_tasks -run-name: dbt_test_tasks - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - - - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main - secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 606b400..3cfb633 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,7 +1,6 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"3,33 * * * *" -dbt_run_scheduled_non_realtime,"19,49 * * * *" -dbt_test_tasks,"8 * * * *" -dbt_run_streamline_decoder,"27,57 * * * *" +dbt_run_scheduled_non_realtime,"19 * * * *" +dbt_run_streamline_decoder,"27 * * * *" dbt_run_streamline_decoded_logs_history,"5 1 * * 6" -dbt_run_scheduled_decoded_logs_history_user_abis,"21 23 * * *" \ No newline at end of file +dbt_run_abi_refresh,"29 23 * * *" \ No newline at end of file diff --git a/models/bronze/api_udf/bronze_api_testnet__contract_abis.sql b/models/bronze/api_udf/bronze_api_testnet__contract_abis.sql index 3fb45ce..729001c 100644 --- a/models/bronze/api_udf/bronze_api_testnet__contract_abis.sql +++ b/models/bronze/api_udf/bronze_api_testnet__contract_abis.sql @@ -28,7 +28,7 @@ AND contract_address NOT IN ( ORDER BY total_interaction_count DESC LIMIT - 50 + 400 ), all_contracts AS ( SELECT contract_address @@ -45,7 +45,7 @@ row_nos AS ( FROM all_contracts ), -batched AS ({% for item in range(51) %} +batched AS ({% for item in range(501) %} SELECT rn.contract_address, live.udf_api('GET', CONCAT('https://api.routescan.io/v2/network/testnet/evm/80084/etherscan/api?module=contract&action=getabi&address=', rn.contract_address),{},{}, '') AS abi_data, SYSDATE() AS _inserted_timestamp FROM