update sources, workflows and packages

This commit is contained in:
drethereum 2024-10-30 13:34:05 -06:00
parent 32716402c2
commit 604c344d93
13 changed files with 90 additions and 24 deletions

View File

@ -1,5 +1,5 @@
name: dbt_run_deployment
run-name: dbt_run_deployment
run-name: ${{ inputs.dbt_command }}
on:
workflow_dispatch:

View File

@ -39,7 +39,8 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
- name: Run Observability Models
run: |
dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability"

View File

@ -39,6 +39,7 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
- name: Run Heal Models
run: |
dbt run -m "base_models,tag:heal" --vars '{"HEAL_MODEL":True}'
dbt run -m "<evm_chain>_models,tag:heal" --vars '{"HEAL_MODEL":True}'

View File

@ -0,0 +1,45 @@
name: dbt_run_scheduled_abis
run-name: dbt_run_scheduled_abis
on:
workflow_dispatch:
branches:
- "main"
env:
DBT_PROFILES_DIR: ./
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_test
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run ABI Models
run: |
dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis"

View File

@ -39,6 +39,7 @@ jobs:
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
- name: Run Curated Models
run: |
dbt run -m "<evm_chain>_models,tag:curated" "fsc_evm,tag:curated"

View File

@ -40,6 +40,10 @@ jobs:
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
- name: Run Main Models
run: |
dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels"
dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels"
- name: Run Streamline Models
run: |
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete"

View File

@ -40,10 +40,10 @@ jobs:
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
- name: Run Chainhead Models
run: |
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime"
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts"
- name: Test Chainhead
- name: Run Chainhead Tests
run: |
dbt test -m "fsc_evm,tag:chainhead"

View File

@ -40,6 +40,6 @@ jobs:
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
- name: Run History Models
run: |
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history"
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts"

View File

@ -35,11 +35,10 @@ jobs:
python-version: "3.10"
cache: "pip"
- name: install dependencies
- name: Build Daily Testing Views
run: |
pip install -r requirements.txt
dbt deps
- name: Test All Data, with exceptions
dbt run -m "fsc_evm,tag:daily_test"
- name: Run Daily Tests
run: |
dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models
dbt test -m "fsc_evm,tag:daily_test"

View File

@ -40,10 +40,10 @@ jobs:
pip install -r requirements.txt
dbt deps
- name: Build Testing Views
- name: Build Recent Testing Views
run: |
dbt run -m "fsc_evm,tag:recent_test"
- name: Test Recent Data
- name: Run Recent Tests
run: |
dbt test -m "fsc_evm,tag:recent_test"

View File

@ -40,10 +40,10 @@ jobs:
pip install -r requirements.txt
dbt deps
- name: Build Testing Views
- name: Build Full Testing Views
run: |
dbt run -m "fsc_evm,tag:full_test"
- name: Test Full Data
- name: Run Full Tests
run: |
dbt test -m "fsc_evm,tag:full_test"

View File

@ -28,4 +28,19 @@ sources:
- name: complete_native_prices
- name: complete_provider_prices
- name: complete_token_asset_metadata
- name: complete_token_prices
- name: complete_token_prices
- name: bronze_api
database: "{{ target.database }}"
schema: bronze_api
tables:
- name: contract_abis
- name: crosschain_public
database: crosschain
schema: bronze_public
tables:
- name: user_abis
- name: silver
database: "{{ target.database }}"
schema: silver
tables:
- name: verified_abis

View File

@ -1,3 +1,3 @@
packages:
- git: https://github.com/FlipsideCrypto/fsc-evm.git
revision: <insert-version>
revision: <insert-version-v3+>