diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index 95dac29..d0735d8 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -1,5 +1,5 @@ name: dbt_run_deployment -run-name: dbt_run_deployment +run-name: ${{ inputs.dbt_command }} on: workflow_dispatch: diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index b8060af..dd4e8b7 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -39,7 +39,8 @@ jobs: run: | pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + + - name: Run Observability Models run: | dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability" diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_heal_models.yml index 97a7bd8..834347b 100644 --- a/.github/workflows/dbt_run_heal_models.yml +++ b/.github/workflows/dbt_run_heal_models.yml @@ -39,6 +39,7 @@ jobs: run: | pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + + - name: Run Heal Models run: | - dbt run -m "base_models,tag:heal" --vars '{"HEAL_MODEL":True}' \ No newline at end of file + dbt run -m "_models,tag:heal" --vars '{"HEAL_MODEL":True}' \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml new file mode 100644 index 0000000..61a044c --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -0,0 +1,45 @@ +name: dbt_run_scheduled_abis +run-name: dbt_run_scheduled_abis + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_test + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run ABI Models + run: | + dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml index 3df7cbd..d430226 100644 --- a/.github/workflows/dbt_run_scheduled_curated.yml +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -39,6 +39,7 @@ jobs: run: | pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + + - name: Run Curated Models run: | dbt run -m "_models,tag:curated" "fsc_evm,tag:curated" \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index aaddaab..e96788e 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -40,6 +40,10 @@ jobs: pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + - name: Run Main Models run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" \ No newline at end of file + dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" + + - name: Run Streamline Models + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index ffe2678..ed7781e 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -40,10 +40,10 @@ jobs: pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + - name: Run Chainhead Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" - - name: Test Chainhead + - name: Run Chainhead Tests run: | dbt test -m "fsc_evm,tag:chainhead" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 617b86f..bf57ca8 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -40,6 +40,6 @@ jobs: pip install -r requirements.txt dbt deps - - name: Run DBT Jobs + - name: Run History Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_history_receipts" diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index 54b0f67..44c6d55 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -35,11 +35,10 @@ jobs: python-version: "3.10" cache: "pip" - - name: install dependencies + - name: Build Daily Testing Views run: | - pip install -r requirements.txt - dbt deps - - - name: Test All Data, with exceptions + dbt run -m "fsc_evm,tag:daily_test" + + - name: Run Daily Tests run: | - dbt test --exclude "fsc_evm,tag:full_test" "fsc_evm,tag:recent_test" "fsc_evm,tag:gha_tasks" livequery_models \ No newline at end of file + dbt test -m "fsc_evm,tag:daily_test" \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index 1d8c0e4..2aeea57 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -40,10 +40,10 @@ jobs: pip install -r requirements.txt dbt deps - - name: Build Testing Views + - name: Build Recent Testing Views run: | dbt run -m "fsc_evm,tag:recent_test" - - name: Test Recent Data + - name: Run Recent Tests run: | dbt test -m "fsc_evm,tag:recent_test" \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml index 9c4aac6..b32e830 100644 --- a/.github/workflows/dbt_test_monthly.yml +++ b/.github/workflows/dbt_test_monthly.yml @@ -40,10 +40,10 @@ jobs: pip install -r requirements.txt dbt deps - - name: Build Testing Views + - name: Build Full Testing Views run: | dbt run -m "fsc_evm,tag:full_test" - - name: Test Full Data + - name: Run Full Tests run: | dbt test -m "fsc_evm,tag:full_test" \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index a79178a..d8d168e 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -28,4 +28,19 @@ sources: - name: complete_native_prices - name: complete_provider_prices - name: complete_token_asset_metadata - - name: complete_token_prices \ No newline at end of file + - name: complete_token_prices + - name: bronze_api + database: "{{ target.database }}" + schema: bronze_api + tables: + - name: contract_abis + - name: crosschain_public + database: crosschain + schema: bronze_public + tables: + - name: user_abis + - name: silver + database: "{{ target.database }}" + schema: silver + tables: + - name: verified_abis \ No newline at end of file diff --git a/packages.yml b/packages.yml index 3563446..52e2031 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: + revision: