diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml new file mode 100644 index 0000000..821a61c --- /dev/null +++ b/.github/workflows/dbt_run_full_observability.yml @@ -0,0 +1,44 @@ +name: dbt_run_full_observability +run-name: dbt_run_full_observability + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_2xl + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m "fsc_evm,tag:observability" diff --git a/.github/workflows/dbt_seed_refresh.yml b/.github/workflows/dbt_seed_refresh.yml new file mode 100644 index 0000000..caf9afd --- /dev/null +++ b/.github/workflows/dbt_seed_refresh.yml @@ -0,0 +1,46 @@ +name: dbt_seed_refresh +run-name: dbt_seed_refresh + +on: + push: + branches: + - main + paths: + - "data/**/*.csv" # Only trigger when CSV files change + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run dbt seed + run: | + dbt seed diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index cd9285e..23914d6 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -24,7 +24,7 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_test steps: @@ -43,7 +43,8 @@ jobs: - name: Build testing views run: | dbt run -m "fsc_evm,tag:recent_test" - + - name: Run intraday tests run: | - dbt test -m "fsc_evm,tag:recent_test" "fsc_evm,tag:recent_test_confirm_blocks" \ No newline at end of file + dbt run -m "fsc_evm,tag:observability" + dbt test -m "fsc_evm,tag:recent_test" "fsc_evm,tag:recent_test_confirm_blocks"