update workflows

This commit is contained in:
WHYTEWYLL 2024-03-13 13:48:36 -06:00
parent 85066ca25c
commit 5512d862a9
7 changed files with 189 additions and 55 deletions

View File

@ -24,11 +24,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run -s tag:atlas
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt run -s tag:atlas
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target

View File

@ -24,11 +24,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run-operation run_sp_create_prod_clone; dbt run -s models/gold
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt run-operation run_sp_create_prod_clone; dbt run -s models/gold
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target

View File

@ -37,10 +37,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: |
${{ inputs.dbt_command }}
warehouse: ${{ inputs.WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
${{ inputs.dbt_command }}
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target

View File

@ -24,11 +24,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run -s tag:nearblocks tag:activity
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt run -s tag:nearblocks tag:activity
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target

View File

@ -25,11 +25,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run-operation get_nearblocks_fts
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt run-operation get_nearblocks_fts
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target

View File

@ -23,11 +23,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt run --vars {"S3_LOOKBACK_HOURS":-6} -m models/silver/streamline/streamline__s3_sync.sql
environment: workflow_prod
warehouse: ${{ vars.SYNC_WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt run --vars {"S3_LOOKBACK_HOURS":-6} -m models/silver/streamline/streamline__s3_sync.sql
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target

View File

@ -24,11 +24,30 @@ concurrency:
group: ${{ github.workflow }}
jobs:
called_workflow_template:
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7
with:
dbt_command: >
dbt seed; dbt run -s models/bronze tag:helper tag:load+ tag:labels+ --exclude streamline__s3_sync tag:livequery tag:atlas tag:exclude_from_schedule
environment: workflow_prod
warehouse: ${{ vars.WAREHOUSE }}
secrets: inherit
dbt:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt seed; dbt run -s models/bronze tag:helper tag:load+ tag:labels+ --exclude streamline__s3_sync tag:livequery tag:atlas tag:exclude_from_schedule
- name: Store logs
uses: actions/upload-artifact@v3
with:
name: dbt-logs
path: |
logs
target