mirror of
https://github.com/FlipsideCrypto/analytics-workflow-templates.git
synced 2026-02-06 11:17:52 +00:00
- add dbt.yml
- refactor workflows to use dbt.yml
This commit is contained in:
Julius Remigio
parent
b2a7dadce1
commit
72d45581b9
66
.github/workflows/dbt.yml
vendored
Normal file
66
.github/workflows/dbt.yml
vendored
Normal file
@ -0,0 +1,66 @@
|
||||
name: dbt
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
warehouse:
|
||||
required: false
|
||||
type: string
|
||||
default: DBT_CLOUD
|
||||
environment:
|
||||
required: true
|
||||
type: string
|
||||
command:
|
||||
required: true
|
||||
type: string
|
||||
python_version:
|
||||
required: false
|
||||
type: string
|
||||
default: "3.10"
|
||||
|
||||
env:
|
||||
DBT_PROFILES_DIR: ./
|
||||
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
jobs:
|
||||
dbt:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DBT_PROFILES_DIR: ./
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: ${{ inputs.warehouse }}
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
environment:
|
||||
name: ${{ inputs.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
cache: "pip"
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
${{ inputs.command }}
|
||||
- name: Store logs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: dbt-logs
|
||||
path: |
|
||||
logs
|
||||
target
|
||||
@ -11,58 +11,33 @@ on:
|
||||
type: string
|
||||
description: 'github environment to get configuration values'
|
||||
required: true
|
||||
default: prod
|
||||
warehouse:
|
||||
type: string
|
||||
description: 'dbt warehouse'
|
||||
required: true
|
||||
default: DBT_CLOUD
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
WAREHOUSE: "${{ inputs.warehouse }}"
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: ${{ inputs.environment }}
|
||||
dbt:
|
||||
uses: ./.github/workflows/dbt.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
warehouse: ${{ inputs.warehouse }}
|
||||
environment: workflow_${{ inputs.environment }}
|
||||
command: ${{ inputs.dbt_command }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: "3.7.x"
|
||||
|
||||
- name: overwrite env vars
|
||||
if: ${{ env.USE_VARS == 'TRUE' }}
|
||||
run: |
|
||||
echo "DBT_VERSION=${{ vars.DBT_VERSION }}" >> $GITHUB_ENV
|
||||
echo "DBT_PROFILES_DIR=${{ vars.DBT_PROFILES_DIR }}" >> $GITHUB_ENV
|
||||
echo "ACCOUNT=${{ vars.ACCOUNT }}" >> $GITHUB_ENV
|
||||
echo "ROLE=${{ vars.ROLE }}" >> $GITHUB_ENV
|
||||
echo "USER=${{ vars.USER }}" >> $GITHUB_ENV
|
||||
echo "REGION=${{ vars.REGION }}" >> $GITHUB_ENV
|
||||
echo "DATABASE=${{ vars.DATABASE }}" >> $GITHUB_ENV
|
||||
echo "SCHEMA=${{ vars.SCHEMA }}" >> $GITHUB_ENV
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
${{ inputs.dbt_command }}
|
||||
|
||||
- name: Run datashare model
|
||||
run: |
|
||||
cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi;
|
||||
update_datashare:
|
||||
uses: ./.github/workflows/dbt.yml
|
||||
needs: dbt
|
||||
secrets: inherit
|
||||
with:
|
||||
warehouse: ${{ inputs.warehouse }}
|
||||
environment: workflow_${{ inputs.environment }}
|
||||
command: |
|
||||
dbt ls -m fsc_utils.datashare._datashare___create_gold
|
||||
cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l )
|
||||
if [ $cnt -eq 1 ]; then
|
||||
dbt run -m fsc_utils.datashare._datashare___create_gold
|
||||
fi
|
||||
53
.github/workflows/dbt_run_template.yml
vendored
53
.github/workflows/dbt_run_template.yml
vendored
@ -11,55 +11,18 @@ on:
|
||||
type: string
|
||||
description: 'github environment to get configuration values'
|
||||
required: true
|
||||
default: prod
|
||||
warehouse:
|
||||
type: string
|
||||
description: 'dbt warehouse'
|
||||
required: true
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_VERSION: "${{ secrets.DBT_VERSION }}"
|
||||
DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}"
|
||||
ACCOUNT: "${{ secrets.ACCOUNT }}"
|
||||
ROLE: "${{ secrets.ROLE }}"
|
||||
USER: "${{ secrets.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ secrets.REGION }}"
|
||||
DATABASE: "${{ secrets.DATABASE }}"
|
||||
SCHEMA: "${{ secrets.SCHEMA }}"
|
||||
|
||||
WAREHOUSE: "${{ inputs.warehouse }}"
|
||||
default: DBT_CLOUD
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: ${{ inputs.environment }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: "3.7.x"
|
||||
|
||||
- name: overwrite env vars
|
||||
if: ${{ env.USE_VARS == 'TRUE' }}
|
||||
run: |
|
||||
echo "DBT_VERSION=${{ vars.DBT_VERSION }}" >> $GITHUB_ENV
|
||||
echo "DBT_PROFILES_DIR=${{ vars.DBT_PROFILES_DIR }}" >> $GITHUB_ENV
|
||||
echo "ACCOUNT=${{ vars.ACCOUNT }}" >> $GITHUB_ENV
|
||||
echo "ROLE=${{ vars.ROLE }}" >> $GITHUB_ENV
|
||||
echo "USER=${{ vars.USER }}" >> $GITHUB_ENV
|
||||
echo "REGION=${{ vars.REGION }}" >> $GITHUB_ENV
|
||||
echo "DATABASE=${{ vars.DATABASE }}" >> $GITHUB_ENV
|
||||
echo "SCHEMA=${{ vars.SCHEMA }}" >> $GITHUB_ENV
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click
|
||||
dbt deps
|
||||
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
${{ inputs.dbt_command }}
|
||||
uses: ./.github/workflows/dbt.yml
|
||||
secrets: inherit
|
||||
with:
|
||||
warehouse: ${{ inputs.warehouse }}
|
||||
environment: workflow_${{ inputs.environment }}
|
||||
command: ${{ inputs.dbt_command }}
|
||||
Loading…
Reference in New Issue
Block a user