diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml new file mode 100644 index 0000000..0321b9a --- /dev/null +++ b/.github/workflows/dbt_docs_update.yml @@ -0,0 +1,65 @@ +name: dbt_docs_update + +on: + push: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + +jobs: + scheduled_run: + name: docs_update + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip install dbt-snowflake==1.3.0 + dbt deps + - name: checkout docs branch + run: | + git checkout -B docs origin/main + + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..160a31d --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,66 @@ +name: dbt_run_adhoc +run-name: dbt_run_adhoc + + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ inputs.warehouse }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + strategy: + matrix: + command: ${{fromJson(inputs.dbt_command)}} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + ${{ matrix.command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..be032ce --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,40 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + schedule: + - cron: '0 5 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_dev_refresh_external_tables.yml b/.github/workflows/dbt_run_dev_refresh_external_tables.yml new file mode 100644 index 0000000..4e43c3e --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh_external_tables.yml @@ -0,0 +1,40 @@ +name: dbt_run_dev_refresh_external_tables +run-name: dbt_run_dev_refresh_external_tables + +on: + schedule: + - cron: '0 6 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_dev + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation stage_external_sources --vars "ext_full_refresh: true" diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml new file mode 100644 index 0000000..4731f6f --- /dev/null +++ b/.github/workflows/dbt_run_incremental.yml @@ -0,0 +1,43 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 3 * * *' + +env: + DBT_PROFILES_DIR: .dbt/ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation stage_external_sources --vars "ext_full_refresh: true" + dbt run -s ./models + dbt run-operation stage_external_sources --vars "ext_full_refresh: true" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_balances.yml b/.github/workflows/dbt_run_streamline_balances.yml new file mode 100644 index 0000000..785a89b --- /dev/null +++ b/.github/workflows/dbt_run_streamline_balances.yml @@ -0,0 +1,40 @@ +name: dbt_run_streamline_balances +run-name: dbt_run_streamline_balances + +on: + schedule: + - cron: '30 3 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_bulk_get_validator_metadata -t prod diff --git a/.github/workflows/dbt_run_streamline_validators copy.yml b/.github/workflows/dbt_run_streamline_validators copy.yml new file mode 100644 index 0000000..a42d8b6 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_validators copy.yml @@ -0,0 +1,40 @@ +name: dbt_run_streamline_validators +run-name: dbt_run_streamline_validators + +on: + schedule: + - cron: '0 0 * * 0' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/streamline__validators_realtime.sql \ No newline at end of file diff --git a/.github/workflows/dbt_run_udf_blockchain.yml b/.github/workflows/dbt_run_udf_blockchain.yml new file mode 100644 index 0000000..bed501b --- /dev/null +++ b/.github/workflows/dbt_run_udf_blockchain.yml @@ -0,0 +1,42 @@ +name: dbt_run_udf_blockchain +run-name: dbt_run_udf_blockchain + +on: + schedule: + # once an hour + - cron: '5 * * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation get_blockchain_api + dbt run -m models/silver/silver__blockchain.sql \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml new file mode 100644 index 0000000..a677024 --- /dev/null +++ b/.github/workflows/dbt_test.yml @@ -0,0 +1,41 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 8 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m ./models diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..a2530f2 --- /dev/null +++ b/profiles.yml @@ -0,0 +1,29 @@ +osmosis: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + prod: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + config: + send_anonymous_usage_stats: False \ No newline at end of file