diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml new file mode 100644 index 0000000..608bed2 --- /dev/null +++ b/.github/workflows/dbt_run_adhoc.yml @@ -0,0 +1,66 @@ +name: dbt_run_adhoc +run-name: dbt_run_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ inputs.warehouse }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + strategy: + matrix: + command: ${{fromJson(inputs.dbt_command)}} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + ${{ matrix.command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_api_feeder_table.yml b/.github/workflows/dbt_run_api_feeder_table.yml new file mode 100644 index 0000000..4ea4458 --- /dev/null +++ b/.github/workflows/dbt_run_api_feeder_table.yml @@ -0,0 +1,42 @@ +name: dbt_run_api_feeder_table +run-name: dbt_run_api_feeder_table + +on: + workflow_dispatch: + schedule: + # Runs "at 8:00 UTC" (see https://crontab.guru) + - cron: '0 8 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m models/silver/api_udf \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml new file mode 100644 index 0000000..7d16d71 --- /dev/null +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -0,0 +1,42 @@ +name: dbt_run_dev_refresh +run-name: dbt_run_dev_refresh + +on: + workflow_dispatch: + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone \ No newline at end of file diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml new file mode 100644 index 0000000..6bceb93 --- /dev/null +++ b/.github/workflows/dbt_run_incremental.yml @@ -0,0 +1,42 @@ +name: dbt_run_scheduled +run-name: dbt_run_scheduled + +on: + workflow_dispatch: + schedule: + # Runs "every 2 hours" (see https://crontab.guru) + - cron: '0 1-23/2 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --exclude models/silver/api_udf \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml new file mode 100644 index 0000000..ea980ff --- /dev/null +++ b/.github/workflows/dbt_test.yml @@ -0,0 +1,45 @@ +name: dbt_test +run-name: dbt_test + +on: + workflow_dispatch: + schedule: + # Runs "at 9:00 UTC" (see https://crontab.guru) + - cron: '0 9 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ secrets.ACCOUNT }}" + ROLE: "${{ secrets.ROLE }}" + USER: "${{ secrets.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ secrets.REGION }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" + SCHEMA: "${{ secrets.SCHEMA }}" + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==1.3.0 cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt test + + + diff --git a/.gitignore b/.gitignore index 2b1b09f..d99e9bd 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,5 @@ logs/ .history/ **/.DS_Store .vscode/ -dbt-env/ -.env \ No newline at end of file +.env +dbt-env/ \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index 674809b..58446ed 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,17 +1,29 @@ arbitrum: - target: dev + target: prod outputs: dev: type: snowflake account: "{{ env_var('ACCOUNT') }}" - # User/password auth - user: "{{ env_var('USERNAME') }}" - password: "{{ env_var('PASSWORD') }}" role: "{{ env_var('ROLE') }}" - schema: "{{ env_var('SCHEMA') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" region: "{{ env_var('REGION') }}" database: "{{ env_var('DATABASE') }}" warehouse: "{{ env_var('WAREHOUSE') }}" + schema: SILVER + threads: 4 + client_session_keep_alive: False + query_tag: arbitrum_curator + prod: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + role: "{{ env_var('ROLE') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + schema: SILVER threads: 4 client_session_keep_alive: False query_tag: arbitrum_curator