From c14a1483e647dcd412bf1da81c8c3677cdab2f4f Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Wed, 22 Feb 2023 18:00:21 -0500 Subject: [PATCH] Backfill/workflow (#112) * new workflows * new workflows * break up ranges * realtime --- .../workflows/dbt_run_streamline_decoder.yml | 2 +- ...run_streamline_decoder_history_10_19m.yml} | 8 ++-- ..._run_streamline_decoder_history_20_24m.yml | 43 +++++++++++++++++++ ..._run_streamline_decoder_history_25_29m.yml | 43 +++++++++++++++++++ ..._run_streamline_decoder_history_30_34m.yml | 43 +++++++++++++++++++ ..._run_streamline_decoder_history_35_39m.yml | 43 +++++++++++++++++++ .gitignore | 2 +- 7 files changed, 178 insertions(+), 6 deletions(-) rename .github/workflows/{dbt_run_streamline_decoder_history.yml => dbt_run_streamline_decoder_history_10_19m.yml} (79%) create mode 100644 .github/workflows/dbt_run_streamline_decoder_history_20_24m.yml create mode 100644 .github/workflows/dbt_run_streamline_decoder_history_25_29m.yml create mode 100644 .github/workflows/dbt_run_streamline_decoder_history_30_34m.yml create mode 100644 .github/workflows/dbt_run_streamline_decoder_history_35_39m.yml diff --git a/.github/workflows/dbt_run_streamline_decoder.yml b/.github/workflows/dbt_run_streamline_decoder.yml index a5d4255..f1d4f4b 100644 --- a/.github/workflows/dbt_run_streamline_decoder.yml +++ b/.github/workflows/dbt_run_streamline_decoder.yml @@ -26,7 +26,7 @@ jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: workflow_prod + name: workflow_prod_backfill steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history_10_19m.yml similarity index 79% rename from .github/workflows/dbt_run_streamline_decoder_history.yml rename to .github/workflows/dbt_run_streamline_decoder_history_10_19m.yml index 0290f3c..092c1e7 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history_10_19m.yml @@ -1,5 +1,5 @@ -name: dbt_run_streamline_decoder_history -run-name: dbt_run_streamline_decoder_history +name: dbt_run_streamline_decoder_history_10_19m +run-name: dbt_run_streamline_decoder_history_10_19m on: workflow_dispatch: @@ -25,7 +25,7 @@ jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: workflow_prod + name: workflow_prod_backfill steps: - uses: actions/checkout@v3 @@ -40,4 +40,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history \ No newline at end of file + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_01* \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history_20_24m.yml b/.github/workflows/dbt_run_streamline_decoder_history_20_24m.yml new file mode 100644 index 0000000..fec88d8 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history_20_24m.yml @@ -0,0 +1,43 @@ +name: dbt_run_streamline_decoder_history_20_24m +run-name: dbt_run_streamline_decoder_history_20_24m + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_backfill + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_020* models/silver/streamline/decoder/history/streamline__decode_logs_history_021* models/silver/streamline/decoder/history/streamline__decode_logs_history_022* models/silver/streamline/decoder/history/streamline__decode_logs_history_023* models/silver/streamline/decoder/history/streamline__decode_logs_history_024* \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history_25_29m.yml b/.github/workflows/dbt_run_streamline_decoder_history_25_29m.yml new file mode 100644 index 0000000..5c4fe05 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history_25_29m.yml @@ -0,0 +1,43 @@ +name: dbt_run_streamline_decoder_history_26_29m +run-name: dbt_run_streamline_decoder_history_26_29m + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_backfill + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_025* models/silver/streamline/decoder/history/streamline__decode_logs_history_026* models/silver/streamline/decoder/history/streamline__decode_logs_history_027* models/silver/streamline/decoder/history/streamline__decode_logs_history_028* models/silver/streamline/decoder/history/streamline__decode_logs_history_029* \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history_30_34m.yml b/.github/workflows/dbt_run_streamline_decoder_history_30_34m.yml new file mode 100644 index 0000000..c3a083d --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history_30_34m.yml @@ -0,0 +1,43 @@ +name: dbt_run_streamline_decoder_history_30_34m +run-name: dbt_run_streamline_decoder_history_30_34m + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_backfill + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_030* models/silver/streamline/decoder/history/streamline__decode_logs_history_031* models/silver/streamline/decoder/history/streamline__decode_logs_history_032* models/silver/streamline/decoder/history/streamline__decode_logs_history_033* models/silver/streamline/decoder/history/streamline__decode_logs_history_034* \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history_35_39m.yml b/.github/workflows/dbt_run_streamline_decoder_history_35_39m.yml new file mode 100644 index 0000000..c0191b9 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history_35_39m.yml @@ -0,0 +1,43 @@ +name: dbt_run_streamline_decoder_history_35_39m +run-name: dbt_run_streamline_decoder_history_35_39m + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_backfill + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_035* models/silver/streamline/decoder/history/streamline__decode_logs_history_036* models/silver/streamline/decoder/history/streamline__decode_logs_history_037* models/silver/streamline/decoder/history/streamline__decode_logs_history_038* models/silver/streamline/decoder/history/streamline__decode_logs_history_039* \ No newline at end of file diff --git a/.gitignore b/.gitignore index 98ab642..8bf82e8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ logs/ .vscode/ dbt-env/ -.* \ No newline at end of file +.env \ No newline at end of file