Backfill/workflow (#112)

* new workflows

* new workflows

* break up ranges

* realtime

.github/workflows/dbt_run_streamline_decoder.yml

@@ -26,7 +26,7 @@ jobs:
   run_dbt_jobs:
     runs-on: ubuntu-latest
     environment: 
-      name: workflow_prod
+      name: workflow_prod_backfill
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/dbt_run_streamline_decoder_history_10_19m.yml

@@ -1,5 +1,5 @@
-name: dbt_run_streamline_decoder_history
-run-name: dbt_run_streamline_decoder_history
+name: dbt_run_streamline_decoder_history_10_19m
+run-name: dbt_run_streamline_decoder_history_10_19m
 
 on:
   workflow_dispatch:
@@ -25,7 +25,7 @@ jobs:
   run_dbt_jobs:
     runs-on: ubuntu-latest
     environment: 
-      name: workflow_prod
+      name: workflow_prod_backfill
 
     steps:
       - uses: actions/checkout@v3
@@ -40,4 +40,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_01*

.github/workflows/dbt_run_streamline_decoder_history_20_24m.yml

@@ -0,0 +1,43 @@
+name: dbt_run_streamline_decoder_history_20_24m
+run-name: dbt_run_streamline_decoder_history_20_24m
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_backfill
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_020* models/silver/streamline/decoder/history/streamline__decode_logs_history_021* models/silver/streamline/decoder/history/streamline__decode_logs_history_022* models/silver/streamline/decoder/history/streamline__decode_logs_history_023* models/silver/streamline/decoder/history/streamline__decode_logs_history_024*

.github/workflows/dbt_run_streamline_decoder_history_25_29m.yml

@@ -0,0 +1,43 @@
+name: dbt_run_streamline_decoder_history_26_29m
+run-name: dbt_run_streamline_decoder_history_26_29m
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_backfill
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_025* models/silver/streamline/decoder/history/streamline__decode_logs_history_026* models/silver/streamline/decoder/history/streamline__decode_logs_history_027* models/silver/streamline/decoder/history/streamline__decode_logs_history_028* models/silver/streamline/decoder/history/streamline__decode_logs_history_029* 

.github/workflows/dbt_run_streamline_decoder_history_30_34m.yml

@@ -0,0 +1,43 @@
+name: dbt_run_streamline_decoder_history_30_34m
+run-name: dbt_run_streamline_decoder_history_30_34m
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_backfill
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_030* models/silver/streamline/decoder/history/streamline__decode_logs_history_031* models/silver/streamline/decoder/history/streamline__decode_logs_history_032* models/silver/streamline/decoder/history/streamline__decode_logs_history_033* models/silver/streamline/decoder/history/streamline__decode_logs_history_034*

.github/workflows/dbt_run_streamline_decoder_history_35_39m.yml

@@ -0,0 +1,43 @@
+name: dbt_run_streamline_decoder_history_35_39m
+run-name: dbt_run_streamline_decoder_history_35_39m
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+    
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod_backfill
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/history/streamline__decode_logs_history_035* models/silver/streamline/decoder/history/streamline__decode_logs_history_036* models/silver/streamline/decoder/history/streamline__decode_logs_history_037* models/silver/streamline/decoder/history/streamline__decode_logs_history_038* models/silver/streamline/decoder/history/streamline__decode_logs_history_039*    

.gitignore

@@ -16,4 +16,4 @@ logs/
 .vscode/
 dbt-env/
 
-.*
+.env