GHA modifications

.github/workflows/dbt_run_adhoc copy.yml

@@ -0,0 +1,64 @@
+name: dbt_run_adhoc
+run-name: ${{ inputs.dbt_command }}
+
+on:
+  workflow_dispatch:
+    branches:
+      - "pre-release/v4-beta"
+    inputs:
+      environment:
+        type: choice 
+        description: DBT Run Environment
+        required: true
+        options:
+          - dev
+          - prod
+        default: dev
+      warehouse:
+        type: choice 
+        description: Snowflake warehouse
+        required: true 
+        options:
+          - DBT
+          - DBT_CLOUD
+        default: DBT
+      dbt_command:
+        type: string
+        description: 'DBT Run Command'
+        required: true
+    
+env:
+    ACCOUNT: "${{ vars.ACCOUNT }}"
+    ROLE: "${{ vars.ROLE }}"
+    USER: "${{ vars.USER }}"
+    PASSWORD: "${{ secrets.PASSWORD }}"
+    REGION: "${{ vars.REGION }}"
+    DATABASE: "${{ vars.DATABASE }}"
+    WAREHOUSE: "${{ inputs.warehouse }}"
+    SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_${{ inputs.environment }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: Run DBT Jobs
+        run: |
+          ${{ inputs.dbt_command }}

.github/workflows/dbt_run_master_keys.yml

@@ -0,0 +1,44 @@
+name: dbt_run_master_keys
+
+on:
+  push:
+    branches:
+      - "pre-release/v4-beta"
+    paths:
+      - 'macros/global/variables/return_vars.sql'
+
+env:
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      
+      - name: Write Variables to fsc_evm.admin._master_keys
+        run: |
+          dbt run-operation return_vars --vars '{"WRITE_VARS_ENABLED": true}'

.github/workflows/dbt_run_template.yml

@@ -0,0 +1,96 @@
+name: dbt_run_template
+
+on:
+  workflow_call:
+    inputs:
+      warehouse:
+        required: false
+        type: string
+        default: DBT_CLOUD
+      target:
+        required: false
+        type: string
+        default: prod
+      command_name:
+        required: true
+        type: string
+        default: Run DBT Command
+      command:
+        required: true
+        type: string
+      command_name_2:
+        required: false
+        type: string
+        default: Run DBT Command 2 (if enabled)
+      command_2:
+        required: false
+        type: string
+        default: ""
+          
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set environment variables
+        run: |
+          echo "TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV  
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+          
+          if [[ "${{ inputs.target }}" == *"prod"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=${{ inputs.warehouse }}" >> $GITHUB_ENV
+          elif [[ "${{ inputs.target }}" == *"test"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_TEST" >> $GITHUB_ENV
+          else
+            echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV
+            echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV
+            echo "WAREHOUSE=${{ inputs.warehouse }}" >> $GITHUB_ENV
+          fi
+          
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      
+      - name: ${{inputs.command_name}}
+        run: |
+          ${{ inputs.command }}
+
+      - name: ${{inputs.command_name_2}}
+        if: "${{ inputs.command_2 != '' }}"
+        run: |
+          ${{ inputs.command_2 }}
+
+  notify-failure:
+    needs: [run_dbt_jobs]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_template_alter_all_gha_tasks.yml

@@ -0,0 +1,77 @@
+name: dbt_run_template_alter_all_gha_tasks
+
+on:
+  workflow_call:
+    inputs:
+      task_action:
+        type: string
+        description: 'Action to perform on all tasks (RESUME, SUSPEND)'
+        required: true
+        default: RESUME
+      target:
+        type: string
+        required: false
+        default: prod
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set environment variables
+        run: |
+          echo "TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV  
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+          
+          if [[ "${{ inputs.target }}" == *"prod"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
+          elif [[ "${{ inputs.target }}" == *"test"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_TEST" >> $GITHUB_ENV
+          else
+            echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV
+            echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
+          fi
+          
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      
+      - name: Alter all GHA tasks
+        run: |
+          dbt run-operation fsc_evm.alter_all_gha_tasks --args '{"task_action": "${{ inputs.task_action }}"}'
+
+  notify-failure:
+    needs: [run_dbt_jobs]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_template_alter_gha_tasks.yml

@@ -0,0 +1,83 @@
+name: dbt_run_template_alter_gha_tasks
+
+on:
+  workflow_call:
+    inputs:
+      workflow_name:
+        type: string
+        description: 'Name of the workflow to perform the action on, no .yml extension'
+        required: true
+      task_action:
+        type: string
+        description: 'Action to perform'
+        required: true
+      target:
+        type: string
+        required: false
+        default: prod
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set environment variables
+        run: |
+          echo "TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV  
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+          
+          if [[ "${{ inputs.target }}" == *"prod"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
+          elif [[ "${{ inputs.target }}" == *"test"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_TEST" >> $GITHUB_ENV
+          else
+            echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV
+            echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
+          fi
+          
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: Set up workflow name
+        run: echo "WORKFLOW_NAME_UPPER=$(echo '${{ inputs.workflow_name }}' | tr '[:lower:]' '[:upper:]')" >> $GITHUB_ENV
+
+      - name: Run DBT Jobs
+        run: |
+          dbt run-operation fsc_evm.alter_gha_tasks --args '{"task_names": "TRIGGER_${{ env.WORKFLOW_NAME_UPPER }}", "task_action": "${{ inputs.task_action }}"}'
+
+  notify-failure:
+    needs: [run_dbt_jobs]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_template_dev_refresh.yml

@@ -0,0 +1,109 @@
+name: dbt_run_template_dev_refresh
+
+on:
+  workflow_call:
+          
+jobs:
+  run_dbt_jobs_refresh:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set production environment variables
+        run: |
+          echo "TARGET=prod" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+          echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      
+      - name: Run Dev Refresh
+        run: |
+          dbt run-operation fsc_evm.run_sp_create_prod_clone
+
+  notify-failure:
+    needs: [run_dbt_jobs_refresh]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+
+  run_dbt_jobs_udfs:
+    runs-on: ubuntu-latest
+    needs: run_dbt_jobs_refresh
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set dev environment variables
+        run: |
+          echo "TARGET=dev" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV
+          echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+          echo "WAREHOUSE=DBT" >> $GITHUB_ENV
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: Run Recreate UDFs
+        run: |
+          dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
+          dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
+
+  notify-failure-2:
+    needs: [run_dbt_jobs_udfs]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_template_docs_update.yml

@@ -0,0 +1,87 @@
+name: dbt_run_template_docs_update
+
+on:
+  workflow_call:
+          
+jobs:
+  run_dbt_jobs_refresh:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set production environment variables
+        run: |
+          echo "TARGET=prod" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+          echo "WAREHOUSE=DBT_CLOUD" >> $GITHUB_ENV
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      
+      - name: refresh ddl for datashare
+        run: |
+          cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; 
+
+      - name: checkout docs branch
+        run: |
+          git checkout -B docs origin/main
+
+      - name: generate dbt docs
+        run: dbt docs generate -t prod
+
+      - name: move files to docs directory
+        run: |
+          mkdir -p ./docs
+          cp target/{catalog.json,manifest.json,index.html} docs/
+
+      - name: clean up target directory
+        run: dbt clean
+
+      - name: check for changes
+        run: git status
+
+      - name: stage changed files
+        run: git add .
+
+      - name: commit changed files
+        run: |
+          git config user.email "abc@xyz"
+          git config user.name "github-actions"
+          git commit -am "Auto-update docs"
+
+      - name: push changes to docs
+        run: |
+          git push -f --set-upstream origin docs
+
+  notify-failure:
+    needs: [run_dbt_jobs_refresh]
+    if: failure()
+    uses: ./.github/workflows/slack_notify.yml
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/dbt_run_template_integration_test.yml

@@ -0,0 +1,84 @@
+name: dbt_run_template
+
+on:
+  workflow_call:
+    inputs:
+      warehouse:
+        required: false
+        type: string
+        default: DBT_CLOUD
+      target:
+        required: false
+        type: string
+        default: dev
+      command:
+        required: true
+        type: string
+      python_version:
+        required: false
+        type: string
+        default: "3.10"
+          
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_secrets
+
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Extract project & profile names from dbt_project.yml
+        id: project-name
+        run: |
+          PROFILE_NAME=$(grep "^profile:" dbt_project.yml | sed 's/^profile:[[:space:]]*"//' | sed 's/".*$//')
+          PROJECT_NAME=$(grep "^name:" dbt_project.yml | sed 's/^name:[[:space:]]*"//' | sed 's/".*$//')
+          echo "PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV
+          echo "PROJECT_NAME=$PROJECT_NAME" >> $GITHUB_ENV
+          echo "PROFILE_NAME: $PROFILE_NAME"
+          echo "PROJECT_NAME: $PROJECT_NAME"
+      
+      - name: Set environment variables
+        run: |
+          echo "TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+          echo "ACCOUNT=vna27887.us-east-1" >> $GITHUB_ENV
+          echo "REGION=us-east-1" >> $GITHUB_ENV
+          echo "SCHEMA=ADMIN" >> $GITHUB_ENV
+          echo "USER=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV  
+          echo "PASSWORD=${{ secrets.PASSWORD }}" >> $GITHUB_ENV
+          
+          if [[ "${{ inputs.target }}" == *"prod"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=${{ inputs.warehouse }}" >> $GITHUB_ENV
+          elif [[ "${{ inputs.target }}" == *"test"* ]]; then
+            echo "DATABASE=$PROFILE_NAME" >> $GITHUB_ENV
+            echo "ROLE=DBT_CLOUD_$PROFILE_NAME" >> $GITHUB_ENV
+            echo "WAREHOUSE=DBT_TEST" >> $GITHUB_ENV
+          else
+            echo "DATABASE=${PROFILE_NAME}_DEV" >> $GITHUB_ENV
+            echo "ROLE=INTERNAL_DEV" >> $GITHUB_ENV
+            echo "WAREHOUSE=${{ inputs.warehouse }}" >> $GITHUB_ENV
+          fi
+          
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python_version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      
+      - name: Run DBT Jobs
+        run: |
+          ${{ inputs.command }}
+          
+      - name: Store logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: dbt-logs
+          path: |
+            logs
+            target

macros/functions/streamline/models.sql

@@ -1,77 +1,70 @@
 {% macro streamline_external_table_query_v2(
         model,
-        partition_function,
+        partition_function
-        partition_name,
+    ) %}
-        other_cols
+
-) %}
+      {% set days = var("BRONZE_LOOKBACK_DAYS")%}
-WITH meta AS (
+
-    SELECT
+    WITH meta AS (
-        LAST_MODIFIED::timestamp_ntz AS _inserted_timestamp,
+        SELECT
-        file_name,
+            last_modified AS inserted_timestamp,
-        {{ partition_function }} AS {{ partition_name }}
+            file_name,
-    FROM
+            {{ partition_function }} AS partition_key
-        TABLE(
+        FROM
-            information_schema.external_table_file_registration_history(
+            TABLE(
-                start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()),
+                information_schema.external_table_file_registration_history(
-                table_name => '{{ source( "bronze_streamline", model) }}')
+                    start_time => DATEADD('day', -ABS({{days}}), CURRENT_TIMESTAMP()),
-            ) A
+                    table_name => '{{ source( "bronze_streamline", model) }}')
-)
+                ) A
-SELECT
+            )
-    {{ other_cols }},
+        SELECT
-    _inserted_timestamp,
+            s.*,
-    s.{{ partition_name }},
+            b.file_name,
-    s.value AS VALUE,
+            inserted_timestamp
-    file_name
+        FROM
-FROM
+            {{ source(
-    {{ source(
+                "bronze_streamline",
-        "bronze_streamline",
+                model
-        model
+            ) }}
-    ) }}
+            s
-    s
+            JOIN meta b
-JOIN 
+            ON b.file_name = metadata$filename
-    meta b
+            AND b.partition_key = s.partition_key
-    ON b.file_name = metadata$filename
+        WHERE
-    AND b.{{ partition_name }} = s.{{ partition_name }}
+            b.partition_key = s.partition_key
-WHERE
+            AND DATA :error IS NULL
-    b.{{ partition_name }} = s.{{ partition_name }}
-    
 {% endmacro %}
 
 {% macro streamline_external_table_FR_query_v2(
         model,
-        partition_function,
+        partition_function
-        partition_name,
+    ) %}
-        other_cols
+    WITH meta AS (
-) %}
+        SELECT
-WITH meta AS (
+            registered_on AS inserted_timestamp,
-    SELECT
+            file_name,
-        LAST_MODIFIED::timestamp_ntz AS _inserted_timestamp,
+            {{ partition_function }} AS partition_key
-        file_name,
+        FROM
-        {{ partition_function }} AS {{ partition_name }}
+            TABLE(
-    FROM
+                information_schema.external_table_files(
-        TABLE(
+                    table_name => '{{ source( "bronze_streamline", model) }}'
-            information_schema.external_table_files(
+                )
-                table_name => '{{ source( "bronze_streamline", model) }}'
+            ) A
-            )
+    )
-        ) A
-)
 SELECT
-    {{ other_cols }},
+    s.*,
-    _inserted_timestamp,
+    b.file_name,
-    s.{{ partition_name }},
+    inserted_timestamp
-    s.value AS VALUE,
-    file_name
 FROM
     {{ source(
         "bronze_streamline",
         model
     ) }}
     s
-JOIN 
+    JOIN meta b
-    meta b
     ON b.file_name = metadata$filename
-    AND b.{{ partition_name }} = s.{{ partition_name }}
+    AND b.partition_key = s.partition_key
 WHERE
-    b.{{ partition_name }} = s.{{ partition_name }}
+    b.partition_key = s.partition_key
-    
+    AND DATA :error IS NULL
 {% endmacro %}