diff --git a/.github/workflows/dbt_run_temp_reload_traces.yml b/.github/workflows/dbt_run_temp_reload_traces.yml deleted file mode 100644 index e08a373..0000000 --- a/.github/workflows/dbt_run_temp_reload_traces.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_temp_reload_traces -run-name: dbt_run_temp_reload_traces - -on: - workflow_dispatch: - schedule: - # Runs “At minute 39,” (see https://crontab.guru) - - cron: '39 * * * *' - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_2xl - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "blast_models,tag:traces2" \ No newline at end of file diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 8ec29fb..aa4c97f 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -5,8 +5,9 @@ unique_key = "block_number", cluster_by = ['modified_timestamp::DATE','partition_key'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['traces2'] -) }} + tags = ['non_realtime'], + full_refresh = false +) }} WITH bronze_traces AS ( @@ -25,17 +26,11 @@ WITH bronze_traces AS ( FROM {% if is_incremental() %} -{{ ref('bronze__streamline_FR_traces') }} +{{ ref('bronze__streamline_traces') }} WHERE - partition_key BETWEEN ( + _inserted_timestamp >= ( SELECT - MAX(partition_key) - FROM - {{ this }} - ) - AND ( - SELECT - MAX(partition_key) + 1000000 + MAX(_inserted_timestamp) _inserted_timestamp FROM {{ this }} )