regular inc (#181)

.github/workflows/dbt_run_temp_reload_traces.yml

@@ -1,45 +0,0 @@
-name: dbt_run_temp_reload_traces
-run-name: dbt_run_temp_reload_traces
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Runs “At minute 49,” (see https://crontab.guru)
-    - cron: '49 * * * *'
-    
-env:
-  DBT_PROFILES_DIR: ./
-
-  ACCOUNT: "${{ vars.ACCOUNT }}"
-  ROLE: "${{ vars.ROLE }}"
-  USER: "${{ vars.USER }}"
-  PASSWORD: "${{ secrets.PASSWORD }}"
-  REGION: "${{ vars.REGION }}"
-  DATABASE: "${{ vars.DATABASE }}"
-  WAREHOUSE: "${{ vars.WAREHOUSE }}"
-  SCHEMA: "${{ vars.SCHEMA }}"
-
-concurrency:
-  group: ${{ github.workflow }}
-
-jobs:
-  run_dbt_jobs:
-    runs-on: ubuntu-latest
-    environment: 
-      name: workflow_prod_2xl
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: install dependencies
-        run: |
-          pip install -r requirements.txt
-          dbt deps
-      - name: Run DBT Jobs
-        run: |
-          dbt run -m "base_models,tag:traces2"

models/silver/core/silver__traces2.sql

@@ -5,7 +5,8 @@
     unique_key = "block_number",
     cluster_by = ['modified_timestamp::DATE','partition_key'],
     post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
-    tags = ['traces2']
+    full_refresh = false,
+    tags = ['core','non_realtime']
 ) }}
 
 WITH bronze_traces AS (
@@ -19,17 +20,11 @@ WITH bronze_traces AS (
     FROM
 
 {% if is_incremental() %}
-{{ ref('bronze__streamline_FR_traces') }}
+{{ ref('bronze__streamline_traces') }}
 WHERE
-    _partition_by_block_id BETWEEN (
+    _inserted_timestamp >= (
         SELECT
-            MAX(partition_key)
-        FROM
-            {{ this }}
-    )
-    AND (
-        SELECT
-            MAX(partition_key) + 1000000
+            MAX(_inserted_timestamp) _inserted_timestamp
         FROM
             {{ this }}
     )