add: action, sysdate, fix typo

.github/workflows/dbt_run_full_observability.yml

@@ -0,0 +1,46 @@
+name: dbt_run_full_observability
+run-name: dbt_run_full_observability
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs “At 00:00 on day-of-month 1.” (see https://crontab.guru)
+    - cron: "0 0 1 * *"
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ inputs.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m tag:observability

models/silver/_observability/silver_observability__blocks_completeness.sql

@@ -18,7 +18,7 @@ WITH blocks_joined AS (
     LEFT JOIN near_dev.silver.streamline_blocks b
     ON a.BLOCK_HASH = b.PREV_HASH
     WHERE a.BLOCK_TIMESTAMP < b.BLOCK_TIMESTAMP -- Ensuring temporal order
-    AND   a.BLOCK_TIMESTAMP <= DATEADD('hour', -12, CURRENT_TIMESTAMP())
+    AND   a.BLOCK_TIMESTAMP <= DATEADD('hour', -12, SYSDATE())
 {% if is_incremental() %}
 AND (
     a.block_id >= (
@@ -31,8 +31,8 @@ AND (
                 FROM
                     {{ ref('silver__streamline_blocks') }}
                 WHERE
-                    block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP())
-                    AND DATEADD('hour', -95, CURRENT_TIMESTAMP())
+                    block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE())
+                    AND DATEADD('hour', -95, SYSDATE())
                 UNION
                 SELECT
                     MIN(VALUE) - 1 AS block_id
@@ -52,7 +52,7 @@ AND (
                     )
             )
     ) {% if var('OBSERV_FULL_TEST') %}
-        OR b.block_id >= 0
+        OR b.block_id >= 9820210
     {% endif %}
 )
 {% endif %}
@@ -90,5 +90,5 @@ SELECT
     blocks_tested,
     blocks_impacted_count,
     blocks_impacted_array,
-    CURRENT_TIMESTAMP() AS test_timestamp
+    SYSDATE() AS test_timestamp
 FROM aggregated_data

models/silver/_observability/silver_observability__chunks_completeness.sql

@@ -18,7 +18,7 @@ WITH block_chunks_included AS (
     FROM
         {{ ref('silver__streamline_blocks') }}
     WHERE
-        block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP())
+        block_timestamp <= DATEADD('hour', -12, SYSDATE())
 {% if is_incremental() %}
 AND (
     block_id >= (
@@ -31,8 +31,8 @@ AND (
                 FROM
                     {{ ref('silver__streamline_blocks') }}
                 WHERE
-                    block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP())
-                    AND DATEADD('hour', -95, CURRENT_TIMESTAMP())
+                    block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE())
+                    AND DATEADD('hour', -95, SYSDATE())
                 UNION
                 SELECT
                     MIN(VALUE) - 1 AS block_id
@@ -52,7 +52,7 @@ AND (
                     )
             )
     ) {% if var('OBSERV_FULL_TEST') %}
-        OR block_id >= 0
+        OR block_id >= 9820210
     {% endif %}
 )
 {% endif %}
@@ -109,8 +109,8 @@ missing AS (
             OR cblock_id IS NULL
             OR chunks_included != chunk_ct
         )
-        AND b_inserted_timestamp <= CURRENT_TIMESTAMP - INTERVAL '1 hour'
-        AND c_inserted_timestamp <= CURRENT_TIMESTAMP - INTERVAL '1 hour'
+        AND b_inserted_timestamp <= SYSDATE() - INTERVAL '1 hour'
+        AND c_inserted_timestamp <= SYSDATE() - INTERVAL '1 hour'
     ORDER BY
         1
 ),
@@ -126,7 +126,7 @@ impacted_blocks AS (
 )
 
 SELECT
-    'chuncks' AS test_name,
+    'chunk' AS test_name,
     min_block,
     max_block,
     min_block_timestamp,
@@ -134,7 +134,7 @@ SELECT
     blocks_tested,
     blocks_impacted_count,
     blocks_impacted_array,
-    CURRENT_TIMESTAMP() AS test_timestamp
+    SYSDATE() AS test_timestamp
 FROM
     summary_stats
     JOIN impacted_blocks

models/silver/_observability/silver_observability__logs_completeness.sql

@@ -16,7 +16,7 @@ WITH summary_stats AS (
     FROM
         {{ ref('silver__streamline_blocks') }}
     WHERE
-        block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP())
+        block_timestamp <= DATEADD('hour', -12, SYSDATE())
 
 {% if is_incremental() %}
 AND (
@@ -30,8 +30,8 @@ AND (
                 FROM
                     {{ ref('silver__streamline_blocks') }}
                 WHERE
-                    block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP())
-                    AND DATEADD('hour', -95, CURRENT_TIMESTAMP())
+                    block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE())
+                    AND DATEADD('hour', -95, SYSDATE())
                 UNION
                 SELECT
                     MIN(VALUE) - 1 AS block_id
@@ -51,7 +51,7 @@ AND (
                     )
             )
     ) {% if var('OBSERV_FULL_TEST') %}
-        OR block_id >= 0
+        OR block_id >= 9820210
     {% endif %}
 )
 {% endif %}
@@ -115,7 +115,7 @@ SELECT
     blocks_tested,
     blocks_impacted_count,
     blocks_impacted_array,
-    CURRENT_TIMESTAMP() AS test_timestamp
+    SYSDATE() AS test_timestamp
 FROM
     summary_stats
     JOIN impacted_blocks