diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml new file mode 100644 index 0000000..72e9e7c --- /dev/null +++ b/.github/workflows/dbt_run_full_observability.yml @@ -0,0 +1,46 @@ +name: dbt_run_full_observability +run-name: dbt_run_full_observability + +on: + workflow_dispatch: + schedule: + # Runs “At 00:00 on day-of-month 1.” (see https://crontab.guru) + - cron: "0 0 1 * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m tag:observability diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 55968e6..3ee96c8 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -18,7 +18,7 @@ WITH blocks_joined AS ( LEFT JOIN near_dev.silver.streamline_blocks b ON a.BLOCK_HASH = b.PREV_HASH WHERE a.BLOCK_TIMESTAMP < b.BLOCK_TIMESTAMP -- Ensuring temporal order - AND a.BLOCK_TIMESTAMP <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + AND a.BLOCK_TIMESTAMP <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( a.block_id >= ( @@ -31,8 +31,8 @@ AND ( FROM {{ ref('silver__streamline_blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_id @@ -52,7 +52,7 @@ AND ( ) ) ) {% if var('OBSERV_FULL_TEST') %} - OR b.block_id >= 0 + OR b.block_id >= 9820210 {% endif %} ) {% endif %} @@ -90,5 +90,5 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp + SYSDATE() AS test_timestamp FROM aggregated_data \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__chuncks_completeness.sql b/models/silver/_observability/silver_observability__chunks_completeness.sql similarity index 89% rename from models/silver/_observability/silver_observability__chuncks_completeness.sql rename to models/silver/_observability/silver_observability__chunks_completeness.sql index 5d56ab1..93ed6f0 100644 --- a/models/silver/_observability/silver_observability__chuncks_completeness.sql +++ b/models/silver/_observability/silver_observability__chunks_completeness.sql @@ -18,7 +18,7 @@ WITH block_chunks_included AS ( FROM {{ ref('silver__streamline_blocks') }} WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( block_id >= ( @@ -31,8 +31,8 @@ AND ( FROM {{ ref('silver__streamline_blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_id @@ -52,7 +52,7 @@ AND ( ) ) ) {% if var('OBSERV_FULL_TEST') %} - OR block_id >= 0 + OR block_id >= 9820210 {% endif %} ) {% endif %} @@ -109,8 +109,8 @@ missing AS ( OR cblock_id IS NULL OR chunks_included != chunk_ct ) - AND b_inserted_timestamp <= CURRENT_TIMESTAMP - INTERVAL '1 hour' - AND c_inserted_timestamp <= CURRENT_TIMESTAMP - INTERVAL '1 hour' + AND b_inserted_timestamp <= SYSDATE() - INTERVAL '1 hour' + AND c_inserted_timestamp <= SYSDATE() - INTERVAL '1 hour' ORDER BY 1 ), @@ -126,7 +126,7 @@ impacted_blocks AS ( ) SELECT - 'chuncks' AS test_name, + 'chunk' AS test_name, min_block, max_block, min_block_timestamp, @@ -134,7 +134,7 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp + SYSDATE() AS test_timestamp FROM summary_stats JOIN impacted_blocks diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql index c08ac1a..8198136 100644 --- a/models/silver/_observability/silver_observability__logs_completeness.sql +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -16,7 +16,7 @@ WITH summary_stats AS ( FROM {{ ref('silver__streamline_blocks') }} WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( @@ -30,8 +30,8 @@ AND ( FROM {{ ref('silver__streamline_blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_id @@ -51,7 +51,7 @@ AND ( ) ) ) {% if var('OBSERV_FULL_TEST') %} - OR block_id >= 0 + OR block_id >= 9820210 {% endif %} ) {% endif %} @@ -115,7 +115,7 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp + SYSDATE() AS test_timestamp FROM summary_stats JOIN impacted_blocks