diff --git a/.github/workflows/dbt_run_ful_observability.yml b/.github/workflows/dbt_run_full_observability.yml similarity index 92% rename from .github/workflows/dbt_run_ful_observability.yml rename to .github/workflows/dbt_run_full_observability.yml index 72e9e7c..ba18275 100644 --- a/.github/workflows/dbt_run_ful_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -37,9 +37,8 @@ jobs: python-version: "3.7.x" - name: install dependencies - run: | - pip install -r requirements.txt + pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml new file mode 100644 index 0000000..0a8765c --- /dev/null +++ b/.github/workflows/dbt_run_observability.yml @@ -0,0 +1,45 @@ +name: dbt_run_full_observability +run-name: dbt_run_full_observability + +on: + workflow_dispatch: + schedule: + # Runs "2 times per day" (see https://crontab.guru) + - cron: "0 6,18 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v1 + with: + python-version: "3.7.x" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":False}' -m tag:observability diff --git a/models/silver/_obervability/silver_observability__blocks_completeness.sql b/models/silver/_obervability/silver_observability__blocks_completeness.sql index 8123fc8..c408802 100644 --- a/models/silver/_obervability/silver_observability__blocks_completeness.sql +++ b/models/silver/_obervability/silver_observability__blocks_completeness.sql @@ -16,7 +16,7 @@ WITH summary_stats AS ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( @@ -30,8 +30,8 @@ AND ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_number @@ -151,7 +151,7 @@ SELECT ORDER BY A.block_number ) AS blocks_impacted_array, - CURRENT_TIMESTAMP AS test_timestamp + SYSDATE() AS test_timestamp FROM block_gen A LEFT JOIN blocks b diff --git a/models/silver/_obervability/silver_observability__logs_completeness.sql b/models/silver/_obervability/silver_observability__logs_completeness.sql index e75acc2..e34008d 100644 --- a/models/silver/_obervability/silver_observability__logs_completeness.sql +++ b/models/silver/_obervability/silver_observability__logs_completeness.sql @@ -16,7 +16,7 @@ WITH summary_stats AS ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( @@ -30,8 +30,8 @@ AND ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_number @@ -115,7 +115,7 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp + SYSDATE() AS test_timestamp FROM summary_stats JOIN impacted_blocks diff --git a/models/silver/_obervability/silver_observability__receipts_completeness.sql b/models/silver/_obervability/silver_observability__receipts_completeness.sql index 1661262..726ae09 100644 --- a/models/silver/_obervability/silver_observability__receipts_completeness.sql +++ b/models/silver/_obervability/silver_observability__receipts_completeness.sql @@ -16,7 +16,7 @@ WITH summary_stats AS ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( @@ -30,8 +30,8 @@ AND ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_number @@ -113,7 +113,7 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp + SYSDATE() AS test_timestamp FROM summary_stats JOIN impacted_blocks diff --git a/models/silver/_obervability/silver_observability__transactions_completeness.sql b/models/silver/_obervability/silver_observability__transactions_completeness.sql index 60d9bb5..c984e8f 100644 --- a/models/silver/_obervability/silver_observability__transactions_completeness.sql +++ b/models/silver/_obervability/silver_observability__transactions_completeness.sql @@ -16,7 +16,7 @@ WITH summary_stats AS ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) + block_timestamp <= DATEADD('hour', -12, SYSDATE()) {% if is_incremental() %} AND ( @@ -30,8 +30,8 @@ AND ( FROM {{ ref('silver__blocks') }} WHERE - block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) - AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) + block_timestamp BETWEEN DATEADD('hour', -96, SYSDATE()) + AND DATEADD('hour', -95, SYSDATE()) UNION SELECT MIN(VALUE) - 1 AS block_number @@ -110,7 +110,7 @@ SELECT blocks_tested, blocks_impacted_count, blocks_impacted_array, - CURRENT_TIMESTAMP() AS test_timestamp + SYSDATE() AS test_timestamp FROM summary_stats JOIN impacted_blocks