From 3c0e18b477d5db360ccfd458aaa9a2fbbb0c5865 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Mon, 7 Apr 2025 11:55:40 -0600 Subject: [PATCH] cutover to gold --- .github/workflows/dbt_run_observability.yml | 45 +++++++++++++++++++ ...ver_observability__blocks_completeness.sql | 6 +-- ...servability__transactions_completeness.sql | 8 ++-- 3 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/dbt_run_observability.yml diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml new file mode 100644 index 0000000..90c6509 --- /dev/null +++ b/.github/workflows/dbt_run_observability.yml @@ -0,0 +1,45 @@ +name: dbt_run_observability +run-name: dbt_run_observability + +on: + workflow_dispatch: + schedule: + # Runs “At minute 0 past every 8th hour.” (see https://crontab.guru) + - cron: '0 */8 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt build -s "movement_models,tag:observability" \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 5c09f85..f3d669b 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -15,7 +15,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) @@ -29,7 +29,7 @@ AND ( SELECT MIN(block_number) AS block_number FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) @@ -91,7 +91,7 @@ blocks AS ( l.block_number ASC ) AS prev_BLOCK_NUMBER FROM - {{ ref("silver__blocks") }} + {{ ref("core__fact_blocks") }} l INNER JOIN block_range b ON l.block_number = b.block_number diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 441e789..3e5d533 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -14,7 +14,7 @@ WITH summary_stats AS ( MAX(block_timestamp) AS max_block_timestamp, COUNT(1) AS blocks_tested FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp <= DATEADD('hour', -12, CURRENT_TIMESTAMP()) @@ -28,7 +28,7 @@ AND ( SELECT MIN(block_number) AS block_number FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_timestamp BETWEEN DATEADD('hour', -96, CURRENT_TIMESTAMP()) AND DATEADD('hour', -95, CURRENT_TIMESTAMP()) @@ -61,7 +61,7 @@ base_blocks AS ( block_number, tx_count_from_versions AS transaction_count FROM - {{ ref('silver__blocks') }} + {{ ref('core__fact_blocks') }} WHERE block_number BETWEEN ( SELECT @@ -81,7 +81,7 @@ actual_tx_counts AS ( block_number, COUNT(1) AS transaction_count FROM - {{ ref('silver__transactions') }} + {{ ref('core__fact_transactions') }} WHERE block_number BETWEEN ( SELECT