mirror of
https://github.com/FlipsideCrypto/stellar-models.git
synced 2026-02-06 15:36:45 +00:00
wf, yml
This commit is contained in:
Eric Laurello
parent
8efcef0a3a
commit
975f4b3233
45
.github/workflows/dbt_run_observability.yml
vendored
Normal file
45
.github/workflows/dbt_run_observability.yml
vendored
Normal file
@ -0,0 +1,45 @@
|
||||
name: dbt_run_observability
|
||||
run-name: dbt_run_observability
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Runs “At minute 40 past every 8th hour.” (see https://crontab.guru)
|
||||
- cron: '0 */8 * * *'
|
||||
|
||||
env:
|
||||
DBT_PROFILES_DIR: ./
|
||||
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt build -s "stellar_models,tag:observability"
|
||||
45
.github/workflows/dbt_run_observability_full.yml
vendored
Normal file
45
.github/workflows/dbt_run_observability_full.yml
vendored
Normal file
@ -0,0 +1,45 @@
|
||||
name: dbt_run_observability_full
|
||||
run-name: dbt_run_observability_full
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Runs “At 19:00 on day-of-month 1.” (see https://crontab.guru)
|
||||
- cron: '0 19 1 * *'
|
||||
|
||||
env:
|
||||
DBT_PROFILES_DIR: ./
|
||||
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt build --vars '{"OBSERV_FULL_TEST":True}' -s "stellar_models,tag:observability"
|
||||
@ -94,7 +94,7 @@ SELECT
|
||||
MAX(
|
||||
b.block_timestamp
|
||||
) AS max_block_timestamp,
|
||||
COUNT(1) AS sequences_tested,
|
||||
COUNT(1) AS ledgers_tested,
|
||||
COUNT(
|
||||
CASE
|
||||
WHEN C.sequence IS NOT NULL THEN A.sequence
|
||||
|
||||
@ -118,7 +118,7 @@ SELECT
|
||||
MAX(
|
||||
block_timestamp
|
||||
) AS max_block_timestamp,
|
||||
COUNT(1) AS sequences_tested,
|
||||
COUNT(1) AS ledgers_tested,
|
||||
COUNT(
|
||||
CASE
|
||||
WHEN bq.successful_transaction_count <> rpc.successful_transaction_count
|
||||
|
||||
@ -0,0 +1,68 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: silver_observability__ledgers_xref_completeness
|
||||
description: Records of all blocks ledger differences between hubble and rpc
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- TEST_TIMESTAMP
|
||||
columns:
|
||||
- name: MIN_SEQUENCE
|
||||
description: The lowest block id in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: MAX_SEQUENCE
|
||||
description: The highest block id in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: MIN_BLOCK_TIMESTAMP
|
||||
description: The lowest block timestamp in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- name: MAX_BLOCK_TIMESTAMP
|
||||
description: The highest block timestamp in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- name: LEDGERS_TESTED
|
||||
description: Count of blocks in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: LEDGERS_IMPACTED_COUNT
|
||||
description: Count of block gaps in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: LEDGERS_IMPACTED_ARRAY
|
||||
description: Array of affected blocks
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- ARRAY
|
||||
- name: TEST_TIMESTAMP
|
||||
description: When the test was run
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
@ -20,9 +20,9 @@ WITH summary_stats AS (
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND (
|
||||
block_number >= (
|
||||
SEQUENCE >= (
|
||||
SELECT
|
||||
MIN(block_number)
|
||||
MIN(SEQUENCE)
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
|
||||
@ -1,25 +1,62 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: silver_observability__transactions_completeness
|
||||
description: Records of all block transaction counts. This is an intermediate table used for transaction completeness testing
|
||||
description: Records of all block transaction counts.
|
||||
columns:
|
||||
- name: BLOCK_ID
|
||||
description: The lowest block id in the test
|
||||
tests:
|
||||
- not_null
|
||||
- unique
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: TRANSACTION_COUNT
|
||||
- name: MIN_SEQUENCE
|
||||
description: The lowest block id in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: _INSERTED_TIMESTAMP
|
||||
- name: MAX_SEQUENCE
|
||||
description: The highest block id in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: MIN_BLOCK_TIMESTAMP
|
||||
description: The lowest block timestamp in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- name: MAX_BLOCK_TIMESTAMP
|
||||
description: The highest block timestamp in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_row_values_to_have_recent_data:
|
||||
datepart: day
|
||||
interval: 2
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- TIMESTAMP_NTZ
|
||||
- name: LEDGERS_TESTED
|
||||
description: Count of blocks in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: LEDGERS_IMPACTED_COUNT
|
||||
description: Count of block gaps in the test
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- name: LEDGERS_IMPACTED_ARRAY
|
||||
description: Array of affected blocks
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- ARRAY
|
||||
- name: TEST_TIMESTAMP
|
||||
description: When the test was run
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user