From 975f4b323367f06be89c9e025635d212f50cd09b Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Wed, 19 Mar 2025 13:05:45 -0400 Subject: [PATCH] wf, yml --- .github/workflows/dbt_run_observability.yml | 45 ++++++++++++ .../workflows/dbt_run_observability_full.yml | 45 ++++++++++++ ...er_observability__ledgers_completeness.sql | 2 +- ...servability__ledgers_xref_completeness.sql | 2 +- ...servability__ledgers_xref_completeness.yml | 68 +++++++++++++++++++ ...servability__transactions_completeness.sql | 4 +- ...servability__transactions_completeness.yml | 59 +++++++++++++--- 7 files changed, 210 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/dbt_run_observability.yml create mode 100644 .github/workflows/dbt_run_observability_full.yml create mode 100644 models/silver/_observability/silver_observability__ledgers_xref_completeness.yml diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml new file mode 100644 index 0000000..7fafcb0 --- /dev/null +++ b/.github/workflows/dbt_run_observability.yml @@ -0,0 +1,45 @@ +name: dbt_run_observability +run-name: dbt_run_observability + +on: + workflow_dispatch: + schedule: + # Runs “At minute 40 past every 8th hour.” (see https://crontab.guru) + - cron: '0 */8 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt build -s "stellar_models,tag:observability" \ No newline at end of file diff --git a/.github/workflows/dbt_run_observability_full.yml b/.github/workflows/dbt_run_observability_full.yml new file mode 100644 index 0000000..5b65f02 --- /dev/null +++ b/.github/workflows/dbt_run_observability_full.yml @@ -0,0 +1,45 @@ +name: dbt_run_observability_full +run-name: dbt_run_observability_full + +on: + workflow_dispatch: + schedule: + # Runs “At 19:00 on day-of-month 1.” (see https://crontab.guru) + - cron: '0 19 1 * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt build --vars '{"OBSERV_FULL_TEST":True}' -s "stellar_models,tag:observability" \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__ledgers_completeness.sql b/models/silver/_observability/silver_observability__ledgers_completeness.sql index 1d6ce26..bf27cb8 100644 --- a/models/silver/_observability/silver_observability__ledgers_completeness.sql +++ b/models/silver/_observability/silver_observability__ledgers_completeness.sql @@ -94,7 +94,7 @@ SELECT MAX( b.block_timestamp ) AS max_block_timestamp, - COUNT(1) AS sequences_tested, + COUNT(1) AS ledgers_tested, COUNT( CASE WHEN C.sequence IS NOT NULL THEN A.sequence diff --git a/models/silver/_observability/silver_observability__ledgers_xref_completeness.sql b/models/silver/_observability/silver_observability__ledgers_xref_completeness.sql index 71d3148..c66df9c 100644 --- a/models/silver/_observability/silver_observability__ledgers_xref_completeness.sql +++ b/models/silver/_observability/silver_observability__ledgers_xref_completeness.sql @@ -118,7 +118,7 @@ SELECT MAX( block_timestamp ) AS max_block_timestamp, - COUNT(1) AS sequences_tested, + COUNT(1) AS ledgers_tested, COUNT( CASE WHEN bq.successful_transaction_count <> rpc.successful_transaction_count diff --git a/models/silver/_observability/silver_observability__ledgers_xref_completeness.yml b/models/silver/_observability/silver_observability__ledgers_xref_completeness.yml new file mode 100644 index 0000000..dcb90bc --- /dev/null +++ b/models/silver/_observability/silver_observability__ledgers_xref_completeness.yml @@ -0,0 +1,68 @@ +version: 2 +models: + - name: silver_observability__ledgers_xref_completeness + description: Records of all blocks ledger differences between hubble and rpc + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TEST_TIMESTAMP + columns: + - name: MIN_SEQUENCE + description: The lowest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MAX_SEQUENCE + description: The highest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MIN_BLOCK_TIMESTAMP + description: The lowest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: MAX_BLOCK_TIMESTAMP + description: The highest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: LEDGERS_TESTED + description: Count of blocks in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LEDGERS_IMPACTED_COUNT + description: Count of block gaps in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LEDGERS_IMPACTED_ARRAY + description: Array of affected blocks + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - name: TEST_TIMESTAMP + description: When the test was run + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ \ No newline at end of file diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 56ac630..6f08988 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -20,9 +20,9 @@ WITH summary_stats AS ( {% if is_incremental() %} AND ( - block_number >= ( + SEQUENCE >= ( SELECT - MIN(block_number) + MIN(SEQUENCE) FROM ( SELECT diff --git a/models/silver/_observability/silver_observability__transactions_completeness.yml b/models/silver/_observability/silver_observability__transactions_completeness.yml index 9d4736b..e035e41 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.yml +++ b/models/silver/_observability/silver_observability__transactions_completeness.yml @@ -1,25 +1,62 @@ version: 2 models: - name: silver_observability__transactions_completeness - description: Records of all block transaction counts. This is an intermediate table used for transaction completeness testing + description: Records of all block transaction counts. columns: - - name: BLOCK_ID - description: The lowest block id in the test - tests: - - not_null - - unique - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - name: TRANSACTION_COUNT + - name: MIN_SEQUENCE description: The lowest block id in the test tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - - name: _INSERTED_TIMESTAMP + - name: MAX_SEQUENCE + description: The highest block id in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: MIN_BLOCK_TIMESTAMP description: The lowest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: MAX_BLOCK_TIMESTAMP + description: The highest block timestamp in the test + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 2 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + - name: LEDGERS_TESTED + description: Count of blocks in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LEDGERS_IMPACTED_COUNT + description: Count of block gaps in the test + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LEDGERS_IMPACTED_ARRAY + description: Array of affected blocks + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - ARRAY + - name: TEST_TIMESTAMP + description: When the test was run tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: