From 841d0b56a5c9dbac3069aab4f25fd3033bf93848 Mon Sep 17 00:00:00 2001 From: WHYTEWYLL Date: Tue, 5 Sep 2023 15:14:40 +0200 Subject: [PATCH] add: obs models test --- models/descriptions/blocks_impacted_array.md | 5 +++ models/descriptions/blocks_impacted_count.md | 6 +++ models/descriptions/blocks_tested.md | 5 +++ models/descriptions/max_block.md | 5 +++ models/descriptions/max_block_timestamp.md | 5 +++ models/descriptions/min_block.md | 5 +++ models/descriptions/min_block_timestamp.md | 5 +++ models/descriptions/test_name.md | 5 +++ models/descriptions/test_timestamp.md | 5 +++ ...ver_observability__blocks_completeness.yml | 43 +++++++++++++++++++ ...ver_observability__chunks_completeness.yml | 43 +++++++++++++++++++ ...ilver_observability__logs_completeness.yml | 43 +++++++++++++++++++ 12 files changed, 175 insertions(+) create mode 100644 models/descriptions/blocks_impacted_array.md create mode 100644 models/descriptions/blocks_impacted_count.md create mode 100644 models/descriptions/blocks_tested.md create mode 100644 models/descriptions/max_block.md create mode 100644 models/descriptions/max_block_timestamp.md create mode 100644 models/descriptions/min_block.md create mode 100644 models/descriptions/min_block_timestamp.md create mode 100644 models/descriptions/test_name.md create mode 100644 models/descriptions/test_timestamp.md create mode 100644 models/silver/_observability/silver_observability__blocks_completeness.yml create mode 100644 models/silver/_observability/silver_observability__chunks_completeness.yml create mode 100644 models/silver/_observability/silver_observability__logs_completeness.yml diff --git a/models/descriptions/blocks_impacted_array.md b/models/descriptions/blocks_impacted_array.md new file mode 100644 index 0000000..314e28a --- /dev/null +++ b/models/descriptions/blocks_impacted_array.md @@ -0,0 +1,5 @@ +{% docs blocks_impacted_array %} + +An array of the impacted blocks during the observability run. + +{% enddocs %} diff --git a/models/descriptions/blocks_impacted_count.md b/models/descriptions/blocks_impacted_count.md new file mode 100644 index 0000000..e1d52dc --- /dev/null +++ b/models/descriptions/blocks_impacted_count.md @@ -0,0 +1,6 @@ +{% docs blocks_impacted_count %} + +The number of blocks impacted during the observability run. +Impacted is defined as missing or having bad data. + +{% enddocs %} diff --git a/models/descriptions/blocks_tested.md b/models/descriptions/blocks_tested.md new file mode 100644 index 0000000..c426178 --- /dev/null +++ b/models/descriptions/blocks_tested.md @@ -0,0 +1,5 @@ +{% docs blocks_tested %} + +The number of blocks tested in the observability run. + +{% enddocs %} diff --git a/models/descriptions/max_block.md b/models/descriptions/max_block.md new file mode 100644 index 0000000..c7e1b1e --- /dev/null +++ b/models/descriptions/max_block.md @@ -0,0 +1,5 @@ +{% docs max_block %} + +The maximum block tested in the range of the observability test run. + +{% enddocs %} diff --git a/models/descriptions/max_block_timestamp.md b/models/descriptions/max_block_timestamp.md new file mode 100644 index 0000000..0a4807b --- /dev/null +++ b/models/descriptions/max_block_timestamp.md @@ -0,0 +1,5 @@ +{% docs max_block_timestamp %} + +The block timestamp of the max block tested in the observability run. + +{% enddocs %} diff --git a/models/descriptions/min_block.md b/models/descriptions/min_block.md new file mode 100644 index 0000000..a85338c --- /dev/null +++ b/models/descriptions/min_block.md @@ -0,0 +1,5 @@ +{% docs min_block %} + +The minimum block tested in the range of the observability test run. + +{% enddocs %} diff --git a/models/descriptions/min_block_timestamp.md b/models/descriptions/min_block_timestamp.md new file mode 100644 index 0000000..d4e1c6e --- /dev/null +++ b/models/descriptions/min_block_timestamp.md @@ -0,0 +1,5 @@ +{% docs min_block_timestamp %} + +The block timestamp of the min block tested in the observability run. + +{% enddocs %} diff --git a/models/descriptions/test_name.md b/models/descriptions/test_name.md new file mode 100644 index 0000000..9881eab --- /dev/null +++ b/models/descriptions/test_name.md @@ -0,0 +1,5 @@ +{% docs test_name %} + +The name of the test for observability models. + +{% enddocs %} diff --git a/models/descriptions/test_timestamp.md b/models/descriptions/test_timestamp.md new file mode 100644 index 0000000..04a62ac --- /dev/null +++ b/models/descriptions/test_timestamp.md @@ -0,0 +1,5 @@ +{% docs test_timestamp %} + +The timestamp of the observability test run. + +{% enddocs %} diff --git a/models/silver/_observability/silver_observability__blocks_completeness.yml b/models/silver/_observability/silver_observability__blocks_completeness.yml new file mode 100644 index 0000000..9fa1c83 --- /dev/null +++ b/models/silver/_observability/silver_observability__blocks_completeness.yml @@ -0,0 +1,43 @@ +version: 2 + +models: + - name: silver_observability__blocks_completeness + description: |- + Observability model that queries the blocks table at a designated interval to record and track the completeness of the data. + tests: + - dbt_utils.recency: + datepart: day + field: test_timestamp + interval: 1 + - dbt_utils.recency: + datepart: hours + field: max_block_timestamp + interval: 24 + + columns: + - name: TEST_NAME + description: "{{ doc('test_name') }}" + + - name: MIN_BLOCK + description: "{{ doc('min_block') }}" + + - name: MAX_BLOCK + description: "{{ doc('max_block') }}" + + - name: MIN_BLOCK_TIMESTAMP + description: "{{ doc('min_block_timestamp') }}" + + - name: MAX_BLOCK_TIMESTAMP + description: "{{ doc('max_block_timestamp') }}" + + - name: BLOCKS_TESTED + description: "{{ doc('blocks_tested') }}" + + - name: BLOCKS_IMPACTED_COUNT + description: "{{ doc('blocks_impacted_count') }}" + + - name: BLOCKS_IMPACTED_ARRAY + description: "{{ doc('blocks_impacted_array') }}" + + - name: TEST_TIMESTAMP + description: "{{ doc('test_timestamp') }}" diff --git a/models/silver/_observability/silver_observability__chunks_completeness.yml b/models/silver/_observability/silver_observability__chunks_completeness.yml new file mode 100644 index 0000000..8ff9435 --- /dev/null +++ b/models/silver/_observability/silver_observability__chunks_completeness.yml @@ -0,0 +1,43 @@ +version: 2 + +models: + - name: silver_observability__chunks_completeness + description: |- + Observability model that queries the blocks table at a designated interval to record and track the completeness of the data. + tests: + - dbt_utils.recency: + datepart: day + field: test_timestamp + interval: 1 + - dbt_utils.recency: + datepart: hours + field: max_block_timestamp + interval: 24 + + columns: + - name: TEST_NAME + description: "{{ doc('test_name') }}" + + - name: MIN_BLOCK + description: "{{ doc('min_block') }}" + + - name: MAX_BLOCK + description: "{{ doc('max_block') }}" + + - name: MIN_BLOCK_TIMESTAMP + description: "{{ doc('min_block_timestamp') }}" + + - name: MAX_BLOCK_TIMESTAMP + description: "{{ doc('max_block_timestamp') }}" + + - name: BLOCKS_TESTED + description: "{{ doc('blocks_tested') }}" + + - name: BLOCKS_IMPACTED_COUNT + description: "{{ doc('blocks_impacted_count') }}" + + - name: BLOCKS_IMPACTED_ARRAY + description: "{{ doc('blocks_impacted_array') }}" + + - name: TEST_TIMESTAMP + description: "{{ doc('test_timestamp') }}" diff --git a/models/silver/_observability/silver_observability__logs_completeness.yml b/models/silver/_observability/silver_observability__logs_completeness.yml new file mode 100644 index 0000000..a29e992 --- /dev/null +++ b/models/silver/_observability/silver_observability__logs_completeness.yml @@ -0,0 +1,43 @@ +version: 2 + +models: + - name: silver_observability__logs_completeness + description: |- + Observability model that queries the blocks table at a designated interval to record and track the completeness of the data. + tests: + - dbt_utils.recency: + datepart: day + field: test_timestamp + interval: 1 + - dbt_utils.recency: + datepart: hours + field: max_block_timestamp + interval: 24 + + columns: + - name: TEST_NAME + description: "{{ doc('test_name') }}" + + - name: MIN_BLOCK + description: "{{ doc('min_block') }}" + + - name: MAX_BLOCK + description: "{{ doc('max_block') }}" + + - name: MIN_BLOCK_TIMESTAMP + description: "{{ doc('min_block_timestamp') }}" + + - name: MAX_BLOCK_TIMESTAMP + description: "{{ doc('max_block_timestamp') }}" + + - name: BLOCKS_TESTED + description: "{{ doc('blocks_tested') }}" + + - name: BLOCKS_IMPACTED_COUNT + description: "{{ doc('blocks_impacted_count') }}" + + - name: BLOCKS_IMPACTED_ARRAY + description: "{{ doc('blocks_impacted_array') }}" + + - name: TEST_TIMESTAMP + description: "{{ doc('test_timestamp') }}"