adjusted nomenclature

This commit is contained in:
Mike Stepanovic 2025-03-11 12:22:41 -06:00
parent cb783de41e
commit d984974e66
8 changed files with 792 additions and 5 deletions

52
.github/workflows/dbt_test_daily.yml vendored Normal file
View File

@ -0,0 +1,52 @@
name: dbt_test_recent
run-name: dbt_test_recent
on:
workflow_dispatch:
schedule:
- cron: "30 0 * * *"
env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
DBT_VERSION: "${{ vars.DBT_VERSION }}"
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "${{ vars.PYTHON_VERSION }}"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt test -m tag:test_recency tag:test_quality tag:test_integrity --vars '{"TEST_HOURS_THRESHOLD":24}'
continue-on-error: true
- name: Log test results
run: |
python python/dbt_test_alert.py

52
.github/workflows/dbt_test_monthly.yml vendored Normal file
View File

@ -0,0 +1,52 @@
name: dbt_test_recent
run-name: dbt_test_recent
on:
workflow_dispatch:
schedule:
- cron: "0 10 28 * *"
env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
DBT_VERSION: "${{ vars.DBT_VERSION }}"
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "${{ vars.PYTHON_VERSION }}"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt test -m tag:test_recency tag:test_quality tag:test_integrity --vars '{"TEST_HOURS_THRESHOLD":744}'
continue-on-error: true
- name: Log test results
run: |
python python/dbt_test_alert.py

53
.github/workflows/dbt_test_recency.yml vendored Normal file
View File

@ -0,0 +1,53 @@
name: dbt_test_recent
run-name: dbt_test_recent
on:
workflow_dispatch:
schedule:
- cron: "0 */2 * * *"
env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
DBT_VERSION: "${{ vars.DBT_VERSION }}"
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "${{ vars.PYTHON_VERSION }}"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt source freshness
dbt test -m "movement_models,tag:test_recency"
continue-on-error: true
- name: Log test results
run: |
python python/dbt_test_alert.py

52
.github/workflows/dbt_test_weekly.yml vendored Normal file
View File

@ -0,0 +1,52 @@
name: dbt_test_recent
run-name: dbt_test_recent
on:
workflow_dispatch:
schedule:
- cron: "30 0 * * 1"
env:
USE_VARS: "${{ vars.USE_VARS }}"
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
DBT_VERSION: "${{ vars.DBT_VERSION }}"
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "${{ vars.PYTHON_VERSION }}"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt test -m tag:test_recency tag:test_quality tag:test_integrity --vars '{"TEST_HOURS_THRESHOLD":168}'
continue-on-error: true
- name: Log test results
run: |
python python/dbt_test_alert.py

View File

@ -28,7 +28,8 @@ clean-targets: # directories to be removed by `dbt clean`
tests:
+store_failures: true # all tests
# +where: "modified_timestamp > dateadd(hour, -{{ var('TEST_HOURS_THRESHOLD') }}, current_timestamp)"
test_quality:
+where: "modified_timestamp > dateadd(hour, -{{ var('TEST_HOURS_THRESHOLD', 3) }}, current_timestamp)"
on-run-start:
- "{{ create_sps() }}"

View File

@ -2,6 +2,13 @@ version: 2
models:
- name: core__fact_blocks
description: '{{ doc("core__fact_blocks") }}'
tests:
- dbt_utils.recency:
datepart: hour
field: MODIFIED_TIMESTAMP
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_number
description: '{{ doc("block_number") }}'
@ -24,6 +31,13 @@ models:
- name: core__fact_changes
description: '{{ doc("core__fact_changes") }}'
tests:
- dbt_utils.recency:
datepart: hour
field: MODIFIED_TIMESTAMP
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_number
description: '{{ doc("block_number") }}'
@ -72,7 +86,14 @@ models:
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_events
description: '{{ doc("core__fact_events") }}'
description: '{{ doc("core__fact_events") }}'
tests:
- dbt_utils.recency:
datepart: hour
field: MODIFIED_TIMESTAMP
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_number
description: '{{ doc("block_number") }}'
@ -116,7 +137,14 @@ models:
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_transactions_block_metadata
description: '{{ doc("core__fact_transactions_block_metadata") }}'
description: '{{ doc("core__fact_transactions_block_metadata") }}'
tests:
- dbt_utils.recency:
datepart: hour
field: MODIFIED_TIMESTAMP
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_number
description: '{{ doc("block_number") }}'
@ -166,7 +194,14 @@ models:
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_transactions_state_checkpoint
description: '{{ doc("core__fact_transactions_state_checkpoint") }}'
description: '{{ doc("core__fact_transactions_state_checkpoint") }}'
tests:
- dbt_utils.recency:
datepart: hour
field: MODIFIED_TIMESTAMP
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_number
description: '{{ doc("block_number") }}'
@ -196,7 +231,14 @@ models:
description: '{{ doc("modified_timestamp") }}'
- name: core__fact_transactions
description: '{{ doc("core__fact_transactions") }}'
description: '{{ doc("core__fact_transactions") }}'
tests:
- dbt_utils.recency:
datepart: hour
field: MODIFIED_TIMESTAMP
interval: 3
severity: error
tags: ['test_recency']
columns:
- name: block_number
description: '{{ doc("block_number") }}'

View File

@ -0,0 +1,408 @@
version: 2
models:
- name: silver__blocks
tests:
- dbt_utils.sequential_values:
column_name: block_number
interval: 1
config:
severity: error
error_if: ">100"
tags: ['test_recency']
columns:
- name: block_number
tests:
- not_null:
tags: ['test_quality']
- unique:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: block_hash
tests:
- not_null:
tags: ['test_quality']
- unique:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: block_timestamp
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: TIMESTAMP_NTZ
tags: ['test_integrity']
- name: tx_count_from_versions
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: block_timestamp_num
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: first_version
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: last_version
tests:
- not_null:
tags: ['test_quality']
- name: blocks_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id
- name: silver__changes
columns:
- name: block_number
- name: block_timestamp
- name: tx_hash
- name: version
- name: success
- name: tx_type
- name: payload_function
- name: change_index
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: change_data
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: change_type
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: address
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: handle
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: inner_change_type
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: change_address
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: change_module
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: change_resource
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: key
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: value
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: state_key_hash
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: changes_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id
- name: silver__events
columns:
- name: block_number
- name: block_timestamp
- name: tx_hash
- name: version
- name: success
- name: tx_type
- name: payload_function
- name: event_index
- name: event_type
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: event_address
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: event_module
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: event_resource
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: event_data
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: account_address
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: creation_number
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: sequence_number
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: events_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id
- name: silver__transactions
columns:
- name: block_timestamp
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: TIMESTAMP_NTZ
tags: ['test_integrity']
- name: tx_hash
tests:
- not_null:
tags: ['test_quality']
- unique:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: version
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: tx_type
tests:
- not_null:
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: success
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: BOOLEAN
tags: ['test_integrity']
- name: sender
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: signature
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: payload
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: payload_function
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: changes
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: events
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: gas_unit_price
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: gas_used
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: max_gas_amount
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: expiration_timestamp_secs
tests:
- dbt_expectations.expect_column_values_to_be_between:
min_value: 0
tags: ['test_quality']
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: vm_status
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: state_change_hash
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: accumulator_root_hash
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: event_root_hash
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: state_checkpoint_hash
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: failed_proposer_indices
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: id
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: previous_block_votes_bitvec
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: proposer
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARCHAR
tags: ['test_integrity']
- name: ROUND
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: NUMBER
tags: ['test_integrity']
- name: data
tests:
- dbt_expectations.expect_column_values_to_be_of_type:
column_type: VARIANT
tags: ['test_integrity']
- name: transactions_id
- name: inserted_timestamp
- name: modified_timestamp
- name: _invocation_id

127
python/dbt_test_alert.py Normal file
View File

@ -0,0 +1,127 @@
import datetime
import requests
import json
import sys
import os
def log_test_result():
"""Reads the run_results.json file and returns a dictionary of targeted test results"""
filepath = "target/run_results.json"
with open(filepath) as f:
run = json.load(f)
logs = []
messages = {
"fail": [],
"warn": []
}
test_count = 0
warn_count = 0
fail_count = 0
for test in run["results"]:
test_count += 1
if test["status"] != "pass":
logs.append(test)
message = f"{test['failures']} record failure(s) in {test['unique_id']}"
if test["status"] == "warn":
messages["warn"].append(message)
warn_count += 1
elif test["status"] == "fail":
messages["fail"].append(message)
fail_count += 1
dbt_test_result = {
"logs": logs,
"messages": messages,
"test_count": test_count,
"warn_count": warn_count,
"fail_count": fail_count,
"elapsed_time": str(datetime.timedelta(seconds=run["elapsed_time"]))
}
return dbt_test_result
def create_message(**kwargs):
messageBody = {
"text": f"Hey{' <!here>' if len(kwargs['messages']['fail']) > 0 else ''}, new DBT test results for :{os.environ.get('DATABASE').split('_DEV')[0]}: {os.environ.get('DATABASE')}",
"attachments": [
{
"color": kwargs["color"],
"fields": [
{
"title": "Total Tests Run",
"value": kwargs["test_count"],
"short": True
},
{
"title": "Total Time Elapsed",
"value": kwargs["elapsed_time"],
"short": True
},
{
"title": "Number of Unsuccessful Tests",
"value": f"Fail: {kwargs['fail_count']}, Warn: {kwargs['warn_count']}",
"short": True
},
{
"title": "Failed Tests:",
"value": "\n".join(kwargs["messages"]["fail"]) if len(kwargs["messages"]["fail"]) > 0 else "None :)",
"short": False
}
],
"actions": [
{
"type": "button",
"text": "View Warnings",
"style": "primary",
"url": "https://github.com/FlipsideCrypto/movement-models/actions",
"confirm": {
"title": f"{kwargs['warn_count']} Warnings",
"text": "\n".join(kwargs["messages"]["warn"]) if len(kwargs["messages"]["warn"]) > 0 else "None :)",
"ok_text": "Continue to GHA",
"dismiss_text": "Dismiss"
}
}
]
}
]
}
return messageBody
def send_alert(webhook_url):
"""Sends a message to a slack channel"""
url = webhook_url
data = log_test_result()
send_message = create_message(
fail_count=data["fail_count"],
warn_count=data["warn_count"],
test_count=data["test_count"],
messages=data["messages"],
elapsed_time=data["elapsed_time"],
color="#f44336" if data["fail_count"] > 0 else "#4CAF50"
)
x = requests.post(url, json=send_message)
# test config to continue on error in workflow, so we want to exit with a non-zero code if there are any failures
if data['fail_count'] > 0:
sys.exit(1)
if __name__ == '__main__':
webhook_url = os.environ.get("SLACK_WEBHOOK_URL")
send_alert(webhook_url)