mirror of
https://github.com/FlipsideCrypto/movement-models.git
synced 2026-02-06 13:26:45 +00:00
adjusted nomenclature
This commit is contained in:
parent
cb783de41e
commit
d984974e66
52
.github/workflows/dbt_test_daily.yml
vendored
Normal file
52
.github/workflows/dbt_test_daily.yml
vendored
Normal file
@ -0,0 +1,52 @@
|
||||
name: dbt_test_recent
|
||||
run-name: dbt_test_recent
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "30 0 * * *"
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt test -m tag:test_recency tag:test_quality tag:test_integrity --vars '{"TEST_HOURS_THRESHOLD":24}'
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
- name: Log test results
|
||||
run: |
|
||||
python python/dbt_test_alert.py
|
||||
52
.github/workflows/dbt_test_monthly.yml
vendored
Normal file
52
.github/workflows/dbt_test_monthly.yml
vendored
Normal file
@ -0,0 +1,52 @@
|
||||
name: dbt_test_recent
|
||||
run-name: dbt_test_recent
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 10 28 * *"
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt test -m tag:test_recency tag:test_quality tag:test_integrity --vars '{"TEST_HOURS_THRESHOLD":744}'
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
- name: Log test results
|
||||
run: |
|
||||
python python/dbt_test_alert.py
|
||||
53
.github/workflows/dbt_test_recency.yml
vendored
Normal file
53
.github/workflows/dbt_test_recency.yml
vendored
Normal file
@ -0,0 +1,53 @@
|
||||
name: dbt_test_recent
|
||||
run-name: dbt_test_recent
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 */2 * * *"
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt source freshness
|
||||
dbt test -m "movement_models,tag:test_recency"
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
- name: Log test results
|
||||
run: |
|
||||
python python/dbt_test_alert.py
|
||||
52
.github/workflows/dbt_test_weekly.yml
vendored
Normal file
52
.github/workflows/dbt_test_weekly.yml
vendored
Normal file
@ -0,0 +1,52 @@
|
||||
name: dbt_test_recent
|
||||
run-name: dbt_test_recent
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "30 0 * * 1"
|
||||
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
jobs:
|
||||
run_dbt_jobs:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "${{ vars.PYTHON_VERSION }}"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt test -m tag:test_recency tag:test_quality tag:test_integrity --vars '{"TEST_HOURS_THRESHOLD":168}'
|
||||
|
||||
continue-on-error: true
|
||||
|
||||
- name: Log test results
|
||||
run: |
|
||||
python python/dbt_test_alert.py
|
||||
@ -28,7 +28,8 @@ clean-targets: # directories to be removed by `dbt clean`
|
||||
|
||||
tests:
|
||||
+store_failures: true # all tests
|
||||
# +where: "modified_timestamp > dateadd(hour, -{{ var('TEST_HOURS_THRESHOLD') }}, current_timestamp)"
|
||||
test_quality:
|
||||
+where: "modified_timestamp > dateadd(hour, -{{ var('TEST_HOURS_THRESHOLD', 3) }}, current_timestamp)"
|
||||
|
||||
on-run-start:
|
||||
- "{{ create_sps() }}"
|
||||
|
||||
@ -2,6 +2,13 @@ version: 2
|
||||
models:
|
||||
- name: core__fact_blocks
|
||||
description: '{{ doc("core__fact_blocks") }}'
|
||||
tests:
|
||||
- dbt_utils.recency:
|
||||
datepart: hour
|
||||
field: MODIFIED_TIMESTAMP
|
||||
interval: 3
|
||||
severity: error
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
description: '{{ doc("block_number") }}'
|
||||
@ -24,6 +31,13 @@ models:
|
||||
|
||||
- name: core__fact_changes
|
||||
description: '{{ doc("core__fact_changes") }}'
|
||||
tests:
|
||||
- dbt_utils.recency:
|
||||
datepart: hour
|
||||
field: MODIFIED_TIMESTAMP
|
||||
interval: 3
|
||||
severity: error
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
description: '{{ doc("block_number") }}'
|
||||
@ -72,7 +86,14 @@ models:
|
||||
description: '{{ doc("modified_timestamp") }}'
|
||||
|
||||
- name: core__fact_events
|
||||
description: '{{ doc("core__fact_events") }}'
|
||||
description: '{{ doc("core__fact_events") }}'
|
||||
tests:
|
||||
- dbt_utils.recency:
|
||||
datepart: hour
|
||||
field: MODIFIED_TIMESTAMP
|
||||
interval: 3
|
||||
severity: error
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
description: '{{ doc("block_number") }}'
|
||||
@ -116,7 +137,14 @@ models:
|
||||
description: '{{ doc("modified_timestamp") }}'
|
||||
|
||||
- name: core__fact_transactions_block_metadata
|
||||
description: '{{ doc("core__fact_transactions_block_metadata") }}'
|
||||
description: '{{ doc("core__fact_transactions_block_metadata") }}'
|
||||
tests:
|
||||
- dbt_utils.recency:
|
||||
datepart: hour
|
||||
field: MODIFIED_TIMESTAMP
|
||||
interval: 3
|
||||
severity: error
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
description: '{{ doc("block_number") }}'
|
||||
@ -166,7 +194,14 @@ models:
|
||||
description: '{{ doc("modified_timestamp") }}'
|
||||
|
||||
- name: core__fact_transactions_state_checkpoint
|
||||
description: '{{ doc("core__fact_transactions_state_checkpoint") }}'
|
||||
description: '{{ doc("core__fact_transactions_state_checkpoint") }}'
|
||||
tests:
|
||||
- dbt_utils.recency:
|
||||
datepart: hour
|
||||
field: MODIFIED_TIMESTAMP
|
||||
interval: 3
|
||||
severity: error
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
description: '{{ doc("block_number") }}'
|
||||
@ -196,7 +231,14 @@ models:
|
||||
description: '{{ doc("modified_timestamp") }}'
|
||||
|
||||
- name: core__fact_transactions
|
||||
description: '{{ doc("core__fact_transactions") }}'
|
||||
description: '{{ doc("core__fact_transactions") }}'
|
||||
tests:
|
||||
- dbt_utils.recency:
|
||||
datepart: hour
|
||||
field: MODIFIED_TIMESTAMP
|
||||
interval: 3
|
||||
severity: error
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
description: '{{ doc("block_number") }}'
|
||||
408
models/silver/core/silver_core.yml
Normal file
408
models/silver/core/silver_core.yml
Normal file
@ -0,0 +1,408 @@
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: silver__blocks
|
||||
tests:
|
||||
- dbt_utils.sequential_values:
|
||||
column_name: block_number
|
||||
interval: 1
|
||||
config:
|
||||
severity: error
|
||||
error_if: ">100"
|
||||
tags: ['test_recency']
|
||||
columns:
|
||||
- name: block_number
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- unique:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: block_hash
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- unique:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: block_timestamp
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: TIMESTAMP_NTZ
|
||||
tags: ['test_integrity']
|
||||
- name: tx_count_from_versions
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: block_timestamp_num
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: first_version
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: last_version
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- name: blocks_id
|
||||
- name: inserted_timestamp
|
||||
- name: modified_timestamp
|
||||
- name: _invocation_id
|
||||
|
||||
- name: silver__changes
|
||||
columns:
|
||||
- name: block_number
|
||||
- name: block_timestamp
|
||||
- name: tx_hash
|
||||
- name: version
|
||||
- name: success
|
||||
- name: tx_type
|
||||
- name: payload_function
|
||||
- name: change_index
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: change_data
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: change_type
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: address
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: handle
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: inner_change_type
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: change_address
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: change_module
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: change_resource
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: key
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: value
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: state_key_hash
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: changes_id
|
||||
- name: inserted_timestamp
|
||||
- name: modified_timestamp
|
||||
- name: _invocation_id
|
||||
|
||||
- name: silver__events
|
||||
columns:
|
||||
- name: block_number
|
||||
- name: block_timestamp
|
||||
- name: tx_hash
|
||||
- name: version
|
||||
- name: success
|
||||
- name: tx_type
|
||||
- name: payload_function
|
||||
- name: event_index
|
||||
- name: event_type
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: event_address
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: event_module
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: event_resource
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: event_data
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: account_address
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: creation_number
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: sequence_number
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: events_id
|
||||
- name: inserted_timestamp
|
||||
- name: modified_timestamp
|
||||
- name: _invocation_id
|
||||
|
||||
- name: silver__transactions
|
||||
columns:
|
||||
- name: block_timestamp
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: TIMESTAMP_NTZ
|
||||
tags: ['test_integrity']
|
||||
- name: tx_hash
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- unique:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: version
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: tx_type
|
||||
tests:
|
||||
- not_null:
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: success
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: BOOLEAN
|
||||
tags: ['test_integrity']
|
||||
- name: sender
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: signature
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: payload
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: payload_function
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: changes
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: events
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: gas_unit_price
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: gas_used
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: max_gas_amount
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: expiration_timestamp_secs
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_between:
|
||||
min_value: 0
|
||||
tags: ['test_quality']
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: vm_status
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: state_change_hash
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: accumulator_root_hash
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: event_root_hash
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: state_checkpoint_hash
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: failed_proposer_indices
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: id
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: previous_block_votes_bitvec
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: proposer
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARCHAR
|
||||
tags: ['test_integrity']
|
||||
- name: ROUND
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: NUMBER
|
||||
tags: ['test_integrity']
|
||||
- name: data
|
||||
tests:
|
||||
- dbt_expectations.expect_column_values_to_be_of_type:
|
||||
column_type: VARIANT
|
||||
tags: ['test_integrity']
|
||||
- name: transactions_id
|
||||
- name: inserted_timestamp
|
||||
- name: modified_timestamp
|
||||
- name: _invocation_id
|
||||
127
python/dbt_test_alert.py
Normal file
127
python/dbt_test_alert.py
Normal file
@ -0,0 +1,127 @@
|
||||
import datetime
|
||||
import requests
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
|
||||
|
||||
def log_test_result():
|
||||
"""Reads the run_results.json file and returns a dictionary of targeted test results"""
|
||||
|
||||
filepath = "target/run_results.json"
|
||||
|
||||
with open(filepath) as f:
|
||||
run = json.load(f)
|
||||
|
||||
logs = []
|
||||
messages = {
|
||||
"fail": [],
|
||||
"warn": []
|
||||
}
|
||||
test_count = 0
|
||||
warn_count = 0
|
||||
fail_count = 0
|
||||
|
||||
for test in run["results"]:
|
||||
test_count += 1
|
||||
if test["status"] != "pass":
|
||||
logs.append(test)
|
||||
|
||||
message = f"{test['failures']} record failure(s) in {test['unique_id']}"
|
||||
|
||||
if test["status"] == "warn":
|
||||
messages["warn"].append(message)
|
||||
warn_count += 1
|
||||
elif test["status"] == "fail":
|
||||
messages["fail"].append(message)
|
||||
fail_count += 1
|
||||
|
||||
dbt_test_result = {
|
||||
"logs": logs,
|
||||
"messages": messages,
|
||||
"test_count": test_count,
|
||||
"warn_count": warn_count,
|
||||
"fail_count": fail_count,
|
||||
"elapsed_time": str(datetime.timedelta(seconds=run["elapsed_time"]))
|
||||
}
|
||||
|
||||
return dbt_test_result
|
||||
|
||||
|
||||
def create_message(**kwargs):
|
||||
messageBody = {
|
||||
"text": f"Hey{' <!here>' if len(kwargs['messages']['fail']) > 0 else ''}, new DBT test results for :{os.environ.get('DATABASE').split('_DEV')[0]}: {os.environ.get('DATABASE')}",
|
||||
"attachments": [
|
||||
{
|
||||
"color": kwargs["color"],
|
||||
"fields": [
|
||||
{
|
||||
"title": "Total Tests Run",
|
||||
"value": kwargs["test_count"],
|
||||
"short": True
|
||||
},
|
||||
{
|
||||
"title": "Total Time Elapsed",
|
||||
"value": kwargs["elapsed_time"],
|
||||
"short": True
|
||||
},
|
||||
{
|
||||
"title": "Number of Unsuccessful Tests",
|
||||
"value": f"Fail: {kwargs['fail_count']}, Warn: {kwargs['warn_count']}",
|
||||
"short": True
|
||||
},
|
||||
{
|
||||
"title": "Failed Tests:",
|
||||
"value": "\n".join(kwargs["messages"]["fail"]) if len(kwargs["messages"]["fail"]) > 0 else "None :)",
|
||||
"short": False
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
|
||||
{
|
||||
"type": "button",
|
||||
"text": "View Warnings",
|
||||
"style": "primary",
|
||||
"url": "https://github.com/FlipsideCrypto/movement-models/actions",
|
||||
"confirm": {
|
||||
"title": f"{kwargs['warn_count']} Warnings",
|
||||
"text": "\n".join(kwargs["messages"]["warn"]) if len(kwargs["messages"]["warn"]) > 0 else "None :)",
|
||||
"ok_text": "Continue to GHA",
|
||||
"dismiss_text": "Dismiss"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return messageBody
|
||||
|
||||
|
||||
def send_alert(webhook_url):
|
||||
"""Sends a message to a slack channel"""
|
||||
|
||||
url = webhook_url
|
||||
|
||||
data = log_test_result()
|
||||
|
||||
send_message = create_message(
|
||||
fail_count=data["fail_count"],
|
||||
warn_count=data["warn_count"],
|
||||
test_count=data["test_count"],
|
||||
messages=data["messages"],
|
||||
elapsed_time=data["elapsed_time"],
|
||||
color="#f44336" if data["fail_count"] > 0 else "#4CAF50"
|
||||
)
|
||||
|
||||
x = requests.post(url, json=send_message)
|
||||
|
||||
# test config to continue on error in workflow, so we want to exit with a non-zero code if there are any failures
|
||||
if data['fail_count'] > 0:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
webhook_url = os.environ.get("SLACK_WEBHOOK_URL")
|
||||
send_alert(webhook_url)
|
||||
Loading…
Reference in New Issue
Block a user