flip/stellar-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/stellar-models.git synced 2026-02-06 11:06:44 +00:00
Code Issues Actions 6 Packages Projects Releases Wiki Activity

modified workflows to include USE_VARS in the env config, modified tests to use modified_timestamp (instead of closed_at) as the recency filter

Browse Source
This commit is contained in:
Mike Stepanovic 2025-02-13 09:04:04 -07:00
parent 4e5c524bd8
commit dddb1df675
15 changed files with 69 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/dbt_test_daily.yml vendored
View File

@ -8,7 +8,7 @@ on:
env:
DBT_PROFILES_DIR: ./
USE_VARS: "${{ vars.USE_VARS }}"
ACCOUNT: "${{ secrets.ACCOUNT }}"
ROLE: "${{ secrets.ROLE }}"
USER: "${{ secrets.USER }}"

2
.github/workflows/dbt_test_monthly.yml vendored
View File

@ -8,7 +8,7 @@ on:
env:
DBT_PROFILES_DIR: ./
USE_VARS: "${{ vars.USE_VARS }}"
ACCOUNT: "${{ secrets.ACCOUNT }}"
ROLE: "${{ secrets.ROLE }}"
USER: "${{ secrets.USER }}"

2
.github/workflows/dbt_test_weekly.yml vendored
View File

@ -8,7 +8,7 @@ on:
env:
DBT_PROFILES_DIR: ./
USE_VARS: "${{ vars.USE_VARS }}"
ACCOUNT: "${{ secrets.ACCOUNT }}"
ROLE: "${{ secrets.ROLE }}"
USER: "${{ secrets.USER }}"

18
models/gold/core/core__fact_accounts.yml
View File

@ -10,13 +10,13 @@ models:
- SEQUENCE_NUMBER
- LEDGER_ENTRY_CHANGE
- LAST_MODIFIED_LEDGER
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: ACCOUNT_ID
description: "{{ doc('account_id') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BALANCE
description: '{{ doc("balance") }}'
@ -31,7 +31,7 @@ models:
description: '{{ doc("sequence_number") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: NUM_SUBENTRIES
description: '{{ doc("num_subentries") }}'
@ -61,19 +61,19 @@ models:
description: '{{ doc("last_modified_ledger") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: LEDGER_ENTRY_CHANGE
description: '{{ doc("ledger_entry_change") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: DELETED
description: '{{ doc("deleted") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: SPONSOR
description: '{{ doc("sponsor") }}'
@ -88,7 +88,7 @@ models:
description: '{{ doc("sequence_ledger") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: SEQUENCE_TIME
description: '{{ doc("sequence_time") }}'
@ -97,7 +97,7 @@ models:
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BLOCK_TIMESTAMP
description: '{{ doc("block_timestamp") }}'
@ -106,7 +106,7 @@ models:
description: '{{ doc("ledger_sequence") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: FACT_ACCOUNTS_ID
description: '{{ doc("pk") }}'

6
models/gold/core/core__fact_ledgers.yml
View File

@ -9,9 +9,9 @@ models:
tests:
- dbt_expectations.expect_column_to_exist
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- unique:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: LEDGER_HASH
description: '{{ doc("ledger_hash") }}'
@ -29,7 +29,7 @@ models:
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BLOCK_TIMESTAMP
description: '{{ doc("block_timestamp") }}'

16
models/gold/core/core__fact_operations.yml
View File

@ -12,19 +12,19 @@ models:
- LEDGER_SEQUENCE
- TYPE
- TYPE_STRING
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
columns:
- name: ID
description: '{{ doc("op_id") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: CLOSED_AT
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BLOCK_TIMESTAMP
description: '{{ doc("block_timestamp") }}'
@ -264,7 +264,7 @@ models:
description: '{{ doc("source_account") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: OP_SOURCE_ACCOUNT_MUXED
description: '{{ doc("op_source_account_muxed") }}'
@ -273,25 +273,25 @@ models:
description: '{{ doc("transaction_id") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TYPE
description: '{{ doc("type") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TYPE_STRING
description: '{{ doc("type_string") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: LEDGER_SEQUENCE
description: '{{ doc("ledger_sequence") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: OP_ACCOUNT_MUXED
description: '{{ doc("account_muxed") }}'

10
models/gold/core/core__fact_transactions.yml
View File

@ -9,9 +9,9 @@ models:
description: Unique identifier for the transaction
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- unique:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TRANSACTION_HASH
description: '{{ doc("transaction_hash") }}'
@ -20,13 +20,13 @@ models:
description: '{{ doc("ledger_sequence") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: CLOSED_AT
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BLOCK_TIMESTAMP
description: '{{ doc("block_timestamp") }}'
@ -35,7 +35,7 @@ models:
description: '{{ doc("account") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: ACCOUNT_SEQUENCE
description: '{{ doc("account_sequence") }}'

4
models/gold/defi/defi__fact_liquidity_pools.yml
View File

@ -10,7 +10,7 @@ models:
description: '{{ doc("liquidity_pool_id") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TYPE
description: '{{ doc("type") }}'
@ -76,7 +76,7 @@ models:
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: LEDGER_SEQUENCE
description: '{{ doc("ledger_sequence") }}'

42
models/gold/defi/defi__fact_trades.yml
View File

@ -8,25 +8,25 @@ models:
description: '{{ doc("history_operation_id") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: trade_order
description: '{{ doc("order") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: ledger_closed_at
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: selling_account_address
description: '{{ doc("selling_account_address") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: selling_asset_code
description: '{{ doc("selling_asset_code") }}'
@ -38,25 +38,25 @@ models:
description: '{{ doc("selling_asset_type") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: selling_asset_id
description: '{{ doc("selling_asset_id") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: selling_amount
description: '{{ doc("selling_amount") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: buying_account_address
description: '{{ doc("buying_account_address") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: buying_asset_code
description: '{{ doc("buying_asset_code") }}'
@ -68,31 +68,31 @@ models:
description: '{{ doc("buying_asset_type") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: buying_asset_id
description: '{{ doc("buying_asset_id") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: buying_amount
description: '{{ doc("buying_amount") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: price_n
description: '{{ doc("price_n") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: price_d
description: '{{ doc("price_d") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: selling_offer_id
description: '{{ doc("selling_offer_id") }}'
@ -110,7 +110,7 @@ models:
description: '{{ doc("trade_type") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: rounding_slippage
description: '{{ doc("rounding_slippage") }}'
@ -120,29 +120,15 @@ models:
- name: fact_trades_id
description: '{{ doc("pk") }}'
tests:
- unique:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
- name: _inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
- name: inserted_timestamp
description: '{{ doc("inserted_timestamp") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
- name: modified_timestamp
description: '{{ doc("modified_timestamp") }}'
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
- name: _invocation_id
description: '{{ doc("invocation_id") }}'

14
models/gold/operations/core__ez_operations.yml
View File

@ -9,13 +9,13 @@ models:
description: '{{ doc("op_id") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: OP_SOURCE_ACCOUNT
description: '{{ doc("op_source_account") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: OP_SOURCE_ACCOUNT_MUXED
description: '{{ doc("op_source_account_muxed") }}'
@ -312,19 +312,19 @@ models:
description: '{{ doc("transaction_id") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TYPE
description: '{{ doc("type") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TYPE_STRING
description: '{{ doc("type_string") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BATCH_ID
description: '{{ doc("batch_id") }}'
@ -372,7 +372,7 @@ models:
description: '{{ doc("closed_at") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: BLOCK_TIMESTAMP
description: '{{ doc("block_timestamp") }}'
@ -384,7 +384,7 @@ models:
description: '{{ doc("ledger_sequence") }}'
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- name: TX_ACCOUNT
description: '{{ doc("tx_account") }}'

8
models/silver/silver__accounts.yml
View File

@ -7,7 +7,7 @@ models:
description: "{{ doc('account_id') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- VARCHAR
@ -15,7 +15,7 @@ models:
description: "{{ doc('sequence_number') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -23,7 +23,7 @@ models:
description: "{{ doc('ledger_entry_change') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -31,7 +31,7 @@ models:
description: "{{ doc('last_modified_ledger') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER

8
models/silver/silver__liquidity_pools.yml
View File

@ -1,13 +1,13 @@
version: 2
models:
- name: silver__liquidity_pools
columns:
- name: LIQUIDITY_POOL_ID
description: "{{ doc('liquidity_pool_id') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- VARCHAR
@ -15,7 +15,7 @@ models:
description: "{{ doc('ledger_entry_change') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -23,7 +23,7 @@ models:
description: "{{ doc('last_modified_ledger') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER

12
models/silver/silver__operations.yml
View File

@ -7,7 +7,7 @@ models:
description: "{{ doc('id') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -15,7 +15,7 @@ models:
description: "{{ doc('source_account') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- VARCHAR
@ -23,7 +23,7 @@ models:
description: "{{ doc('transaction_id') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -31,7 +31,7 @@ models:
description: "{{ doc('ledger_sequence') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -39,7 +39,7 @@ models:
description: "{{ doc('type') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -47,7 +47,7 @@ models:
description: "{{ doc('type_string') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- VARCHAR

4
models/silver/silver__trades.yml
View File

@ -7,7 +7,7 @@ models:
description: "{{ doc('history_operation_id') }}"
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
@ -15,7 +15,7 @@ models:
description: "{{ doc('order') }}"
tests:
- not_null:
where: ledger_closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER

4
models/silver/silver__transactions.yml
View File

@ -6,9 +6,9 @@ models:
description: "{{ doc('id') }}"
tests:
- not_null:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- unique:
where: closed_at > current_date - {{ var('test_days_threshold', 3) }}
where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }}
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER