From 006e314b5ee64b60e00c0c600ed797b8ef11ef50 Mon Sep 17 00:00:00 2001 From: Mike Stepanovic Date: Thu, 13 Feb 2025 13:30:50 -0700 Subject: [PATCH] Add testing strategy (#4) * added testing strategy * updated recency within tests, added workflows with different lookback vars * changed daily lookback to 1 * modified workflows to include USE_VARS in the env config, modified tests to use modified_timestamp (instead of closed_at) as the recency filter * fixed env variables (switched to vars from secrets) --------- Co-authored-by: Mike Stepanovic Co-authored-by: Eric Laurello --- .github/workflows/dbt_test_daily.yml | 48 ++ .github/workflows/dbt_test_monthly.yml | 48 ++ .github/workflows/dbt_test_recent.yml | 60 ++ .github/workflows/dbt_test_weekly.yml | 48 ++ models/descriptions/__overview__.md | 3 + models/descriptions/account_muxed_id.md | 3 + models/descriptions/asset.md | 3 + models/descriptions/begin_sponsor_muxed.md | 3 + models/descriptions/begin_sponsor_muxed_id.md | 3 + models/descriptions/block_timestamp.md | 3 + models/descriptions/claimable_balance_id.md | 3 + models/descriptions/claimant.md | 3 + models/descriptions/claimant_muxed.md | 3 + models/descriptions/claimant_muxed_id.md | 3 + models/descriptions/claimants.md | 3 + models/descriptions/data_account_id.md | 3 + models/descriptions/data_name.md | 3 + models/descriptions/from_account.md | 3 + models/descriptions/from_muxed.md | 3 + models/descriptions/from_muxed_id.md | 3 + models/descriptions/funder_muxed.md | 3 + models/descriptions/funder_muxed_id.md | 3 + models/descriptions/high_threshold.md | 3 + models/descriptions/inflation_dest.md | 3 + models/descriptions/into_muxed.md | 3 + models/descriptions/into_muxed_id.md | 3 + models/descriptions/ledger_id.md | 3 + models/descriptions/ledger_key_hash.md | 3 + models/descriptions/ledgers_to_expire.md | 3 + models/descriptions/limit_amount.md | 3 + models/descriptions/low_threshold.md | 3 + models/descriptions/master_key_weight.md | 3 + models/descriptions/med_threshold.md | 3 + models/descriptions/price_r.md | 3 + models/descriptions/signer_account_id.md | 3 + models/descriptions/source_account.md | 3 + models/descriptions/to_muxed.md | 3 + models/descriptions/to_muxed_id.md | 3 + models/descriptions/trustee_muxed.md | 3 + models/descriptions/trustee_muxed_id.md | 3 + models/descriptions/trustline_account_id.md | 3 + models/descriptions/trustor_muxed.md | 3 + models/descriptions/trustor_muxed_id.md | 3 + .../{txn_account.md => tx_account.md} | 2 +- ...ation_order.md => tx_application_order.md} | 2 +- .../{txn_created_at.md => tx_created_at.md} | 2 +- ...eration_count.md => tx_operation_count.md} | 2 +- models/descriptions/type_string.md | 3 + models/gold/core/core__dim_assets.yml | 48 +- models/gold/core/core__fact_accounts.yml | 83 +-- models/gold/core/core__fact_ledgers.yml | 80 +-- models/gold/core/core__fact_operations.yml | 295 +++++----- models/gold/core/core__fact_transactions.yml | 276 ++++----- .../gold/defi/defi__fact_liquidity_pools.yml | 17 +- models/gold/defi/defi__fact_trades.yml | 51 +- .../gold/operations/core__ez_operations.yml | 535 ++++++++++++++++-- models/silver/silver__accounts.yml | 37 ++ models/silver/silver__assets.yml | 29 + models/silver/silver__ledgers.yml | 9 + models/silver/silver__liquidity_pools.yml | 29 + models/silver/silver__operations.yml | 53 ++ models/silver/silver__trades.yml | 21 + models/silver/silver__transactions.yml | 14 + .../streamline/core/_max_ledger_by_date.sql | 27 + .../recent/test_core__accounts_recent.sql | 27 + .../recent/test_core__accounts_recent.yml | 11 + .../test_core__ez_operations_recent.sql | 27 + .../test_core__ez_operations_recent.yml | 11 + .../recent/test_core__ledgers_recent.sql | 27 + .../recent/test_core__ledgers_recent.yml | 11 + .../recent/test_core__operations_recent.sql | 27 + .../recent/test_core__operations_recent.yml | 11 + .../recent/test_core__transactions_recent.sql | 27 + .../recent/test_core__transactions_recent.yml | 11 + .../test_defi__fact_liquidity_pools.sql | 27 + .../test_defi__fact_liquidity_pools.yml | 11 + .../tests/recent/test_defi__trades_recent.sql | 27 + .../tests/recent/test_defi__trades_recent.yml | 11 + python/dbt_test_alert.py | 127 +++++ 79 files changed, 1791 insertions(+), 538 deletions(-) create mode 100644 .github/workflows/dbt_test_daily.yml create mode 100644 .github/workflows/dbt_test_monthly.yml create mode 100644 .github/workflows/dbt_test_recent.yml create mode 100644 .github/workflows/dbt_test_weekly.yml create mode 100644 models/descriptions/account_muxed_id.md create mode 100644 models/descriptions/asset.md create mode 100644 models/descriptions/begin_sponsor_muxed.md create mode 100644 models/descriptions/begin_sponsor_muxed_id.md create mode 100644 models/descriptions/block_timestamp.md create mode 100644 models/descriptions/claimable_balance_id.md create mode 100644 models/descriptions/claimant.md create mode 100644 models/descriptions/claimant_muxed.md create mode 100644 models/descriptions/claimant_muxed_id.md create mode 100644 models/descriptions/claimants.md create mode 100644 models/descriptions/data_account_id.md create mode 100644 models/descriptions/data_name.md create mode 100644 models/descriptions/from_account.md create mode 100644 models/descriptions/from_muxed.md create mode 100644 models/descriptions/from_muxed_id.md create mode 100644 models/descriptions/funder_muxed.md create mode 100644 models/descriptions/funder_muxed_id.md create mode 100644 models/descriptions/high_threshold.md create mode 100644 models/descriptions/inflation_dest.md create mode 100644 models/descriptions/into_muxed.md create mode 100644 models/descriptions/into_muxed_id.md create mode 100644 models/descriptions/ledger_id.md create mode 100644 models/descriptions/ledger_key_hash.md create mode 100644 models/descriptions/ledgers_to_expire.md create mode 100644 models/descriptions/limit_amount.md create mode 100644 models/descriptions/low_threshold.md create mode 100644 models/descriptions/master_key_weight.md create mode 100644 models/descriptions/med_threshold.md create mode 100644 models/descriptions/price_r.md create mode 100644 models/descriptions/signer_account_id.md create mode 100644 models/descriptions/source_account.md create mode 100644 models/descriptions/to_muxed.md create mode 100644 models/descriptions/to_muxed_id.md create mode 100644 models/descriptions/trustee_muxed.md create mode 100644 models/descriptions/trustee_muxed_id.md create mode 100644 models/descriptions/trustline_account_id.md create mode 100644 models/descriptions/trustor_muxed.md create mode 100644 models/descriptions/trustor_muxed_id.md rename models/descriptions/{txn_account.md => tx_account.md} (68%) rename models/descriptions/{txn_application_order.md => tx_application_order.md} (59%) rename models/descriptions/{txn_created_at.md => tx_created_at.md} (67%) rename models/descriptions/{txn_operation_count.md => tx_operation_count.md} (63%) create mode 100644 models/descriptions/type_string.md create mode 100644 models/silver/silver__accounts.yml create mode 100644 models/silver/silver__assets.yml create mode 100644 models/silver/silver__ledgers.yml create mode 100644 models/silver/silver__liquidity_pools.yml create mode 100644 models/silver/silver__operations.yml create mode 100644 models/silver/silver__trades.yml create mode 100644 models/silver/silver__transactions.yml create mode 100644 models/streamline/core/_max_ledger_by_date.sql create mode 100644 models/tests/recent/test_core__accounts_recent.sql create mode 100644 models/tests/recent/test_core__accounts_recent.yml create mode 100644 models/tests/recent/test_core__ez_operations_recent.sql create mode 100644 models/tests/recent/test_core__ez_operations_recent.yml create mode 100644 models/tests/recent/test_core__ledgers_recent.sql create mode 100644 models/tests/recent/test_core__ledgers_recent.yml create mode 100644 models/tests/recent/test_core__operations_recent.sql create mode 100644 models/tests/recent/test_core__operations_recent.yml create mode 100644 models/tests/recent/test_core__transactions_recent.sql create mode 100644 models/tests/recent/test_core__transactions_recent.yml create mode 100644 models/tests/recent/test_defi__fact_liquidity_pools.sql create mode 100644 models/tests/recent/test_defi__fact_liquidity_pools.yml create mode 100644 models/tests/recent/test_defi__trades_recent.sql create mode 100644 models/tests/recent/test_defi__trades_recent.yml create mode 100644 python/dbt_test_alert.py diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml new file mode 100644 index 0000000..44e565e --- /dev/null +++ b/.github/workflows/dbt_test_daily.yml @@ -0,0 +1,48 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * *' + +env: + DBT_PROFILES_DIR: ./ + USE_VARS: "${{ vars.USE_VARS }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ vars.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10.x" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "stellar_models,models/silver" "stellar_models,models/gold" --vars 'test_days_threshold: 1' + + continue-on-error: true + + - name: Log test results + run: | + python python/dbt_test_alert.py \ No newline at end of file diff --git a/.github/workflows/dbt_test_monthly.yml b/.github/workflows/dbt_test_monthly.yml new file mode 100644 index 0000000..601932a --- /dev/null +++ b/.github/workflows/dbt_test_monthly.yml @@ -0,0 +1,48 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 4 1 * *' + +env: + DBT_PROFILES_DIR: ./ + USE_VARS: "${{ vars.USE_VARS }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ vars.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10.x" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "stellar_models,models/silver" "stellar_models,models/gold" --vars 'test_days_threshold: 30' + + continue-on-error: true + + - name: Log test results + run: | + python python/dbt_test_alert.py \ No newline at end of file diff --git a/.github/workflows/dbt_test_recent.yml b/.github/workflows/dbt_test_recent.yml new file mode 100644 index 0000000..7a6e97b --- /dev/null +++ b/.github/workflows/dbt_test_recent.yml @@ -0,0 +1,60 @@ +name: dbt_test_recent +run-name: dbt_test_recent + +on: + workflow_dispatch: + # schedule: + # # Daily at 8:30am UTC + # - cron: '30 8 * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "${{ vars.PYTHON_VERSION }}" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "stellar_models,tag:recent_test" + dbt test -m "stellar_models,tag:recent_test" + + continue-on-error: true + + - name: Log test results + run: | + python python/dbt_test_alert.py + + continue-on-error: true + + - name: Log test results + run: | + python python/dbt_test_alert.py \ No newline at end of file diff --git a/.github/workflows/dbt_test_weekly.yml b/.github/workflows/dbt_test_weekly.yml new file mode 100644 index 0000000..170bcdc --- /dev/null +++ b/.github/workflows/dbt_test_weekly.yml @@ -0,0 +1,48 @@ +name: dbt_test_scheduled +run-name: dbt_test_scheduled + +on: + workflow_dispatch: + schedule: + - cron: '0 4 * * 1' + +env: + DBT_PROFILES_DIR: ./ + USE_VARS: "${{ vars.USE_VARS }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ vars.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10.x" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "stellar_models,models/silver" "stellar_models,models/gold" --vars 'test_days_threshold: 7' + + continue-on-error: true + + - name: Log test results + run: | + python python/dbt_test_alert.py \ No newline at end of file diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 09df0fa..c2cb9d5 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -28,6 +28,9 @@ There is more information on how to use dbt docs in the last section of this doc - [core.fact_operations](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_operations) - [core.fact_transactions](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__fact_transactions) +**Convenience Views:** +- [core.ez_operations](https://flipsidecrypto.github.io/stellar-models/#!/model/model.stellar_models.core__ez_operations) + ### DeFi Tables (`stellar`.`DEFI`.``) **Fact Tables:** diff --git a/models/descriptions/account_muxed_id.md b/models/descriptions/account_muxed_id.md new file mode 100644 index 0000000..ff34f84 --- /dev/null +++ b/models/descriptions/account_muxed_id.md @@ -0,0 +1,3 @@ +{% docs account_muxed_id %} +Record that contains details based on the type of operation executed. Each operation will return its own relevant details, with the rest of the details as null +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/asset.md b/models/descriptions/asset.md new file mode 100644 index 0000000..b8335b7 --- /dev/null +++ b/models/descriptions/asset.md @@ -0,0 +1,3 @@ +{% docs asset %} +The asset available to be claimed in the form of "asset_code:issuing_address". If the claimable balance is in XLM, it is reported as "native" +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/begin_sponsor_muxed.md b/models/descriptions/begin_sponsor_muxed.md new file mode 100644 index 0000000..38defb0 --- /dev/null +++ b/models/descriptions/begin_sponsor_muxed.md @@ -0,0 +1,3 @@ +{% docs begin_sponsor_muxed %} +If the initiating sponsorship account is multiplexed, the virtual address. +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/begin_sponsor_muxed_id.md b/models/descriptions/begin_sponsor_muxed_id.md new file mode 100644 index 0000000..f2d4da1 --- /dev/null +++ b/models/descriptions/begin_sponsor_muxed_id.md @@ -0,0 +1,3 @@ +{% docs begin_sponsor_muxed_id %} +If the initiating sponsorship account is multiplexed, the integer representation of the virtual address. +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 0000000..03ad39c --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,3 @@ +{% docs block_timestamp %} +A clone of closed_at for join convenience. +{% enddocs %} diff --git a/models/descriptions/claimable_balance_id.md b/models/descriptions/claimable_balance_id.md new file mode 100644 index 0000000..c3f91f4 --- /dev/null +++ b/models/descriptions/claimable_balance_id.md @@ -0,0 +1,3 @@ +{% docs claimable_balance_id %} +The balance id of the claimable balance which is no longer sponsored +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/claimant.md b/models/descriptions/claimant.md new file mode 100644 index 0000000..af56bc4 --- /dev/null +++ b/models/descriptions/claimant.md @@ -0,0 +1,3 @@ +{% docs claimant %} +The account address of the account which claimed the claimable balance +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/claimant_muxed.md b/models/descriptions/claimant_muxed.md new file mode 100644 index 0000000..692e5f1 --- /dev/null +++ b/models/descriptions/claimant_muxed.md @@ -0,0 +1,3 @@ +{% docs claimant_muxed %} +If the account is multiplexed, the virtual address of the account which claimed the claimable balance +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/claimant_muxed_id.md b/models/descriptions/claimant_muxed_id.md new file mode 100644 index 0000000..2c3f8bd --- /dev/null +++ b/models/descriptions/claimant_muxed_id.md @@ -0,0 +1,3 @@ +{% docs claimant_muxed_id %} +If the account is multiplexed, an integer representation of the muxed account which claimed the balance +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/claimants.md b/models/descriptions/claimants.md new file mode 100644 index 0000000..de0ea74 --- /dev/null +++ b/models/descriptions/claimants.md @@ -0,0 +1,3 @@ +{% docs claimants %} +An unstructured field that lists account addresses eligible to claim a balance and the conditions which must be satisfied to claim the balance (typically time bound conditions) +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/data_account_id.md b/models/descriptions/data_account_id.md new file mode 100644 index 0000000..15bbcde --- /dev/null +++ b/models/descriptions/data_account_id.md @@ -0,0 +1,3 @@ +{% docs data_account_id %} +The account address of the account whose data entry is no longer sponsored +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/data_name.md b/models/descriptions/data_name.md new file mode 100644 index 0000000..7aa852a --- /dev/null +++ b/models/descriptions/data_name.md @@ -0,0 +1,3 @@ +{% docs data_name %} +The name of the data entry which is no longer sponsored +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/from_account.md b/models/descriptions/from_account.md new file mode 100644 index 0000000..45e8b70 --- /dev/null +++ b/models/descriptions/from_account.md @@ -0,0 +1,3 @@ +{% docs from_account %} +The account address from which the payment/contract originates (the sender account) +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/from_muxed.md b/models/descriptions/from_muxed.md new file mode 100644 index 0000000..e6d1a5d --- /dev/null +++ b/models/descriptions/from_muxed.md @@ -0,0 +1,3 @@ +{% docs from_muxed %} +If the account is multiplexed, the virtual address of the sender account +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/from_muxed_id.md b/models/descriptions/from_muxed_id.md new file mode 100644 index 0000000..022192d --- /dev/null +++ b/models/descriptions/from_muxed_id.md @@ -0,0 +1,3 @@ +{% docs from_muxed_id %} +If the account is multiplexed, the integer representation of the virtual address of the sender account +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/funder_muxed.md b/models/descriptions/funder_muxed.md new file mode 100644 index 0000000..50ef895 --- /dev/null +++ b/models/descriptions/funder_muxed.md @@ -0,0 +1,3 @@ +{% docs funder_muxed %} +If the account is multiplexed, the virtual address of the account funding the new account +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/funder_muxed_id.md b/models/descriptions/funder_muxed_id.md new file mode 100644 index 0000000..94bd00b --- /dev/null +++ b/models/descriptions/funder_muxed_id.md @@ -0,0 +1,3 @@ +{% docs funder_muxed_id %} +If the account is multiplexed, the integer representation of the account funding the new account +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/high_threshold.md b/models/descriptions/high_threshold.md new file mode 100644 index 0000000..727d6a7 --- /dev/null +++ b/models/descriptions/high_threshold.md @@ -0,0 +1,3 @@ +{% docs high_threshold %} +The sum of the weight of all signatures that sign a transaction for the high threshold operation. The weight must exceed the set threshold for the operation to succeed. +{% enddocs %} diff --git a/models/descriptions/inflation_dest.md b/models/descriptions/inflation_dest.md new file mode 100644 index 0000000..7da31d5 --- /dev/null +++ b/models/descriptions/inflation_dest.md @@ -0,0 +1,3 @@ +{% docs inflation_dest %} +The account address specifying where to send inflation funds. The concept of inflation on the network has been discontinued +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/into_muxed.md b/models/descriptions/into_muxed.md new file mode 100644 index 0000000..0fdd188 --- /dev/null +++ b/models/descriptions/into_muxed.md @@ -0,0 +1,3 @@ +{% docs into_muxed %} +If the account is multiplexed, the virtual address of the account receive the deleted account's lumens +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/into_muxed_id.md b/models/descriptions/into_muxed_id.md new file mode 100644 index 0000000..408ca5d --- /dev/null +++ b/models/descriptions/into_muxed_id.md @@ -0,0 +1,3 @@ +{% docs into_muxed_id %} +If the account is multiplexed, the integer representation of the account receiving the deleted account's lumens +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/ledger_id.md b/models/descriptions/ledger_id.md new file mode 100644 index 0000000..089aa63 --- /dev/null +++ b/models/descriptions/ledger_id.md @@ -0,0 +1,3 @@ +{% docs ledger_id %} +Unique identifier for the ledger +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/ledger_key_hash.md b/models/descriptions/ledger_key_hash.md new file mode 100644 index 0000000..7eed4b7 --- /dev/null +++ b/models/descriptions/ledger_key_hash.md @@ -0,0 +1,3 @@ +{% docs ledger_key_hash %} +Hash of the ledgerKey which is a subset of the ledgerEntry. The subset of ledgerEntry fields depends on ledgerEntryType. +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/ledgers_to_expire.md b/models/descriptions/ledgers_to_expire.md new file mode 100644 index 0000000..45f9925 --- /dev/null +++ b/models/descriptions/ledgers_to_expire.md @@ -0,0 +1,3 @@ +{% docs ledgers_to_expire %} +The number of ledgers after which the operation will expire if its not executed. +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/limit_amount.md b/models/descriptions/limit_amount.md new file mode 100644 index 0000000..c9e6ba2 --- /dev/null +++ b/models/descriptions/limit_amount.md @@ -0,0 +1,3 @@ +{% docs limit_amount %} +The upper bound amount of an asset that an account can hold +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/low_threshold.md b/models/descriptions/low_threshold.md new file mode 100644 index 0000000..c0da8a6 --- /dev/null +++ b/models/descriptions/low_threshold.md @@ -0,0 +1,3 @@ +{% docs low_threshold %} +The sum of the weight of all signatures that sign a transaction for the low threshold operation. The weight must exceed the set threshold for the operation to succeed. +{% enddocs %} diff --git a/models/descriptions/master_key_weight.md b/models/descriptions/master_key_weight.md new file mode 100644 index 0000000..96735bb --- /dev/null +++ b/models/descriptions/master_key_weight.md @@ -0,0 +1,3 @@ +{% docs master_key_weight %} +An accounts private key is called the master key. For signing transactions, the account holder can specify a weight for the master key, which contributes to thresholds validation when processing a transaction. +{% enddocs %} diff --git a/models/descriptions/med_threshold.md b/models/descriptions/med_threshold.md new file mode 100644 index 0000000..52eab0f --- /dev/null +++ b/models/descriptions/med_threshold.md @@ -0,0 +1,3 @@ +{% docs med_threshold %} +The sum of the weight of all signatures that sign a transaction for the medium threshold operation. The weight must exceed the set threshold for the operation to succeed. +{% enddocs %} diff --git a/models/descriptions/price_r.md b/models/descriptions/price_r.md new file mode 100644 index 0000000..2a9583c --- /dev/null +++ b/models/descriptions/price_r.md @@ -0,0 +1,3 @@ +{% docs price_r %} +Precise representation of the buy and sell price of the assets on an offer. The n is the numerator, the d is the denominator. By calculating the ratio of n/d you can calculate the price of the bid or ask. +{% enddocs %} diff --git a/models/descriptions/signer_account_id.md b/models/descriptions/signer_account_id.md new file mode 100644 index 0000000..41ec18a --- /dev/null +++ b/models/descriptions/signer_account_id.md @@ -0,0 +1,3 @@ +{% docs signer_account_id %} +The address of the account of the signer no longer sponsored +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/source_account.md b/models/descriptions/source_account.md new file mode 100644 index 0000000..8314ce1 --- /dev/null +++ b/models/descriptions/source_account.md @@ -0,0 +1,3 @@ +{% docs source_account %} +The account that originated the transaction. +{% enddocs %} diff --git a/models/descriptions/to_muxed.md b/models/descriptions/to_muxed.md new file mode 100644 index 0000000..fd9b0ca --- /dev/null +++ b/models/descriptions/to_muxed.md @@ -0,0 +1,3 @@ +{% docs to_muxed %} +If the account is multiplexed, the virtual address of the account receiving the payment +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/to_muxed_id.md b/models/descriptions/to_muxed_id.md new file mode 100644 index 0000000..9d23635 --- /dev/null +++ b/models/descriptions/to_muxed_id.md @@ -0,0 +1,3 @@ +{% docs to_muxed_id %} +If the account is multiplexed, the integer representation of the virtual address of the recipient account +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/trustee_muxed.md b/models/descriptions/trustee_muxed.md new file mode 100644 index 0000000..96f14c4 --- /dev/null +++ b/models/descriptions/trustee_muxed.md @@ -0,0 +1,3 @@ +{% docs trustee_muxed %} +If the issuing account address is multiplexed, the virtual address of the trustee +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/trustee_muxed_id.md b/models/descriptions/trustee_muxed_id.md new file mode 100644 index 0000000..de62f2a --- /dev/null +++ b/models/descriptions/trustee_muxed_id.md @@ -0,0 +1,3 @@ +{% docs trustee_muxed_id %} +If the issuing account address is multiplexed, the integer representation of the virtual address of the trustee +{% enddocs %} diff --git a/models/descriptions/trustline_account_id.md b/models/descriptions/trustline_account_id.md new file mode 100644 index 0000000..ec395fd --- /dev/null +++ b/models/descriptions/trustline_account_id.md @@ -0,0 +1,3 @@ +{% docs trustline_account_id %} +The address of the account whose trustline is no longer sponsored +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/trustor_muxed.md b/models/descriptions/trustor_muxed.md new file mode 100644 index 0000000..6d1a2d2 --- /dev/null +++ b/models/descriptions/trustor_muxed.md @@ -0,0 +1,3 @@ +{% docs trustor_muxed %} +If the trusting account is multiplexed, the virtual address of the account +{% enddocs %} diff --git a/models/descriptions/trustor_muxed_id.md b/models/descriptions/trustor_muxed_id.md new file mode 100644 index 0000000..97c45aa --- /dev/null +++ b/models/descriptions/trustor_muxed_id.md @@ -0,0 +1,3 @@ +{% docs trustor_muxed_id %} +If the trusting account is multiplexed, the integer representation of the virtual address of the account +{% enddocs %} diff --git a/models/descriptions/txn_account.md b/models/descriptions/tx_account.md similarity index 68% rename from models/descriptions/txn_account.md rename to models/descriptions/tx_account.md index 4f54f4a..1cff2c2 100644 --- a/models/descriptions/txn_account.md +++ b/models/descriptions/tx_account.md @@ -1,3 +1,3 @@ -{% docs txn_account %} +{% docs tx_account %} Account executing the transaction. {% enddocs %} diff --git a/models/descriptions/txn_application_order.md b/models/descriptions/tx_application_order.md similarity index 59% rename from models/descriptions/txn_application_order.md rename to models/descriptions/tx_application_order.md index b69bc52..64a512a 100644 --- a/models/descriptions/txn_application_order.md +++ b/models/descriptions/tx_application_order.md @@ -1,3 +1,3 @@ -{% docs txn_application_order %} +{% docs tx_application_order %} Order of transaction application. {% enddocs %} diff --git a/models/descriptions/txn_created_at.md b/models/descriptions/tx_created_at.md similarity index 67% rename from models/descriptions/txn_created_at.md rename to models/descriptions/tx_created_at.md index 6c13de6..92e5548 100644 --- a/models/descriptions/txn_created_at.md +++ b/models/descriptions/tx_created_at.md @@ -1,3 +1,3 @@ -{% docs txn_created_at %} +{% docs tx_created_at %} Creation timestamp of the transaction. {% enddocs %} diff --git a/models/descriptions/txn_operation_count.md b/models/descriptions/tx_operation_count.md similarity index 63% rename from models/descriptions/txn_operation_count.md rename to models/descriptions/tx_operation_count.md index 09c94bf..ea05862 100644 --- a/models/descriptions/txn_operation_count.md +++ b/models/descriptions/tx_operation_count.md @@ -1,3 +1,3 @@ -{% docs txn_operation_count %} +{% docs tx_operation_count %} Number of operations in the transaction. {% enddocs %} diff --git a/models/descriptions/type_string.md b/models/descriptions/type_string.md new file mode 100644 index 0000000..180ba8d --- /dev/null +++ b/models/descriptions/type_string.md @@ -0,0 +1,3 @@ +{% docs type_string %} +The verbose type of operation. +{% enddocs %} diff --git a/models/gold/core/core__dim_assets.yml b/models/gold/core/core__dim_assets.yml index 320428f..5fe3537 100644 --- a/models/gold/core/core__dim_assets.yml +++ b/models/gold/core/core__dim_assets.yml @@ -3,52 +3,54 @@ version: 2 models: - name: core__dim_assets description: Dimension table containing information about assets on the network. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_TYPE + - ASSET_CODE + - ASSET_ISSUER + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - - name: ID - description: '{{ doc("id") }}' - tests: - - dbt_expectations.expect_column_to_exist + - name: ASSET_ID + description: '{{ doc("asset_id") }}' - name: ASSET_TYPE description: '{{ doc("asset_type") }}' tests: - - dbt_expectations.expect_column_to_exist - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: ASSET_CODE description: '{{ doc("asset_code") }}' tests: - - dbt_expectations.expect_column_to_exist + - not_null: + where: asset_type != 'native' AND modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: ASSET_ISSUER description: '{{ doc("asset_issuer") }}' tests: - - dbt_expectations.expect_column_to_exist + - not_null: + where: asset_type != 'native' AND modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - - name: ASSET_ID - description: '{{ doc("asset_id") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null - - unique + - name: ID + description: 'Numeric hash of asset_id, asset_type, and asset_issuer for join operations' - name: DIM_ASSETS_ID description: '{{ doc("pk") }}' tests: - - unique - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: MODIFIED_TIMESTAMP description: '{{ doc("modified_timestamp") }}' tests: - - not_null - - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - asset_id + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} diff --git a/models/gold/core/core__fact_accounts.yml b/models/gold/core/core__fact_accounts.yml index fe3e733..c0a7583 100644 --- a/models/gold/core/core__fact_accounts.yml +++ b/models/gold/core/core__fact_accounts.yml @@ -3,143 +3,116 @@ version: 2 models: - name: core__fact_accounts description: Core fact table containing all Stellar account information and their current states + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ACCOUNT_ID + - SEQUENCE_NUMBER + - LEDGER_ENTRY_CHANGE + - LAST_MODIFIED_LEDGER + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - name: ACCOUNT_ID description: "{{ doc('account_id') }}" tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: BALANCE description: '{{ doc("balance") }}' - tests: - - not_null - name: BUYING_LIABILITIES description: '{{ doc("buying_liabilities") }}' - tests: - - not_null - name: SELLING_LIABILITIES description: '{{ doc("selling_liabilities") }}' - tests: - - not_null - name: SEQUENCE_NUMBER description: '{{ doc("sequence_number") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: NUM_SUBENTRIES description: '{{ doc("num_subentries") }}' - tests: - - not_null - name: INFLATION_DESTINATION description: '{{ doc("inflation_destination") }}' - name: FLAGS description: '{{ doc("flags") }}' - tests: - - not_null - name: HOME_DOMAIN description: '{{ doc("home_domain") }}' - name: MASTER_WEIGHT description: '{{ doc("master_weight") }}' - tests: - - not_null - name: THRESHOLD_LOW description: '{{ doc("threshold_low") }}' - tests: - - not_null - name: THRESHOLD_MEDIUM description: '{{ doc("threshold_medium") }}' - tests: - - not_null - name: THRESHOLD_HIGH description: '{{ doc("threshold_high") }}' - tests: - - not_null - name: LAST_MODIFIED_LEDGER description: '{{ doc("last_modified_ledger") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: LEDGER_ENTRY_CHANGE description: '{{ doc("ledger_entry_change") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: DELETED description: '{{ doc("deleted") }}' tests: - - not_null - - - name: BATCH_ID - description: '{{ doc("batch_id") }}' - - - name: BATCH_RUN_DATE - description: '{{ doc("batch_run_date") }}' - - - name: BATCH_INSERT_TS - description: '{{ doc("batch_insert_ts") }}' + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: SPONSOR description: '{{ doc("sponsor") }}' - name: NUM_SPONSORED description: '{{ doc("num_sponsored") }}' - tests: - - not_null - name: NUM_SPONSORING description: '{{ doc("num_sponsoring") }}' - tests: - - not_null - name: SEQUENCE_LEDGER description: '{{ doc("sequence_ledger") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: SEQUENCE_TIME description: '{{ doc("sequence_time") }}' - tests: - - not_null - name: CLOSED_AT description: '{{ doc("closed_at") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' - name: LEDGER_SEQUENCE description: '{{ doc("ledger_sequence") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: FACT_ACCOUNTS_ID description: '{{ doc("pk") }}' - tests: - - unique - - not_null - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' - tests: - - not_null - name: MODIFIED_TIMESTAMP - description: '{{ doc("modified_timestamp") }}' - tests: - - not_null - - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - ACCOUNT_ID - - CLOSED_AT \ No newline at end of file + description: '{{ doc("modified_timestamp") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_ledgers.yml b/models/gold/core/core__fact_ledgers.yml index d4c8993..7d5e436 100644 --- a/models/gold/core/core__fact_ledgers.yml +++ b/models/gold/core/core__fact_ledgers.yml @@ -8,135 +8,91 @@ models: description: '{{ doc("sequence") }}' tests: - dbt_expectations.expect_column_to_exist - - not_null - - unique + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: LEDGER_HASH description: '{{ doc("ledger_hash") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null - name: PREVIOUS_LEDGER_HASH description: '{{ doc("previous_ledger_hash") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: TRANSACTION_COUNT description: '{{ doc("transaction_count") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: OPERATION_COUNT description: '{{ doc("operation_count") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: CLOSED_AT description: '{{ doc("closed_at") }}' tests: - - dbt_expectations.expect_column_to_exist - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' - name: ID description: '{{ doc("id") }}' - tests: - - dbt_expectations.expect_column_to_exist - - not_null - name: TOTAL_COINS description: '{{ doc("total_coins") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: FEE_POOL description: '{{ doc("fee_pool") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: BASE_FEE description: '{{ doc("base_fee") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: BASE_RESERVE description: '{{ doc("base_reserve") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: MAX_TX_SET_SIZE description: '{{ doc("max_tx_set_size") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: PROTOCOL_VERSION description: '{{ doc("protocol_version") }}' - tests: - - dbt_expectations.expect_column_to_exist - - # - name: LEDGER_HEADER - # description: '{{ doc("ledger_header") }}' - # tests: - # - dbt_expectations.expect_column_to_exist - name: SUCCESSFUL_TRANSACTION_COUNT description: '{{ doc("successful_transaction_count") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: FAILED_TRANSACTION_COUNT description: '{{ doc("failed_transaction_count") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: TX_SET_OPERATION_COUNT description: '{{ doc("tx_set_operation_count") }}' - tests: - - dbt_expectations.expect_column_to_exist + + - name: BATCH_ID + description: '{{ doc("batch_id") }}' + + - name: BATCH_RUN_DATE + description: '{{ doc("batch_run_date") }}' + + - name: BATCH_INSERT_TS + description: '{{ doc("batch_insert_ts") }}' - name: SOROBAN_FEE_WRITE_1KB description: '{{ doc("soroban_fee_write_1kb") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: NODE_ID description: '{{ doc("node_id") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: SIGNATURE description: '{{ doc("signature") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: TOTAL_BYTE_SIZE_OF_BUCKET_LIST description: '{{ doc("total_byte_size_of_bucket_list") }}' - tests: - - dbt_expectations.expect_column_to_exist - name: FACT_LEDGERS_ID description: '{{ doc("pk") }}' - tests: - - unique - - not_null - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' - tests: - - not_null - name: MODIFIED_TIMESTAMP description: '{{ doc("modified_timestamp") }}' - tests: - - not_null - name: _INVOCATION_ID description: '{{ doc("invocation_id") }}' - - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - SEQUENCE diff --git a/models/gold/core/core__fact_operations.yml b/models/gold/core/core__fact_operations.yml index 2a567ec..63591b8 100644 --- a/models/gold/core/core__fact_operations.yml +++ b/models/gold/core/core__fact_operations.yml @@ -3,12 +3,31 @@ version: 2 models: - name: core__fact_operations description: A comprehensive view of all operations executed on the Stellar network, including transaction details, operation types, and their outcomes. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ID + - SOURCE_ACCOUNT + - TRANSACTION_ID + - LEDGER_SEQUENCE + - TYPE + - TYPE_STRING + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} columns: - name: ID description: '{{ doc("op_id") }}' tests: - - unique - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: CLOSED_AT + description: '{{ doc("closed_at") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' - name: ACCOUNT description: '{{ doc("account") }}' @@ -88,11 +107,8 @@ models: - name: PRICE description: '{{ doc("price") }}' - - name: D - description: '{{ doc("d") }}' - - - name: N - description: '{{ doc("n") }}' + - name: PRICE_R + description: '{{ doc("price_r") }}' - name: SELLING_ASSET_CODE description: '{{ doc("selling_asset_code") }}' @@ -111,6 +127,9 @@ models: - name: SET_FLAGS_S description: '{{ doc("set_flags_s") }}' + + - name: SIGNER_ACCOUNT_ID + description: '{{ doc("signer_account_id") }}' - name: SIGNER_KEY description: '{{ doc("signer_key") }}' @@ -219,22 +238,19 @@ models: - name: RESERVE_B_DEPOSIT_AMOUNT description: '{{ doc("reserve_b_deposit_amount") }}' + + - name: MIN_PRICE + description: '{{ doc("min_price") }}' - name: MIN_PRICE_R description: '{{ doc("min_price_r") }}' + + - name: MAX_PRICE + description: '{{ doc("max_price") }}' - name: MAX_PRICE_R description: '{{ doc("max_price_r") }}' - - name: SHARES_RECEIVED - description: '{{ doc("shares_received") }}' - - - name: RESERVE_A_MIN_AMOUNT - description: '{{ doc("reserve_a_min_amount") }}' - - - name: RESERVE_B_MIN_AMOUNT - description: '{{ doc("reserve_b_min_amount") }}' - - name: SHARES description: '{{ doc("shares") }}' @@ -244,194 +260,164 @@ models: - name: RESERVE_B_WITHDRAW_AMOUNT description: '{{ doc("reserve_b_withdraw_amount") }}' - - name: OP_SOURCE_ACCOUNT - description: '{{ doc("op_source_account") }}' + - name: SOURCE_ACCOUNT + description: '{{ doc("source_account") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: OP_SOURCE_ACCOUNT_MUXED description: '{{ doc("op_source_account_muxed") }}' - name: TRANSACTION_ID description: '{{ doc("transaction_id") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: TYPE description: '{{ doc("type") }}' - - - name: TRANSACTION_HASH - description: '{{ doc("transaction_hash") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: TYPE_STRING + description: '{{ doc("type_string") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: LEDGER_SEQUENCE description: '{{ doc("ledger_sequence") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - - name: TXN_ACCOUNT - description: '{{ doc("txn_account") }}' - - - name: ACCOUNT_SEQUENCE - description: '{{ doc("account_sequence") }}' - - - name: MAX_FEE - description: '{{ doc("max_fee") }}' - - - name: TXN_OPERATION_COUNT - description: '{{ doc("txn_operation_count") }}' - - - name: TXN_CREATED_AT - description: '{{ doc("txn_created_at") }}' - - - name: MEMO_TYPE - description: '{{ doc("memo_type") }}' - - - name: MEMO - description: '{{ doc("memo") }}' - - - name: TIME_BOUNDS - description: '{{ doc("time_bounds") }}' - - - name: SUCCESSFUL - description: '{{ doc("successful") }}' - - - name: FEE_CHARGED - description: '{{ doc("fee_charged") }}' - - - name: FEE_ACCOUNT - description: '{{ doc("fee_account") }}' - - - name: NEW_MAX_FEE - description: '{{ doc("new_max_fee") }}' - - - name: ACCOUNT_MUXED + - name: OP_ACCOUNT_MUXED description: '{{ doc("account_muxed") }}' - - name: FEE_ACCOUNT_MUXED - description: '{{ doc("fee_account_muxed") }}' - - - name: LEDGER_HASH - description: '{{ doc("ledger_hash") }}' - - - name: PREVIOUS_LEDGER_HASH - description: '{{ doc("previous_ledger_hash") }}' - - - name: TRANSACTION_COUNT - description: '{{ doc("transaction_count") }}' - - - name: LEDGER_OPERATION_COUNT - description: '{{ doc("ledger_operation_count") }}' - - - name: CLOSED_AT - description: '{{ doc("closed_at") }}' - - - name: TOTAL_COINS - description: '{{ doc("total_coins") }}' - - - name: FEE_POOL - description: '{{ doc("fee_pool") }}' - - - name: BASE_FEE - description: '{{ doc("base_fee") }}' - - - name: BASE_RESERVE - description: '{{ doc("base_reserve") }}' - - - name: MAX_TX_SET_SIZE - description: '{{ doc("max_tx_set_size") }}' - - - name: PROTOCOL_VERSION - description: '{{ doc("protocol_version") }}' - - - name: SUCCESSFUL_TRANSACTION_COUNT - description: '{{ doc("successful_transaction_count") }}' - - - name: FAILED_TRANSACTION_COUNT - description: '{{ doc("failed_transaction_count") }}' - + - name: OP_ACCOUNT_MUXED_ID + description: '{{ doc("account_muxed_id") }}' + + - name: LEDGER_KEY_HASH + description: '{{ doc("ledger_key_hash") }}' + - name: BATCH_ID description: '{{ doc("batch_id") }}' - name: BATCH_RUN_DATE - description: '{{ doc("batch_run_date") }}' + description: '{{ doc("batch_run_date") }}' - name: BATCH_INSERT_TS description: '{{ doc("batch_insert_ts") }}' - - name: LEDGER_BOUNDS - description: '{{ doc("ledger_bounds") }}' - - - name: MIN_ACCOUNT_SEQUENCE - description: '{{ doc("min_account_sequence") }}' - - - name: MIN_ACCOUNT_SEQUENCE_AGE - description: '{{ doc("min_account_sequence_age") }}' - - - name: MIN_ACCOUNT_SEQUENCE_LEDGER_GAP - description: '{{ doc("min_account_sequence_ledger_gap") }}' - - - name: EXTRA_SIGNERS - description: '{{ doc("extra_signers") }}' - - name: ASSET_BALANCE_CHANGES description: '{{ doc("asset_balance_changes") }}' - + - name: PARAMETERS description: '{{ doc("parameters") }}' - name: PARAMETERS_DECODED - description: '{{ doc("parameters_decoded") }}' - + description: '{{ doc("parameters_decoded") }}' + - name: FUNCTION description: '{{ doc("function") }}' - name: ADDRESS description: '{{ doc("address") }}' - + - name: SOROBAN_OPERATION_TYPE description: '{{ doc("soroban_operation_type") }}' - name: EXTEND_TO description: '{{ doc("extend_to") }}' - + - name: CONTRACT_ID description: '{{ doc("contract_id") }}' - name: CONTRACT_CODE_HASH description: '{{ doc("contract_code_hash") }}' - - - name: RESOURCE_FEE - description: '{{ doc("resource_fee") }}' - - - name: SOROBAN_RESOURCES_INSTRUCTIONS - description: '{{ doc("soroban_resources_instructions") }}' - - - name: SOROBAN_RESOURCES_READ_BYTES - description: '{{ doc("soroban_resources_read_bytes") }}' - - - name: SOROBAN_RESOURCES_WRITE_BYTES - description: '{{ doc("soroban_resources_write_bytes") }}' - - - name: TRANSACTION_RESULT_CODE - description: '{{ doc("transaction_result_code") }}' - - - name: INCLUSION_FEE_BID - description: '{{ doc("inclusion_fee_bid") }}' - - - name: INCLUSION_FEE_CHARGED - description: '{{ doc("inclusion_fee_charged") }}' - - - name: RESOURCE_FEE_REFUND - description: '{{ doc("resource_fee_refund") }}' - + - name: OPERATION_RESULT_CODE description: '{{ doc("operation_result_code") }}' - name: OPERATION_TRACE_CODE description: '{{ doc("operation_trace_code") }}' + + - name: BEGIN_SPONSOR_MUXED + description: '{{ doc("begin_sponsor_muxed") }}' - - name: OP_APPLICATION_ORDER - description: '{{ doc("op_application_order") }}' + - name: BEGIN_SPONSOR_MUXED_ID + description: '{{ doc("begin_sponsor_muxed_id") }}' - - name: TXN_APPLICATION_ORDER - description: '{{ doc("txn_application_order") }}' + - name: CLAIMABLE_BALANCE_ID + description: '{{ doc("claimable_balance_id") }}' - - name: _INSERTED_TIMESTAMP - description: '{{ doc("inserted_timestamp") }}' + - name: CLAIMANT + description: '{{ doc("claimant") }}' + + - name: CLAIMANTS + description: '{{ doc("claimants") }}' + + - name: CLAIMANT_MUXED + description: '{{ doc("claimant_muxed") }}' + + - name: CLAIMANT_MUXED_ID + description: '{{ doc("claimant_muxed_id") }}' + + - name: DATA_ACCOUNT_ID + description: '{{ doc("data_account_id") }}' + + - name: DATA_NAME + description: '{{ doc("data_name") }}' + + - name: DETAILS_JSON + description: '{{ doc("details_json") }}' + + - name: FROM_MUXED + description: '{{ doc("from_muxed") }}' + + - name: FROM_MUXED_ID + description: '{{ doc("from_muxed_id") }}' + + - name: FUNDER_MUXED + description: '{{ doc("funder_muxed") }}' + + - name: FUNDER_MUXED_ID + description: '{{ doc("funder_muxed_id") }}' + + - name: INTO_MUXED + description: '{{ doc("into_muxed") }}' + + - name: INTO_MUXED_ID + description: '{{ doc("into_muxed_id") }}' + + - name: LEDGERS_TO_EXPIRE + description: '{{ doc("ledgers_to_expire") }}' + + - name: OP_ACCOUNT_ID + description: '{{ doc("account_id") }}' + + - name: TO_MUXED + description: '{{ doc("to_muxed") }}' + + - name: TO_MUXED_ID + description: '{{ doc("to_muxed_id") }}' + + - name: TRUSTEE_MUXED + description: '{{ doc("trustee_muxed") }}' + + - name: TRUSTEE_MUXED_ID + description: '{{ doc("trustee_muxed_id") }}' + + - name: TRUSTLINE_ACCOUNT_ID + description: '{{ doc("trustline_account_id") }}' + + - name: TRUSTOR_MUXED + description: '{{ doc("trustor_muxed") }}' + + - name: TRUSTOR_MUXED_ID + description: '{{ doc("trustor_muxed_id") }}' - name: FACT_OPERATIONS_ID description: '{{ doc("pk") }}' @@ -443,7 +429,4 @@ models: description: '{{ doc("modified_timestamp") }}' - name: _INVOCATION_ID - description: '{{ doc("invocation_id") }}' - - - name: DETAILS_JSON - description: '{{ doc("details_json") }}' \ No newline at end of file + description: '{{ doc("invocation_id") }}' \ No newline at end of file diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index b3c5fb0..aaa4938 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -4,190 +4,164 @@ models: - name: core__fact_transactions description: Fact table containing transaction details from the Stellar network. columns: - # - name: id - # description: '{{ doc("pk") }}' - # tests: - # - unique - # - not_null - - name: id + - name: ID description: Unique identifier for the transaction tests: - - not_null - - unique + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - - name: transaction_hash + - name: TRANSACTION_HASH description: '{{ doc("transaction_hash") }}' - tests: - - not_null - - name: ledger_sequence + - name: LEDGER_SEQUENCE description: '{{ doc("ledger_sequence") }}' tests: - - not_null - - - name: account - description: '{{ doc("account") }}' - tests: - - not_null - - - name: account_sequence - description: '{{ doc("account_sequence") }}' - tests: - - not_null - - - name: max_fee - description: '{{ doc("max_fee") }}' - tests: - - not_null - - - name: operation_count - description: '{{ doc("operation_count") }}' - tests: - - not_null - - - name: created_at - description: '{{ doc("txn_created_at") }}' - tests: - - not_null - - - name: memo_type - description: '{{ doc("memo_type") }}' - - - name: memo - description: '{{ doc("memo") }}' - - - name: time_bounds - description: '{{ doc("time_bounds") }}' - - - name: successful - description: '{{ doc("successful") }}' - # tests: - # - not_null - - - name: fee_charged - description: '{{ doc("fee_charged") }}' - tests: - - not_null - - - name: inner_transaction_hash - description: '{{ doc("inner_transaction_hash") }}' - - - name: fee_account - description: '{{ doc("fee_account") }}' - - - name: new_max_fee - description: '{{ doc("new_max_fee") }}' - - - name: account_muxed - description: '{{ doc("account_muxed") }}' - - - name: fee_account_muxed - description: '{{ doc("fee_account_muxed") }}' - - - name: batch_id - description: '{{ doc("batch_id") }}' - tests: - - not_null - - - name: batch_run_date - description: '{{ doc("batch_run_date") }}' - tests: - - not_null - - - name: batch_insert_ts - description: '{{ doc("batch_insert_ts") }}' - tests: - - not_null - - - name: ledger_bounds - description: '{{ doc("ledger_bounds") }}' - - - name: min_account_sequence - description: '{{ doc("min_account_sequence") }}' - - - name: min_account_sequence_age - description: '{{ doc("min_account_sequence_age") }}' - - - name: min_account_sequence_ledger_gap - description: '{{ doc("min_account_sequence_ledger_gap") }}' - - - name: tx_envelope - description: '{{ doc("tx_envelope") }}' - - - name: tx_result - description: '{{ doc("tx_result") }}' - - - name: tx_meta - description: '{{ doc("tx_meta") }}' - - - name: tx_fee_meta - description: '{{ doc("tx_fee_meta") }}' - - - name: extra_signers - description: '{{ doc("extra_signers") }}' - - - name: resource_fee - description: '{{ doc("resource_fee") }}' - - - name: soroban_resources_instructions - description: '{{ doc("soroban_resources_instructions") }}' - - - name: soroban_resources_read_bytes - description: '{{ doc("soroban_resources_read_bytes") }}' - - - name: soroban_resources_write_bytes - description: '{{ doc("soroban_resources_write_bytes") }}' - - - name: closed_at + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: CLOSED_AT description: '{{ doc("closed_at") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' - - name: transaction_result_code + - name: ACCOUNT + description: '{{ doc("account") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: ACCOUNT_SEQUENCE + description: '{{ doc("account_sequence") }}' + + - name: MAX_FEE + description: '{{ doc("max_fee") }}' + + - name: OPERATION_COUNT + description: '{{ doc("operation_count") }}' + + - name: CREATED_AT + description: '{{ doc("tx_created_at") }}' + + - name: MEMO_TYPE + description: '{{ doc("memo_type") }}' + + - name: MEMO + description: '{{ doc("memo") }}' + + - name: TIME_BOUNDS + description: '{{ doc("time_bounds") }}' + + - name: SUCCESSFUL + description: '{{ doc("successful") }}' + + - name: FEE_CHARGED + description: '{{ doc("fee_charged") }}' + + - name: INNER_TRANSACTION_HASH + description: '{{ doc("inner_transaction_hash") }}' + + - name: FEE_ACCOUNT + description: '{{ doc("fee_account") }}' + + - name: NEW_MAX_FEE + description: '{{ doc("new_max_fee") }}' + + - name: ACCOUNT_MUXED + description: '{{ doc("account_muxed") }}' + + - name: FEE_ACCOUNT_MUXED + description: '{{ doc("fee_account_muxed") }}' + + - name: BATCH_ID + description: '{{ doc("batch_id") }}' + + - name: BATCH_RUN_DATE + description: '{{ doc("batch_run_date") }}' + + - name: BATCH_INSERT_TS + description: '{{ doc("batch_insert_ts") }}' + + - name: LEDGER_BOUNDS + description: '{{ doc("ledger_bounds") }}' + + - name: MIN_ACCOUNT_SEQUENCE + description: '{{ doc("min_account_sequence") }}' + + - name: MIN_ACCOUNT_SEQUENCE_AGE + description: '{{ doc("min_account_sequence_age") }}' + + - name: MIN_ACCOUNT_SEQUENCE_LEDGER_GAP + description: '{{ doc("min_account_sequence_ledger_gap") }}' + + - name: TX_ENVELOPE + description: '{{ doc("tx_envelope") }}' + + - name: TX_RESULT + description: '{{ doc("tx_result") }}' + + - name: TX_META + description: '{{ doc("tx_meta") }}' + + - name: TX_FEE_META + description: '{{ doc("tx_fee_meta") }}' + + - name: EXTRA_SIGNERS + description: '{{ doc("extra_signers") }}' + + - name: RESOURCE_FEE + description: '{{ doc("resource_fee") }}' + + - name: SOROBAN_RESOURCES_INSTRUCTIONS + description: '{{ doc("soroban_resources_instructions") }}' + + - name: SOROBAN_RESOURCES_READ_BYTES + description: '{{ doc("soroban_resources_read_bytes") }}' + + - name: SOROBAN_RESOURCES_WRITE_BYTES + description: '{{ doc("soroban_resources_write_bytes") }}' + + - name: TRANSACTION_RESULT_CODE description: '{{ doc("transaction_result_code") }}' - - name: inclusion_fee_bid + - name: INCLUSION_FEE_BID description: '{{ doc("inclusion_fee_bid") }}' - - name: inclusion_fee_charged + - name: INCLUSION_FEE_CHARGED description: '{{ doc("inclusion_fee_charged") }}' - - name: resource_fee_refund + - name: RESOURCE_FEE_REFUND description: '{{ doc("resource_fee_refund") }}' - - name: non_refundable_resource_fee_charged + - name: NON_REFUNDABLE_RESOURCE_FEE_CHARGED description: '{{ doc("non_refundable_resource_fee_charged") }}' - - name: refundable_resource_fee_charged + - name: REFUNDABLE_RESOURCE_FEE_CHARGED description: '{{ doc("refundable_resource_fee_charged") }}' - - name: rent_fee_charged + - name: RENT_FEE_CHARGED description: '{{ doc("rent_fee_charged") }}' - - name: tx_signers + - name: TX_SIGNERS description: '{{ doc("tx_signers") }}' - - name: refundable_fee + - name: REFUNDABLE_FEE description: '{{ doc("refundable_fee") }}' - - name: fact_transactions_id + - name: FACT_TRANSACTIONS_ID description: '{{ doc("pk") }}' - tests: - - not_null - - unique - - name: inserted_timestamp + - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' - tests: - - not_null - - name: modified_timestamp + - name: MODIFIED_TIMESTAMP description: '{{ doc("modified_timestamp") }}' - tests: - - not_null - - name: _invocation_id + - name: _INVOCATION_ID description: '{{ doc("invocation_id") }}' - tests: - - not_null diff --git a/models/gold/defi/defi__fact_liquidity_pools.yml b/models/gold/defi/defi__fact_liquidity_pools.yml index ad056ac..d64716d 100644 --- a/models/gold/defi/defi__fact_liquidity_pools.yml +++ b/models/gold/defi/defi__fact_liquidity_pools.yml @@ -9,7 +9,8 @@ models: - name: LIQUIDITY_POOL_ID description: '{{ doc("liquidity_pool_id") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: TYPE description: '{{ doc("type") }}' @@ -74,28 +75,20 @@ models: - name: CLOSED_AT description: '{{ doc("closed_at") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: LEDGER_SEQUENCE description: '{{ doc("ledger_sequence") }}' - name: FACT_LIQUIDITY_POOLS_ID description: '{{ doc("pk") }}' - tests: - - unique - - not_null - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' - tests: - - not_null - name: MODIFIED_TIMESTAMP description: '{{ doc("modified_timestamp") }}' - tests: - - not_null - name: _INVOCATION_ID - description: '{{ doc("invocation_id") }}' - tests: - - not_null + description: '{{ doc("invocation_id") }}' \ No newline at end of file diff --git a/models/gold/defi/defi__fact_trades.yml b/models/gold/defi/defi__fact_trades.yml index 5c25d30..f8e93ed 100644 --- a/models/gold/defi/defi__fact_trades.yml +++ b/models/gold/defi/defi__fact_trades.yml @@ -7,22 +7,26 @@ models: - name: history_operation_id description: '{{ doc("history_operation_id") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: trade_order description: '{{ doc("order") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: ledger_closed_at description: '{{ doc("closed_at") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: selling_account_address description: '{{ doc("selling_account_address") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: selling_asset_code description: '{{ doc("selling_asset_code") }}' @@ -33,22 +37,26 @@ models: - name: selling_asset_type description: '{{ doc("selling_asset_type") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: selling_asset_id description: '{{ doc("selling_asset_id") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: selling_amount description: '{{ doc("selling_amount") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: buying_account_address description: '{{ doc("buying_account_address") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: buying_asset_code description: '{{ doc("buying_asset_code") }}' @@ -59,27 +67,32 @@ models: - name: buying_asset_type description: '{{ doc("buying_asset_type") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: buying_asset_id description: '{{ doc("buying_asset_id") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: buying_amount description: '{{ doc("buying_amount") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: price_n description: '{{ doc("price_n") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: price_d description: '{{ doc("price_d") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: selling_offer_id description: '{{ doc("selling_offer_id") }}' @@ -96,7 +109,8 @@ models: - name: trade_type description: '{{ doc("trade_type") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - name: rounding_slippage description: '{{ doc("rounding_slippage") }}' @@ -106,24 +120,15 @@ models: - name: fact_trades_id description: '{{ doc("pk") }}' - tests: - - unique - - not_null - name: _inserted_timestamp description: '{{ doc("inserted_timestamp") }}' - tests: - - not_null - name: inserted_timestamp description: '{{ doc("inserted_timestamp") }}' - tests: - - not_null - name: modified_timestamp description: '{{ doc("modified_timestamp") }}' - tests: - - not_null - name: _invocation_id description: '{{ doc("invocation_id") }}' diff --git a/models/gold/operations/core__ez_operations.yml b/models/gold/operations/core__ez_operations.yml index 579d1f0..2bb838f 100644 --- a/models/gold/operations/core__ez_operations.yml +++ b/models/gold/operations/core__ez_operations.yml @@ -3,113 +3,544 @@ version: 2 models: - name: core__ez_operations description: A comprehensive view combining operations with their associated transaction and ledger data. - config: - tags: ["core"] - columns: - - name: fact_operations_id - description: '{{ doc("pk") }}' - tests: - - unique - - not_null - - name: op_id + - name: OP_ID description: '{{ doc("op_id") }}' tests: - - not_null + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - - name: transaction_id - description: '{{ doc("transaction_id") }}' + - name: OP_SOURCE_ACCOUNT + description: '{{ doc("op_source_account") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} - - name: type - description: '{{ doc("type") }}' + - name: OP_SOURCE_ACCOUNT_MUXED + description: '{{ doc("op_source_account_muxed") }}' - - name: account - description: '{{ doc("account") }}' + - name: OP_ACCOUNT_ID + description: '{{ doc("account_id") }}' - - name: amount + - name: AMOUNT description: '{{ doc("amount") }}' - - name: asset_code + - name: ASSET + description: '{{ doc("asset") }}' + + - name: ASSET_CODE description: '{{ doc("asset_code") }}' - - name: asset_issuer + - name: ASSET_ISSUER description: '{{ doc("asset_issuer") }}' - - name: asset_type + - name: ASSET_TYPE description: '{{ doc("asset_type") }}' - - name: transaction_hash - description: '{{ doc("transaction_hash") }}' + - name: AUTHORIZE + description: '{{ doc("authorize") }}' - - name: ledger_sequence - description: '{{ doc("ledger_sequence") }}' + - name: BALANCE_ID + description: '{{ doc("balance_id") }}' - - name: closed_at - description: '{{ doc("closed_at") }}' + - name: CLAIMANT + description: '{{ doc("claimant") }}' - - name: successful - description: '{{ doc("successful") }}' + - name: CLAIMANT_MUXED + description: '{{ doc("claimant_muxed") }}' - - name: operation_result_code - description: '{{ doc("operation_result_code") }}' + - name: CLAIMANT_MUXED_ID + description: '{{ doc("claimant_muxed_id") }}' - - name: operation_trace_code - description: '{{ doc("operation_trace_code") }}' + - name: CLAIMANTS + description: '{{ doc("claimants") }}' - - name: details_json - description: '{{ doc("details_json") }}' + - name: DATA_ACCOUNT_ID + description: '{{ doc("data_account_id") }}' - - name: asset_balance_changes + - name: DATA_NAME + description: '{{ doc("data_name") }}' + + - name: BUYING_ASSET_CODE + description: '{{ doc("buying_asset_code") }}' + + - name: BUYING_ASSET_ISSUER + description: '{{ doc("buying_asset_issuer") }}' + + - name: BUYING_ASSET_TYPE + description: '{{ doc("buying_asset_type") }}' + + - name: FROM_ACCOUNT + description: '{{ doc("from_account") }}' + + - name: FROM_MUXED + description: '{{ doc("from_muxed") }}' + + - name: FROM_MUXED_ID + description: '{{ doc("from_muxed_id") }}' + + - name: FUNDER + description: '{{ doc("funder") }}' + + - name: FUNDER_MUXED + description: '{{ doc("funder_muxed") }}' + + - name: FUNDER_MUXED_ID + description: '{{ doc("funder_muxed_id") }}' + + - name: HIGH_THRESHOLD + description: '{{ doc("high_threshold") }}' + + - name: HOME_DOMAIN + description: '{{ doc("home_domain") }}' + + - name: INFLATION_DEST + description: '{{ doc("inflation_dest") }}' + + - name: INTO_ACCOUNT + description: '{{ doc("into") }}' + + - name: INTO_MUXED + description: '{{ doc("into_muxed") }}' + + - name: INTO_MUXED_ID + description: '{{ doc("into_muxed_id") }}' + + - name: LIMIT_AMOUNT + description: '{{ doc("limit_amount") }}' + + - name: LOW_THRESHOLD + description: '{{ doc("low_threshold") }}' + + - name: MASTER_KEY_WEIGHT + description: '{{ doc("master_key_weight") }}' + + - name: MED_THRESHOLD + description: '{{ doc("med_threshold") }}' + + - name: NAME + description: '{{ doc("name") }}' + + - name: OFFER_ID + description: '{{ doc("offer_id") }}' + + - name: PATH + description: '{{ doc("path") }}' + + - name: PRICE + description: '{{ doc("price") }}' + + - name: PRICE_R + description: '{{ doc("price_r") }}' + + - name: SELLING_ASSET_CODE + description: '{{ doc("selling_asset_code") }}' + + - name: SELLING_ASSET_ISSUER + description: '{{ doc("selling_asset_issuer") }}' + + - name: SELLING_ASSET_TYPE + description: '{{ doc("selling_asset_type") }}' + + - name: SET_FLAGS + description: '{{ doc("set_flags") }}' + + - name: SET_FLAGS_S + description: '{{ doc("set_flags_s") }}' + + - name: SIGNER_ACCOUNT_ID + description: '{{ doc("signer_account_id") }}' + + - name: SIGNER_KEY + description: '{{ doc("signer_key") }}' + + - name: SIGNER_WEIGHT + description: '{{ doc("signer_weight") }}' + + - name: SOURCE_AMOUNT + description: '{{ doc("source_amount") }}' + + - name: SOURCE_ASSET_CODE + description: '{{ doc("source_asset_code") }}' + + - name: SOURCE_ASSET_ISSUER + description: '{{ doc("source_asset_issuer") }}' + + - name: SOURCE_ASSET_TYPE + description: '{{ doc("source_asset_type") }}' + + - name: SOURCE_MAX + description: '{{ doc("source_max") }}' + + - name: STARTING_BALANCE + description: '{{ doc("starting_balance") }}' + + - name: TO_ACCOUNT + description: '{{ doc("to") }}' + + - name: TO_MUXED + description: '{{ doc("to_muxed") }}' + + - name: TO_MUXED_ID + description: '{{ doc("to_muxed_id") }}' + + - name: TRUSTEE + description: '{{ doc("trustee") }}' + + - name: TRUSTEE_MUXED + description: '{{ doc("trustee_muxed") }}' + + - name: TRUSTEE_MUXED_ID + description: '{{ doc("trustee_muxed_id") }}' + + - name: TRUSTOR + description: '{{ doc("trustor") }}' + + - name: TRUSTOR_MUXED + description: '{{ doc("trustor_muxed") }}' + + - name: TRUSTOR_MUXED_ID + description: '{{ doc("trustor_muxed_id") }}' + + - name: TRUSTLINE_ACCOUNT_ID + description: '{{ doc("trustline_account_id") }}' + + - name: TRUSTLINE_ASSET + description: '{{ doc("trustline_asset") }}' + + - name: VALUE + description: '{{ doc("value") }}' + + - name: CLEAR_FLAGS + description: '{{ doc("clear_flags") }}' + + - name: CLEAR_FLAGS_S + description: '{{ doc("clear_flags_s") }}' + + - name: DESTINATION_MIN + description: '{{ doc("destination_min") }}' + + - name: BUMP_TO + description: '{{ doc("bump_to") }}' + + - name: SPONSOR + description: '{{ doc("sponsor") }}' + + - name: SPONSORED_ID + description: '{{ doc("sponsored_id") }}' + + - name: BEGIN_SPONSOR + description: '{{ doc("begin_sponsor") }}' + + - name: BEGIN_SPONSOR_MUXED + description: '{{ doc("begin_sponsor_muxed") }}' + + - name: BEGIN_SPONSOR_MUXED_ID + description: '{{ doc("begin_sponsor_muxed_id") }}' + + - name: AUTHORIZE_TO_MAINTAIN_LIABILITIES + description: '{{ doc("authorize_to_maintain_liabilities") }}' + + - name: CLAWBACK_ENABLED + description: '{{ doc("clawback_enabled") }}' + + - name: LIQUIDITY_POOL_ID + description: '{{ doc("liquidity_pool_id") }}' + + - name: RESERVE_A_ASSET_TYPE + description: '{{ doc("reserve_a_asset_type") }}' + + - name: RESERVE_A_ASSET_CODE + description: '{{ doc("reserve_a_asset_code") }}' + + - name: RESERVE_A_ASSET_ISSUER + description: '{{ doc("reserve_a_asset_issuer") }}' + + - name: RESERVE_A_MAX_AMOUNT + description: '{{ doc("reserve_a_max_amount") }}' + + - name: RESERVE_A_DEPOSIT_AMOUNT + description: '{{ doc("reserve_a_deposit_amount") }}' + + - name: RESERVE_B_ASSET_TYPE + description: '{{ doc("reserve_b_asset_type") }}' + + - name: RESERVE_B_ASSET_CODE + description: '{{ doc("reserve_b_asset_code") }}' + + - name: RESERVE_B_ASSET_ISSUER + description: '{{ doc("reserve_b_asset_issuer") }}' + + - name: RESERVE_B_MAX_AMOUNT + description: '{{ doc("reserve_b_max_amount") }}' + + - name: RESERVE_B_DEPOSIT_AMOUNT + description: '{{ doc("reserve_b_deposit_amount") }}' + + - name: MIN_PRICE + description: '{{ doc("min_price") }}' + + - name: MIN_PRICE_R + description: '{{ doc("min_price_r") }}' + + - name: MAX_PRICE + description: '{{ doc("max_price") }}' + + - name: MAX_PRICE_R + description: '{{ doc("max_price_r") }}' + + - name: SHARES_RECEIVED + description: '{{ doc("shares_received") }}' + + - name: RESERVE_A_MIN_AMOUNT + description: '{{ doc("reserve_a_min_amount") }}' + + - name: RESERVE_B_MIN_AMOUNT + description: '{{ doc("reserve_b_min_amount") }}' + + - name: SHARES + description: '{{ doc("shares") }}' + + - name: RESERVE_A_WITHDRAW_AMOUNT + description: '{{ doc("reserve_a_withdraw_amount") }}' + + - name: RESERVE_B_WITHDRAW_AMOUNT + description: '{{ doc("reserve_b_withdraw_amount") }}' + + - name: TX_ID + description: '{{ doc("transaction_id") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: TYPE + description: '{{ doc("type") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: TYPE_STRING + description: '{{ doc("type_string") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: BATCH_ID + description: '{{ doc("batch_id") }}' + + - name: BATCH_RUN_DATE + description: '{{ doc("batch_run_date") }}' + + - name: ASSET_BALANCE_CHANGES description: '{{ doc("asset_balance_changes") }}' - - name: parameters + - name: PARAMETERS description: '{{ doc("parameters") }}' - - name: parameters_decoded + - name: PARAMETERS_DECODED description: '{{ doc("parameters_decoded") }}' - - name: function + - name: FUNCTION description: '{{ doc("function") }}' - - name: contract_id + - name: ADDRESS + description: '{{ doc("address") }}' + + - name: SOROBAN_OPERATION_TYPE + description: '{{ doc("soroban_operation_type") }}' + + - name: EXTEND_TO + description: '{{ doc("extend_to") }}' + + - name: CONTRACT_ID description: '{{ doc("contract_id") }}' - - name: contract_code_hash + - name: CONTRACT_CODE_HASH description: '{{ doc("contract_code_hash") }}' - - name: resource_fee + - name: OPERATION_RESULT_CODE + description: '{{ doc("operation_result_code") }}' + + - name: OPERATION_TRACE_CODE + description: '{{ doc("operation_trace_code") }}' + + - name: DETAILS_JSON + description: '{{ doc("details_json") }}' + + - name: CLOSED_AT + description: '{{ doc("closed_at") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + + - name: TRANSACTION_HASH + description: '{{ doc("transaction_hash") }}' + + - name: LEDGER_SEQUENCE + description: '{{ doc("ledger_sequence") }}' + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + + - name: TX_ACCOUNT + description: '{{ doc("tx_account") }}' + + - name: ACCOUNT_SEQUENCE + description: '{{ doc("account_sequence") }}' + + - name: MAX_FEE + description: '{{ doc("max_fee") }}' + + - name: TX_OPERATION_COUNT + description: '{{ doc("tx_operation_count") }}' + + - name: TX_CREATED_AT + description: '{{ doc("tx_created_at") }}' + + - name: MEMO_TYPE + description: '{{ doc("memo_type") }}' + + - name: MEMO + description: '{{ doc("memo") }}' + + - name: TIME_BOUNDS + description: '{{ doc("time_bounds") }}' + + - name: SUCCESSFUL + description: '{{ doc("successful") }}' + + - name: FEE_CHARGED + description: '{{ doc("fee_charged") }}' + + - name: FEE_ACCOUNT + description: '{{ doc("fee_account") }}' + + - name: NEW_MAX_FEE + description: '{{ doc("new_max_fee") }}' + + - name: ACCOUNT_MUXED + description: '{{ doc("account_muxed") }}' + + - name: FEE_ACCOUNT_MUXED + description: '{{ doc("fee_account_muxed") }}' + + - name: LEDGER_BOUNDS + description: '{{ doc("ledger_bounds") }}' + + - name: MIN_ACCOUNT_SEQUENCE + description: '{{ doc("min_account_sequence") }}' + + - name: MIN_ACCOUNT_SEQUENCE_AGE + description: '{{ doc("min_account_sequence_age") }}' + + - name: MIN_ACCOUNT_SEQUENCE_LEDGER_GAP + description: '{{ doc("min_account_sequence_ledger_gap") }}' + + - name: EXTRA_SIGNERS + description: '{{ doc("extra_signers") }}' + + - name: RESOURCE_FEE description: '{{ doc("resource_fee") }}' - - name: soroban_resources_instructions + - name: SOROBAN_RESOURCES_INSTRUCTIONS description: '{{ doc("soroban_resources_instructions") }}' - - name: soroban_resources_read_bytes + - name: SOROBAN_RESOURCES_READ_BYTES description: '{{ doc("soroban_resources_read_bytes") }}' - - name: soroban_resources_write_bytes + - name: SOROBAN_RESOURCES_WRITE_BYTES description: '{{ doc("soroban_resources_write_bytes") }}' - - name: non_refundable_resource_fee_charged + - name: TRANSACTION_RESULT_CODE + description: '{{ doc("transaction_result_code") }}' + + - name: INCLUSION_FEE_BID + description: '{{ doc("inclusion_fee_bid") }}' + + - name: INCLUSION_FEE_CHARGED + description: '{{ doc("inclusion_fee_charged") }}' + + - name: RESOURCE_FEE_REFUND + description: '{{ doc("resource_fee_refund") }}' + + - name: LEDGER_HASH + description: '{{ doc("ledger_hash") }}' + + - name: PREVIOUS_LEDGER_HASH + description: '{{ doc("previous_ledger_hash") }}' + + - name: TRANSACTION_COUNT + description: '{{ doc("transaction_count") }}' + + - name: LEDGER_OPERATION_COUNT + description: '{{ doc("ledger_operation_count") }}' + + - name: LEDGER_ID + description: '{{ doc("ledger_id") }}' + + - name: TOTAL_COINS + description: '{{ doc("total_coins") }}' + + - name: FEE_POOL + description: '{{ doc("fee_pool") }}' + + - name: BASE_FEE + description: '{{ doc("base_fee") }}' + + - name: BASE_RESERVE + description: '{{ doc("base_reserve") }}' + + - name: MAX_TX_SET_SIZE + description: '{{ doc("max_tx_set_size") }}' + + - name: PROTOCOL_VERSION + description: '{{ doc("protocol_version") }}' + + - name: SUCCESSFUL_TRANSACTION_COUNT + description: '{{ doc("successful_transaction_count") }}' + + - name: FAILED_TRANSACTION_COUNT + description: '{{ doc("failed_transaction_count") }}' + + - name: SOROBAN_FEE_WRITE_1KB + description: '{{ doc("soroban_fee_write_1kb") }}' + + - name: NODE_ID + description: '{{ doc("node_id") }}' + + - name: SIGNATURE + description: '{{ doc("signature") }}' + + - name: TOTAL_BYTE_SIZE_OF_BUCKET_LIST + description: '{{ doc("total_byte_size_of_bucket_list") }}' + + - name: NON_REFUNDABLE_RESOURCE_FEE_CHARGED description: '{{ doc("non_refundable_resource_fee_charged") }}' - - name: refundable_resource_fee_charged + - name: REFUNDABLE_RESOURCE_FEE_CHARGED description: '{{ doc("refundable_resource_fee_charged") }}' - - name: rent_fee_charged + - name: RENT_FEE_CHARGED description: '{{ doc("rent_fee_charged") }}' - - name: tx_signers + - name: TX_SIGNERS description: '{{ doc("tx_signers") }}' - - name: refundable_fee + - name: REFUNDABLE_FEE description: '{{ doc("refundable_fee") }}' - - name: inserted_timestamp + - name: FACT_OPERATIONS_ID + description: '{{ doc("pk") }}' + + - name: INSERTED_TIMESTAMP description: '{{ doc("inserted_timestamp") }}' - - name: modified_timestamp + - name: MODIFIED_TIMESTAMP description: '{{ doc("modified_timestamp") }}' - - name: _invocation_id + - name: _INVOCATION_ID description: '{{ doc("invocation_id") }}' diff --git a/models/silver/silver__accounts.yml b/models/silver/silver__accounts.yml new file mode 100644 index 0000000..9bca5e9 --- /dev/null +++ b/models/silver/silver__accounts.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__accounts + + columns: + - name: ACCOUNT_ID + description: "{{ doc('account_id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - name: SEQUENCE_NUMBER + description: "{{ doc('sequence_number') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LEDGER_ENTRY_CHANGE + description: "{{ doc('ledger_entry_change') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LAST_MODIFIED_LEDGER + description: "{{ doc('last_modified_ledger') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER diff --git a/models/silver/silver__assets.yml b/models/silver/silver__assets.yml new file mode 100644 index 0000000..75ad0ca --- /dev/null +++ b/models/silver/silver__assets.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: silver__assets + + columns: + - name: ASSET_TYPE + description: "{{ doc('asset_type') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - name: ASSET_CODE + description: "{{ doc('asset_code') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - name: ASSET_ISSUER + description: "{{ doc('asset_issuer') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR \ No newline at end of file diff --git a/models/silver/silver__ledgers.yml b/models/silver/silver__ledgers.yml new file mode 100644 index 0000000..3c861bd --- /dev/null +++ b/models/silver/silver__ledgers.yml @@ -0,0 +1,9 @@ +version: 2 +models: + - name: silver__ledgers + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - SEQUENCE + - fsc_utils.sequence_gaps: + column_name: SEQUENCE \ No newline at end of file diff --git a/models/silver/silver__liquidity_pools.yml b/models/silver/silver__liquidity_pools.yml new file mode 100644 index 0000000..c41b5e8 --- /dev/null +++ b/models/silver/silver__liquidity_pools.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: silver__liquidity_pools + + columns: + - name: LIQUIDITY_POOL_ID + description: "{{ doc('liquidity_pool_id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - name: LEDGER_ENTRY_CHANGE + description: "{{ doc('ledger_entry_change') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LAST_MODIFIED_LEDGER + description: "{{ doc('last_modified_ledger') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER \ No newline at end of file diff --git a/models/silver/silver__operations.yml b/models/silver/silver__operations.yml new file mode 100644 index 0000000..9455124 --- /dev/null +++ b/models/silver/silver__operations.yml @@ -0,0 +1,53 @@ +version: 2 +models: + - name: silver__operations + + columns: + - name: ID + description: "{{ doc('id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: SOURCE_ACCOUNT + description: "{{ doc('source_account') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR + - name: TRANSACTION_ID + description: "{{ doc('transaction_id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: LEDGER_SEQUENCE + description: "{{ doc('ledger_sequence') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: TYPE + description: "{{ doc('type') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: TYPE_STRING + description: "{{ doc('type_string') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARCHAR \ No newline at end of file diff --git a/models/silver/silver__trades.yml b/models/silver/silver__trades.yml new file mode 100644 index 0000000..844716e --- /dev/null +++ b/models/silver/silver__trades.yml @@ -0,0 +1,21 @@ +version: 2 +models: + - name: silver__trades + + columns: + - name: HISTORY_OPERATION_ID + description: "{{ doc('history_operation_id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - name: '"order"' + description: "{{ doc('order') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER \ No newline at end of file diff --git a/models/silver/silver__transactions.yml b/models/silver/silver__transactions.yml new file mode 100644 index 0000000..6284578 --- /dev/null +++ b/models/silver/silver__transactions.yml @@ -0,0 +1,14 @@ +version: 2 +models: + - name: silver__transactions + columns: + - name: ID + description: "{{ doc('id') }}" + tests: + - not_null: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - unique: + where: modified_timestamp > current_date - {{ var('test_days_threshold', 3) }} + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER \ No newline at end of file diff --git a/models/streamline/core/_max_ledger_by_date.sql b/models/streamline/core/_max_ledger_by_date.sql new file mode 100644 index 0000000..2a4cc13 --- /dev/null +++ b/models/streamline/core/_max_ledger_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "closed_at", +) }} + +WITH base AS ( + + SELECT + closed_at :: DATE AS closed_at, + MAX(sequence) AS sequence + FROM + {{ ref("silver__ledgers") }} + GROUP BY + closed_at :: DATE +) +SELECT + closed_at, + sequence +FROM + base +WHERE + closed_at <> ( + SELECT + MAX(closed_at) + FROM + base + ) diff --git a/models/tests/recent/test_core__accounts_recent.sql b/models/tests/recent/test_core__accounts_recent.sql new file mode 100644 index 0000000..9d38692 --- /dev/null +++ b/models/tests/recent/test_core__accounts_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_accounts') }} +WHERE + closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_core__accounts_recent.yml b/models/tests/recent/test_core__accounts_recent.yml new file mode 100644 index 0000000..b2ef14e --- /dev/null +++ b/models/tests/recent/test_core__accounts_recent.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_core__accounts_recent + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/tests/recent/test_core__ez_operations_recent.sql b/models/tests/recent/test_core__ez_operations_recent.sql new file mode 100644 index 0000000..4a234df --- /dev/null +++ b/models/tests/recent/test_core__ez_operations_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__ez_operations') }} +WHERE + closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_core__ez_operations_recent.yml b/models/tests/recent/test_core__ez_operations_recent.yml new file mode 100644 index 0000000..b1a7156 --- /dev/null +++ b/models/tests/recent/test_core__ez_operations_recent.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_core__ez_operations_recent + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/tests/recent/test_core__ledgers_recent.sql b/models/tests/recent/test_core__ledgers_recent.sql new file mode 100644 index 0000000..9e43ace --- /dev/null +++ b/models/tests/recent/test_core__ledgers_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_ledgers') }} +WHERE + closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_core__ledgers_recent.yml b/models/tests/recent/test_core__ledgers_recent.yml new file mode 100644 index 0000000..8ad2d28 --- /dev/null +++ b/models/tests/recent/test_core__ledgers_recent.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_core__ledgers_recent + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/tests/recent/test_core__operations_recent.sql b/models/tests/recent/test_core__operations_recent.sql new file mode 100644 index 0000000..998fa2f --- /dev/null +++ b/models/tests/recent/test_core__operations_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_operations') }} +WHERE + closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_core__operations_recent.yml b/models/tests/recent/test_core__operations_recent.yml new file mode 100644 index 0000000..fee45bd --- /dev/null +++ b/models/tests/recent/test_core__operations_recent.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_core__operations_recent + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/tests/recent/test_core__transactions_recent.sql b/models/tests/recent/test_core__transactions_recent.sql new file mode 100644 index 0000000..44511f6 --- /dev/null +++ b/models/tests/recent/test_core__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('core__fact_transactions') }} +WHERE + closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_core__transactions_recent.yml b/models/tests/recent/test_core__transactions_recent.yml new file mode 100644 index 0000000..6376b32 --- /dev/null +++ b/models/tests/recent/test_core__transactions_recent.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_core__transactions_recent + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/tests/recent/test_defi__fact_liquidity_pools.sql b/models/tests/recent/test_defi__fact_liquidity_pools.sql new file mode 100644 index 0000000..ec66022 --- /dev/null +++ b/models/tests/recent/test_defi__fact_liquidity_pools.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('defi__fact_liquidity_pools') }} +WHERE + closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_defi__fact_liquidity_pools.yml b/models/tests/recent/test_defi__fact_liquidity_pools.yml new file mode 100644 index 0000000..7866cf4 --- /dev/null +++ b/models/tests/recent/test_defi__fact_liquidity_pools.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_defi__fact_liquidity_pools + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/tests/recent/test_defi__trades_recent.sql b/models/tests/recent/test_defi__trades_recent.sql new file mode 100644 index 0000000..c9a8273 --- /dev/null +++ b/models/tests/recent/test_defi__trades_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + closed_at + FROM + {{ ref("_max_ledger_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + closed_at DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('defi__fact_trades') }} +WHERE + ledger_closed_at :: DATE >= ( + SELECT + closed_at + FROM + last_3_days + ) diff --git a/models/tests/recent/test_defi__trades_recent.yml b/models/tests/recent/test_defi__trades_recent.yml new file mode 100644 index 0000000..eb630d5 --- /dev/null +++ b/models/tests/recent/test_defi__trades_recent.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_defi__trades_recent + columns: + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/python/dbt_test_alert.py b/python/dbt_test_alert.py new file mode 100644 index 0000000..0c00ca4 --- /dev/null +++ b/python/dbt_test_alert.py @@ -0,0 +1,127 @@ +import datetime +import requests +import json +import sys +import os + + +def log_test_result(): + """Reads the run_results.json file and returns a dictionary of targeted test results""" + + filepath = "target/run_results.json" + + with open(filepath) as f: + run = json.load(f) + + logs = [] + messages = { + "fail": [], + "warn": [] + } + test_count = 0 + warn_count = 0 + fail_count = 0 + + for test in run["results"]: + test_count += 1 + if test["status"] != "pass": + logs.append(test) + + message = f"{test['failures']} record failure(s) in {test['unique_id']}" + + if test["status"] == "warn": + messages["warn"].append(message) + warn_count += 1 + elif test["status"] == "fail": + messages["fail"].append(message) + fail_count += 1 + + dbt_test_result = { + "logs": logs, + "messages": messages, + "test_count": test_count, + "warn_count": warn_count, + "fail_count": fail_count, + "elapsed_time": str(datetime.timedelta(seconds=run["elapsed_time"])) + } + + return dbt_test_result + + +def create_message(**kwargs): + messageBody = { + "text": f"Hey{' ' if len(kwargs['messages']['fail']) > 0 else ''}, new DBT test results for :{os.environ.get('DATABASE').split('_DEV')[0]}: {os.environ.get('DATABASE')}", + "attachments": [ + { + "color": kwargs["color"], + "fields": [ + { + "title": "Total Tests Run", + "value": kwargs["test_count"], + "short": True + }, + { + "title": "Total Time Elapsed", + "value": kwargs["elapsed_time"], + "short": True + }, + { + "title": "Number of Unsuccessful Tests", + "value": f"Fail: {kwargs['fail_count']}, Warn: {kwargs['warn_count']}", + "short": True + }, + { + "title": "Failed Tests:", + "value": "\n".join(kwargs["messages"]["fail"]) if len(kwargs["messages"]["fail"]) > 0 else "None :)", + "short": False + } + ], + "actions": [ + + { + "type": "button", + "text": "View Warnings", + "style": "primary", + "url": "https://github.com/FlipsideCrypto/stellar-models/actions", + "confirm": { + "title": f"{kwargs['warn_count']} Warnings", + "text": "\n".join(kwargs["messages"]["warn"]) if len(kwargs["messages"]["warn"]) > 0 else "None :)", + "ok_text": "Continue to GHA", + "dismiss_text": "Dismiss" + } + } + ] + } + ] + } + + return messageBody + + +def send_alert(webhook_url): + """Sends a message to a slack channel""" + + url = webhook_url + + data = log_test_result() + + send_message = create_message( + fail_count=data["fail_count"], + warn_count=data["warn_count"], + test_count=data["test_count"], + messages=data["messages"], + elapsed_time=data["elapsed_time"], + color="#f44336" if data["fail_count"] > 0 else "#4CAF50" + ) + + x = requests.post(url, json=send_message) + + # test config to continue on error in workflow, so we want to exit with a non-zero code if there are any failures + if data['fail_count'] > 0: + sys.exit(1) + + +if __name__ == '__main__': + + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + send_alert(webhook_url) \ No newline at end of file