flip/near-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/near-models.git synced 2026-02-06 11:26:52 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

actions, atlas, curated dirs

Browse Source
This commit is contained in:
Jack Forgash 2023-12-11 10:34:32 -07:00
parent 1d4726eafb
commit c7b353be05
59 changed files with 419 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
models/silver/actions/silver__actions_events_addkey_s3.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
incremental_strategy = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
unique_key = 'action_id',
cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'],
tags = ['actions', 'curated']
@ -13,8 +14,7 @@ WITH action_events AS (
FROM
{{ ref('silver__actions_events_s3') }}
WHERE
action_name = 'AddKey'
{% if var("MANUAL_FIX") %}
action_name = 'AddKey' {% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
AND {{ incremental_load_filter('_inserted_timestamp') }}
@ -39,6 +39,12 @@ addkey_events AS (
action_events
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['action_id']
) }} AS actions_events_addkey_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
addkey_events

12
models/silver/actions/silver__actions_events_addkey_s3.yml
View File

@ -44,3 +44,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: actions_events_addkey_id
description: "{{doc('id')}}"
- name: inserted_timestamp
description: "{{doc('inserted_timestamp')}}"
- name: modified_timestamp
description: "{{doc('modified_timestamp')}}"
- name: _invocation_id
description: "{{doc('invocation_id')}}"

12
models/silver/actions/silver__actions_events_function_call_s3.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
incremental_strategy = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'],
unique_key = 'action_id',
tags = ['actions', 'curated']
@ -61,6 +62,13 @@ function_calls AS (
decoding
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['action_id']
) }} AS actions_events_function_call_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
function_calls

12
models/silver/actions/silver__actions_events_function_call_s3.yml
View File

@ -61,3 +61,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: ACTIONS_EVENTS_FUNCTION_CALL_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

11
models/silver/actions/silver__actions_events_s3.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
incremental_strategy = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'],
unique_key = 'action_id',
tags = ['actions', 'curated']
@ -83,6 +84,12 @@ SELECT
action_data,
_load_timestamp,
_partition_by_block_number,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['receipt_object_id', 'action_index']
) }} AS actions_events_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/actions/silver__actions_events_s3.yml
View File

@ -68,3 +68,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: ACTIONS_EVENTS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

2
models/silver/atlas/silver__atlas_accounts_created.sql
View File

@ -38,7 +38,7 @@ FINAL AS (
COUNT(*) AS wallets_created,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
accts
GROUP BY

4
models/silver/atlas/silver__atlas_accounts_created.yml
View File

@ -31,7 +31,5 @@ models:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{doc('invocation_id')}}"
tests:
- not_null

2
models/silver/atlas/silver__atlas_address_first_action.sql
View File

@ -48,7 +48,7 @@ SELECT
_inserted_timestamp,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL qualify ROW_NUMBER() over (
PARTITION BY address

4
models/silver/atlas/silver__atlas_address_first_action.yml
View File

@ -44,7 +44,5 @@ models:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"
tests:
- not_null

2
models/silver/atlas/silver__atlas_maa.sql
View File

@ -105,6 +105,6 @@ SELECT
_inserted_timestamp,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

4
models/silver/atlas/silver__atlas_maa.yml
View File

@ -49,7 +49,5 @@ models:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"
tests:
- not_null

2
models/silver/atlas/silver__atlas_nft_30_trailing.sql
View File

@ -44,6 +44,6 @@ SELECT
txns,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

4
models/silver/atlas/silver__atlas_nft_30_trailing.yml
View File

@ -32,7 +32,5 @@ models:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{doc('invocation_id')}}"
tests:
- not_null

2
models/silver/atlas/silver__atlas_nft_detailed.sql
View File

@ -36,7 +36,7 @@ SELECT
) AS mints,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id,
'{{ invocation_id }}' AS _invocation_id,
MAX(_inserted_timestamp) AS _inserted_timestamp
FROM
nft_data

4
models/silver/atlas/silver__atlas_nft_detailed.yml
View File

@ -57,7 +57,5 @@ columns:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{doc('invocation_id')}}"
tests:
- not_null

2
models/silver/atlas/silver__atlas_nft_table.sql
View File

@ -45,7 +45,7 @@ SELECT
) AS mints,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
nft_data
GROUP BY

4
models/silver/atlas/silver__atlas_nft_table.yml
View File

@ -62,7 +62,5 @@ models:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{doc('invocation_id')}}"
tests:
- not_null

2
models/silver/atlas/silver__atlas_nft_transactions.sql
View File

@ -83,7 +83,7 @@ SELECT
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
_inserted_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
unioned_nft_data
WHERE

4
models/silver/atlas/silver__atlas_nft_transactions.yml
View File

@ -67,7 +67,5 @@ models:
tests:
- not_null
- name: invocation_id
- name: _invocation_id
description: "{{doc('invocation_id')}}"
tests:
- not_null

2
models/silver/atlas/supply/silver__atlas_supply.sql
View File

@ -135,6 +135,6 @@ SELECT
{{ dbt_utils.generate_surrogate_key(['utc_date']) }} AS atlas_supply_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
output

2
models/silver/atlas/supply/silver__atlas_supply.yml
View File

@ -80,5 +80,5 @@ models:
- name: modified_timestamp
description: "{{ doc('modified_timestamp') }}"
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"

2
models/silver/atlas/supply/silver__atlas_supply_daily_lockup_locked_balances.sql
View File

@ -438,6 +438,6 @@ SELECT
{{ dbt_utils.generate_surrogate_key(['utc_date', 'lockup_account_id', 'lockup_index', 'owner_account_id']) }} AS atlas_daily_lockup_locked_balances_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
lockup_contracts_daily_balance

2
models/silver/atlas/supply/silver__atlas_supply_daily_lockup_locked_balances.yml
View File

@ -99,5 +99,5 @@ models:
- name: modified_timestamp
description: "{{ doc('modified_timestamp') }}"
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"

2
models/silver/atlas/supply/silver__atlas_supply_daily_lockup_staking_balances.sql
View File

@ -185,6 +185,6 @@ SELECT
{{ dbt_utils.generate_surrogate_key(['lockup_account_id', 'utc_date']) }} AS atlas_daily_lockup_staking_balances_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
lockup_stakers_daily_balances

2
models/silver/atlas/supply/silver__atlas_supply_daily_lockup_staking_balances.yml
View File

@ -33,5 +33,5 @@ models:
- name: modified_timestamp
description: "{{ doc('modified_timestamp') }}"
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"

2
models/silver/atlas/supply/silver__atlas_supply_daily_staked_supply.sql
View File

@ -309,6 +309,6 @@ SELECT
{{ dbt_utils.generate_surrogate_key(['utc_date']) }} AS atlas_daily_staked_supply_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

2
models/silver/atlas/supply/silver__atlas_supply_epochs.sql
View File

@ -52,6 +52,6 @@ SELECT
{{ dbt_utils.generate_surrogate_key(['epoch_id']) }} AS atlas_epochs_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
epochs

2
models/silver/atlas/supply/silver__atlas_supply_epochs.yml
View File

@ -51,5 +51,5 @@ models:
description: "{{ doc('inserted_timestamp') }}"
- name: modified_timestamp
description: "{{ doc('modified_timestamp') }}"
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"

2
models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.sql
View File

@ -40,7 +40,7 @@ FINAL AS (
{{ dbt_utils.generate_surrogate_key(['receipt_object_id']) }} AS atlas_supply_lockup_receipts_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS invocation_id
'{{ invocation_id }}' AS _invocation_id
FROM
receipts
WHERE

2
models/silver/atlas/supply/silver__atlas_supply_lockup_receipts.yml
View File

@ -49,5 +49,5 @@ models:
description: "{{ doc('inserted_timestamp') }}"
- name: modified_timestamp
description: "{{ doc('modified_timestamp') }}"
- name: invocation_id
- name: _invocation_id
description: "{{ doc('invocation_id') }}"

105
models/silver/curated/nft/silver__standard_nft_mint_s3.sql
View File

@ -2,7 +2,8 @@
materialized = "incremental",
cluster_by = ["block_timestamp::DATE"],
unique_key = "mint_action_id",
incremental_strategy = "delete+insert",
incremental_strategy = "merge",
merge_exclude_columns = ["inserted_timestamp"],
tags = ['curated']
) }}
@ -50,7 +51,11 @@ function_call AS (
standard_logs AS (
SELECT
action_id AS logs_id,
concat_ws('-', receipt_object_id, '0') as action_id,
concat_ws(
'-',
receipt_object_id,
'0'
) AS action_id,
tx_hash,
receipt_object_id,
block_id,
@ -62,7 +67,9 @@ standard_logs AS (
_PARTITION_BY_BLOCK_NUMBER,
_inserted_timestamp,
TRY_PARSE_JSON(clean_log) AS clean_log,
COUNT(*) OVER (PARTITION BY tx_hash) AS log_counter
COUNT(*) over (
PARTITION BY tx_hash
) AS log_counter
FROM
logs
WHERE
@ -81,7 +88,7 @@ nft_events AS (
FROM
standard_logs
LEFT JOIN function_call
ON standard_logs.ACTION_ID = function_call.ACTION_ID
ON standard_logs.action_id = function_call.action_id
WHERE
STANDARD = 'nep171' -- nep171 nft STANDARD, version nep245 IS multitoken STANDARD, nep141 IS fungible token STANDARD
AND event = 'nft_mint'
@ -146,9 +153,18 @@ mint_events AS (
concat_ws(
'-',
action_id,
COALESCE(batch_index, '0'),
COALESCE(token_index, '0'),
COALESCE(token_id, '0')
COALESCE(
batch_index,
'0'
),
COALESCE(
token_index,
'0'
),
COALESCE(
token_id,
'0'
)
) AS mint_action_id,
log_counter
FROM
@ -173,36 +189,51 @@ mint_tx AS (
FROM
mint_events
)
),
FINAL AS (
SELECT
mint_events.action_id,
mint_events.mint_action_id,
mint_events.tx_hash,
mint_events.block_id,
mint_events.block_timestamp,
mint_events.method_name,
mint_events.args_json AS args,
mint_events.deposit,
mint_tx.tx_signer AS tx_signer,
mint_tx.tx_receiver AS tx_receiver,
mint_tx.tx_status AS tx_status,
mint_events.receipt_object_id,
mint_events.receiver_id,
mint_events.signer_id,
mint_events.owner_id,
mint_events.token_id,
mint_events.memo,
mint_events.owner_per_tx,
mint_events.mint_per_tx,
mint_events.gas_burnt,
-- gas burnt during receipt processing
mint_tx.transaction_fee,
-- gas burnt during entire transaction processing
mint_events._LOAD_TIMESTAMP,
mint_events._PARTITION_BY_BLOCK_NUMBER,
mint_events.log_counter,
(
mint_events.deposit / mint_events.log_counter
) :: FLOAT AS implied_price,
mint_events._inserted_timestamp
FROM
mint_events
LEFT JOIN mint_tx
ON mint_events.tx_hash = mint_tx.tx_hash
)
SELECT
mint_events.action_id,
mint_events.mint_action_id,
mint_events.tx_hash,
mint_events.block_id,
mint_events.block_timestamp,
mint_events.method_name,
mint_events.args_json as args,
mint_events.deposit,
mint_tx.tx_signer AS tx_signer,
mint_tx.tx_receiver AS tx_receiver,
mint_tx.tx_status AS tx_status,
mint_events.receipt_object_id,
mint_events.receiver_id,
mint_events.signer_id,
mint_events.owner_id,
mint_events.token_id,
mint_events.memo,
mint_events.owner_per_tx,
mint_events.mint_per_tx,
mint_events.gas_burnt, -- gas burnt during receipt processing
mint_tx.transaction_fee, -- gas burnt during entire transaction processing
mint_events._LOAD_TIMESTAMP,
mint_events._PARTITION_BY_BLOCK_NUMBER,
mint_events.log_counter,
(mint_events.deposit / mint_events.log_counter) :: FLOAT as implied_price,
mint_events._inserted_timestamp
*,
{{ dbt_utils.generate_surrogate_key(
['mint_action_id']
) }} AS standard_nft_mint_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
mint_events
LEFT JOIN mint_tx
ON mint_events.tx_hash = mint_tx.tx_hash
FINAL

13
models/silver/curated/nft/silver__standard_nft_mint_s3.yml
View File

@ -111,3 +111,16 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: STANDARD_NFT_MINT_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

16
models/silver/curated/silver__dex_swaps_s3.sql
View File

@ -1,7 +1,8 @@
{{ config(
materialized = "incremental",
unique_key = "swap_id",
incremental_strategy = "delete+insert",
incremental_strategy = "merge",
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ["block_timestamp::DATE"],
tags = ['curated']
) }}
@ -23,8 +24,7 @@ WITH base_swap_calls AS (
method_name IN (
'swap',
'ft_transfer_call'
)
{% if var("MANUAL_FIX") %}
) {% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
AND {{ incremental_load_filter('_inserted_timestamp') }}
@ -59,7 +59,7 @@ agg_swaps AS (
action.index
) AS action_list,
ANY_VALUE(_load_timestamp) AS _load_timestamp,
ANY_VALUE(_inserted_timestamp) as _inserted_timestamp
ANY_VALUE(_inserted_timestamp) AS _inserted_timestamp
FROM
base_swaps,
LATERAL FLATTEN(
@ -252,6 +252,12 @@ FINAL AS (
AND log_data IS NOT NULL
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['swap_id']
) }} AS dex_swaps_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/silver__dex_swaps_s3.yml
View File

@ -65,3 +65,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: DEX_SWAPS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

9
models/silver/curated/silver__lockup_actions.sql
View File

@ -1,5 +1,6 @@
{{ config(
materialized = 'incremental',
merge_exclude_columns = ["inserted_timestamp"],
unique_key = 'tx_hash',
tags = ['curated'],
) }}
@ -206,6 +207,12 @@ FINAL AS (
tx_status != 'Fail'
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS lockup_actions_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/silver__lockup_actions.yml
View File

@ -56,3 +56,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: LOCKUP_ACTIONS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

11
models/silver/curated/silver__logs_s3.sql
View File

@ -1,8 +1,9 @@
{{ config(
materialized = "incremental",
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ["_inserted_timestamp::DATE","block_timestamp::DATE"],
unique_key = "action_id",
incremental_strategy = "delete+insert",
incremental_strategy = "merge",
tags = ['curated']
) }}
@ -59,6 +60,12 @@ SELECT
block_timestamp,
_load_timestamp,
_partition_by_block_number,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['action_id']
) }} AS logs_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/silver__logs_s3.yml
View File

@ -73,3 +73,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: LOGS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

11
models/silver/curated/silver__prices_oracle_s3.sql
View File

@ -1,11 +1,14 @@
{{ config(
materialized = 'incremental',
merge_exclude_columns = ["inserted_timestamp"],
unique_key = 'block_id',
incremental_strategy = 'delete+insert',
cluster_by = ['block_timestamp::DATE'],
tags = ['curated']
) }}
{# TODO NOTE 12/11/2023 - d+i on block_id to update timestamp when it comes in. Can be improved. Block id is not properly unique. #}
WITH token_labels AS (
SELECT
@ -85,7 +88,13 @@ FINAL AS (
LEFT JOIN token_labels l USING (token_contract)
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['block_id', 'token_contract']
) }} AS prices_oracle_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL
WHERE

12
models/silver/curated/silver__prices_oracle_s3.yml
View File

@ -71,3 +71,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: PRICES_ORACLE_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

14
models/silver/curated/silver__transfers_s3.sql
View File

@ -1,8 +1,9 @@
{{ config(
materialized = 'incremental',
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['block_timestamp::DATE'],
unique_key = 'action_id',
incremental_strategy = 'delete+insert',
incremental_strategy = 'merge',
tags = ['curated']
) }}
@ -18,8 +19,7 @@ WITH action_events AS(
FROM
{{ ref('silver__actions_events_s3') }}
WHERE
action_name = 'Transfer'
{% if var("MANUAL_FIX") %}
action_name = 'Transfer' {% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
AND {{ incremental_load_filter("_inserted_timestamp") }}
@ -110,6 +110,12 @@ FINAL AS (
actions
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['action_id']
) }} AS transfers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/silver__transfers_s3.yml
View File

@ -64,3 +64,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: TRANSFERS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

75
models/silver/curated/silver__usn_supply_s3.sql
View File

@ -1,75 +0,0 @@
{{ config(
materialized = 'incremental',
unique_key = 'tx_hash',
incremental_strategy = 'delete+insert',
cluster_by = ['block_timestamp::DATE'],
tags = ['curated'],
enabled = False
) }}
WITH txs AS (
SELECT
*
FROM
{{ ref('silver__streamline_receipts_final') }}
{% if var("MANUAL_FIX") %}
WHERE
{{ partition_load_manual('no_buffer') }}
{% else %}
WHERE
{{ incremental_load_filter('_load_timestamp') }}
{% endif %}
),
logs AS (
SELECT
block_timestamp,
block_id,
tx_hash,
REPLACE(
VALUE,
'EVENT_JSON:'
) AS json,
TRY_PARSE_JSON(json) :event :: STRING AS event,
TRY_PARSE_JSON(json) :standard AS STANDARD,
TRY_PARSE_JSON(json) :data AS data_log,
REGEXP_SUBSTR(
status_value,
'Success'
) AS reg_success,
_load_timestamp
FROM
txs,
TABLE(FLATTEN(input => logs))
WHERE
1 = 1
AND reg_success IS NOT NULL
AND event IS NOT NULL
AND receiver_id = 'usn'
),
FINAL AS (
SELECT
block_timestamp,
block_id,
tx_hash,
reg_success AS status,
VALUE :amount / pow(
10,
18
) AS amount,
event,
COALESCE(
VALUE :owner_id,
VALUE :old_owner_id
) :: STRING AS from_address,
VALUE :new_owner_id :: STRING AS to_address,
_load_timestamp
FROM
logs,
TABLE(FLATTEN(input => data_log))
)
SELECT
*
FROM
FINAL

39
models/silver/curated/silver__usn_supply_s3.yml
View File

@ -1,39 +0,0 @@
version: 2
models:
- name: silver__usn_supply_s3
description: |-
This table records all the usn transaction in the near db
columns:
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp')}}"
- name: BLOCK_ID
description: "{{ doc('block_id')}}"
- name: STATUS
description: "{{ doc('tx_status')}}"
- name: TX_HASH
description: "{{ doc('tx_hash')}}"
- name: FROM_ADDRESS
description: "{{ doc('from_address')}}"
- name: TO_ADDRESS
description: "{{ doc('to_address')}}"
- name: EVENT
description: "{{ doc('event')}}"
- name: AMOUNT
description: "{{ doc('amount')}}"
- name: _LOAD_TIMESTAMP
description: "{{ doc('_load_timestamp')}}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ

8
models/silver/curated/staking/silver__pool_balance_daily.sql
View File

@ -82,6 +82,12 @@ SELECT
'-',
date_day,
address
) AS _id
) AS _id,
{{ dbt_utils.generate_surrogate_key(
['date_day', 'address']
) }} AS pool_balance_daily_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
imputed_balance

12
models/silver/curated/staking/silver__pool_balance_daily.yml
View File

@ -26,3 +26,15 @@ models:
tests:
- not_null
- unique
- name: POOL_BALANCE_DAILY_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

9
models/silver/curated/staking/silver__pool_balances.sql
View File

@ -1,7 +1,6 @@
{{ config(
materialized = 'table',
unique_key = 'tx_hash',
incremental_strategy = 'delete+insert',
tags = ['curated'],
cluster_by = ['_partition_by_block_number', 'block_timestamp::date']
) }}
@ -51,6 +50,12 @@ FINAL AS (
LOG LIKE 'Contract total staked balance is%'
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS pool_balances_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/staking/silver__pool_balances.yml
View File

@ -55,3 +55,15 @@ models:
- name: _partition_by_block_number
description: "{{ doc('_partition_by_block_number') }}"
- name: POOL_BALANCES_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

9
models/silver/curated/staking/silver__pool_events.sql
View File

@ -1,7 +1,6 @@
{{ config(
materialized = 'table',
unique_key = 'tx_hash',
incremental_strategy = 'delete+insert',
tags = ['curated'],
cluster_by = ['_partition_by_block_number', 'block_timestamp::date']
) }}
@ -42,6 +41,12 @@ FINAL AS (
AND receiver_id ILIKE '%.pool%.near'
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS pool_events_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/staking/silver__pool_events.yml
View File

@ -74,3 +74,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: POOL_EVENTS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

12
models/silver/curated/staking/silver__staking_actions_s3.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
incremental = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
cluster_by = ['block_timestamp'],
unique_key = 'tx_hash',
tags = ['curated']
@ -21,8 +22,7 @@ WITH actions_events_function_call AS (
'stake',
'unstake',
'unstake_all'
)
{% if var("MANUAL_FIX") %}
) {% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
AND {{ incremental_load_filter('_inserted_timestamp') }}
@ -215,6 +215,12 @@ FINAL AS (
unstake_all_txs
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS staking_actions_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/staking/silver__staking_actions_s3.yml
View File

@ -37,3 +37,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: STAKING_ACTIONS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

11
models/silver/curated/staking/silver__staking_actions_v2.sql
View File

@ -1,7 +1,8 @@
{{ config(
materialized = 'table',
unique_key = 'tx_hash',
incremental_strategy = 'delete+insert',
incremental_strategy = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
tags = ['curated'],
cluster_by = ['_partition_by_block_number', 'block_timestamp::date']
) }}
@ -85,6 +86,12 @@ FINAL AS (
staking_actions
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS staking_actions_v2_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/staking/silver__staking_actions_v2.yml
View File

@ -97,3 +97,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: STAKING_ACTIONS_V2_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

9
models/silver/curated/staking/silver__staking_epochs.sql
View File

@ -1,7 +1,8 @@
{{ config(
materialized = 'table',
unique_key = '_epoch_id',
incremental_strategy = 'delete+insert',
incremental_strategy = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
tags = ['curated'],
cluster_by = ['block_id']
) }}
@ -57,6 +58,10 @@ FINAL AS (
pool_events
)
SELECT
*
*,
_epoch_id AS staking_epochs_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL

12
models/silver/curated/staking/silver__staking_epochs.yml
View File

@ -78,3 +78,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: STAKING_EPOCHS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"

14
models/silver/curated/staking/silver__staking_pools_s3.sql
View File

@ -3,6 +3,7 @@
cluster_by = ['block_timestamp'],
unique_key = 'tx_hash',
incremental_strategy = 'merge',
merge_exclude_columns = ["inserted_timestamp"],
tags = ['curated']
) }}
@ -51,8 +52,7 @@ function_calls AS (
'create_staking_pool',
'update_reward_fee_fraction',
'new'
)
{% if var("MANUAL_FIX") %}
) {% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
AND {{ incremental_load_filter('_inserted_timestamp') }}
@ -127,14 +127,20 @@ FINAL AS (
*
FROM
new_pools
UNION ALL
UNION ALL
SELECT
*
FROM
updated_pools
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS staking_pools_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL
WHERE

12
models/silver/curated/staking/silver__staking_pools_s3.yml
View File

@ -34,3 +34,15 @@ models:
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp')}}"
- name: STAKING_POOLS_ID
description: "{{doc('id')}}"
- name: INSERTED_TIMESTAMP
description: "{{doc('inserted_timestamp')}}"
- name: MODIFIED_TIMESTAMP
description: "{{doc('modified_timestamp')}}"
- name: _INVOCATION_ID
description: "{{doc('invocation_id')}}"