From a34db4c2507e348652ecdf8f30390720885a797f Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:59:38 -0600 Subject: [PATCH 1/6] stake action transfer and state change silver --- models/silver/core/silver__state_changes.sql | 64 ++++++++++++++++ .../curated/staking/silver__pool_events.sql | 15 +++- .../native/silver__token_transfer_stake.sql | 73 +++++++++++++++++++ 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 models/silver/core/silver__state_changes.sql create mode 100644 models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql diff --git a/models/silver/core/silver__state_changes.sql b/models/silver/core/silver__state_changes.sql new file mode 100644 index 0000000..f8703e9 --- /dev/null +++ b/models/silver/core/silver__state_changes.sql @@ -0,0 +1,64 @@ +{{ config( + materialized = 'incremental', + incremental_predicates = ['DBT_INTERNAL_DEST._partition_by_block_number >= (select min(_partition_by_block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], + incremental_strategy = 'merge', + merge_exclude_columns = ['inserted_timestamp'], + unique_key = 'state_change_id', + cluster_by = ['modified_timestamp::date', '_partition_by_block_number'] +) }} + +WITH shards AS ( + + SELECT + block_id, + shard_id, + state_change, + _partition_by_block_number, + COALESCE( + modified_timestamp, + _inserted_timestamp + ) AS _modified_timestamp + FROM + {{ ref('silver__streamline_shards') }} + WHERE + ARRAY_SIZE(state_change) > 0 {% if var('MANUAL_FIX') %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + +{% if is_incremental() %} +AND _modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +{% endif %} +), +flatten_state_change AS ( + SELECT + block_id, + shard_id, + VALUE, + INDEX, + VALUE :cause :: variant AS state_change_cause, + VALUE :change :: variant AS state_changes, + VALUE :type :: STRING AS state_change_type, + _partition_by_block_number + FROM + shards, + LATERAL FLATTEN (state_changes) +) +SELECT + block_id, + shard_id, + state_change_cause, + state_changes, + state_change_type, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key(['shard_id', 'index::STRING']) }} AS state_change_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + flatten_state_change diff --git a/models/silver/curated/staking/silver__pool_events.sql b/models/silver/curated/staking/silver__pool_events.sql index cd2cc73..ed280a2 100644 --- a/models/silver/curated/staking/silver__pool_events.sql +++ b/models/silver/curated/staking/silver__pool_events.sql @@ -8,7 +8,19 @@ WITH receipts AS ( SELECT - * + tx_hash, + block_id, + block_timestamp, + receipt_object_id, + predecessor_id, + receiver_id, + signer_id, + status_value, + logs, + receipt_succeeded, + _partition_by_block_number, + _inserted_timestamp, + modified_timestamp AS _modified_timestamp FROM {{ ref('silver__streamline_receipts_final') }} WHERE @@ -25,6 +37,7 @@ FINAL AS ( block_id, block_timestamp, receipt_object_id, + predecessor_id, receiver_id, signer_id, status_value, diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql new file mode 100644 index 0000000..f3a2c88 --- /dev/null +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql @@ -0,0 +1,73 @@ +{{ config( + materialized = 'incremental', + merge_exclude_columns = ["inserted_timestamp"], + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + cluster_by = ['block_timestamp::DATE', 'modified_timestamp::DATE'], + unique_key = 'token_transfer_stake_id', + incremental_strategy = 'merge', + tags = ['curated','scheduled_non_core'] +) }} + +WITH action_events AS( + + SELECT + block_id, + block_timestamp, + action_id, + tx_hash, + action_data :public_key :: STRING AS public_key, + action_data :stake :: STRING AS amount_unadj, + predecessor_id, + receiver_id, + signer_id, + receipt_succeeded, + gas_price, + gas_burnt, + tokens_burnt, + _partition_by_block_number, + _inserted_timestamp, + modified_timestamp AS _modified_timestamp + FROM + {{ ref('silver__actions_events_s3') }} + WHERE + action_name = 'Stake' {% if var("MANUAL_FIX") %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} + +{% if is_incremental() %} +AND _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} +) +{% endif %} +{% endif %} +) +SELECT + action_id, + tx_hash, + block_id, + block_timestamp, + public_key, + amount_unadj, + amount_unadj :: DOUBLE / pow( + 10, + 24 + ) AS amount_adj, + predecessor_id, + receiver_id, + signer_id, + receipt_succeeded, + gas_price, + gas_burnt, + tokens_burnt, + _partition_by_block_number, + {{ dbt_utils.generate_surrogate_key( + ['action_id', 'predecessor_id', 'receiver_id', 'amount_unadj'] + ) }} AS token_transfer_stake_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + action_events From f48af7d3e89974986c673387acc168498d54629c Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:49:56 -0600 Subject: [PATCH 2/6] upd native transfer w/o join --- models/silver/core/silver__state_changes.sql | 31 ++-- .../silver/curated/silver__lockup_actions.sql | 4 +- .../native/silver__token_transfer_native.sql | 74 ++++++++++ ....yml => silver__token_transfer_native.yml} | 43 ++---- .../native/silver__token_transfer_stake.sql | 3 +- .../native/silver__transfers_s3.sql | 134 ------------------ .../silver__token_transfers_complete.sql | 6 +- 7 files changed, 113 insertions(+), 182 deletions(-) create mode 100644 models/silver/curated/token_transfers/native/silver__token_transfer_native.sql rename models/silver/curated/token_transfers/native/{silver__transfers_s3.yml => silver__token_transfer_native.yml} (61%) delete mode 100644 models/silver/curated/token_transfers/native/silver__transfers_s3.sql diff --git a/models/silver/core/silver__state_changes.sql b/models/silver/core/silver__state_changes.sql index f8703e9..ab8e982 100644 --- a/models/silver/core/silver__state_changes.sql +++ b/models/silver/core/silver__state_changes.sql @@ -6,13 +6,14 @@ unique_key = 'state_change_id', cluster_by = ['modified_timestamp::date', '_partition_by_block_number'] ) }} - +-- https://app.snowflake.com/zsniary/exa10207/#/compute/history/queries/01b73a95-0411-29e0-3d4f-83027386d7bb/detail +-- Took 3h16m on 2XL WITH shards AS ( SELECT block_id, shard_id, - state_change, + state_changes, _partition_by_block_number, COALESCE( modified_timestamp, @@ -21,18 +22,20 @@ WITH shards AS ( FROM {{ ref('silver__streamline_shards') }} WHERE - ARRAY_SIZE(state_change) > 0 {% if var('MANUAL_FIX') %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} + ARRAY_SIZE(state_changes) > 0 -{% if is_incremental() %} -AND _modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} -) -{% endif %} +{% if var('MANUAL_FIX') %} + AND {{ partition_load_manual('no_buffer') }} +{% else %} + + {% if is_incremental() %} + AND _modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) + {% endif %} {% endif %} ), flatten_state_change AS ( @@ -52,6 +55,8 @@ flatten_state_change AS ( SELECT block_id, shard_id, + value, + index, state_change_cause, state_changes, state_change_type, diff --git a/models/silver/curated/silver__lockup_actions.sql b/models/silver/curated/silver__lockup_actions.sql index 7eff746..ad28ea4 100644 --- a/models/silver/curated/silver__lockup_actions.sql +++ b/models/silver/curated/silver__lockup_actions.sql @@ -71,12 +71,12 @@ xfers AS ( action_id, block_timestamp, block_id, - deposit, + amount_unadj :: INT AS deposit, _partition_by_block_number, _inserted_timestamp, modified_timestamp AS _modified_timestamp FROM - {{ ref('silver__transfers_s3') }} + {{ ref('silver__token_transfer_native') }} {% if var("MANUAL_FIX") %} WHERE {{ partition_load_manual('no_buffer') }} diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql new file mode 100644 index 0000000..2ecbddf --- /dev/null +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql @@ -0,0 +1,74 @@ +{{ config( + materialized = 'incremental', + merge_exclude_columns = ["inserted_timestamp"], + incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], + cluster_by = ['block_timestamp::DATE'], + unique_key = 'token_transfer_native_id', + incremental_strategy = 'merge', + tags = ['curated','scheduled_non_core'] +) }} + +WITH action_events AS( + + SELECT + tx_hash, + block_id, + block_timestamp, + action_id, + action_data :deposit :: STRING AS amount_unadj, + predecessor_id, + receiver_id, + signer_id, + receipt_succeeded, + gas_price, + gas_burnt, + tokens_burnt, + _partition_by_block_number, + _inserted_timestamp, + modified_timestamp AS _modified_timestamp + FROM + {{ ref('silver__actions_events_s3') }} + WHERE + action_name = 'Transfer' + + {% if var("MANUAL_FIX") %} + AND {{ partition_load_manual('no_buffer') }} + {% else %} +{% if is_incremental() %} +AND _modified_timestamp >= ( + SELECT + MAX(_modified_timestamp) + FROM + {{ this }} +) +{% endif %} +{% endif %} +) + +SELECT + tx_hash, + block_id, + block_timestamp, + action_id, + amount_unadj, + amount_unadj :: DOUBLE / pow( + 10, + 24 + ) AS amount_adj, + predecessor_id, + receiver_id, + signer_id, + receipt_succeeded, + gas_price, + gas_burnt, + tokens_burnt, + _partition_by_block_number, + _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['action_id', 'predecessor_id', 'receiver_id', 'amount_unadj'] + ) }} AS token_transfer_native_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + action_events diff --git a/models/silver/curated/token_transfers/native/silver__transfers_s3.yml b/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml similarity index 61% rename from models/silver/curated/token_transfers/native/silver__transfers_s3.yml rename to models/silver/curated/token_transfers/native/silver__token_transfer_native.yml index ef3a15c..1459924 100644 --- a/models/silver/curated/token_transfers/native/silver__transfers_s3.yml +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: silver__transfers_s3 + - name: silver__token_transfer_native description: |- This table records all the Transfer actions of the Near blockchain. @@ -21,11 +21,11 @@ models: - not_null: where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - name: TX_SIGNER - description: "{{ doc('tx_signer')}}" + - name: AMOUNT_UNADJ + description: "{{ doc('amount_unadj')}}" - - name: TX_RECEIVER - description: "{{ doc('tx_receiver')}}" + - name: AMOUNT_ADJ + description: "{{ doc('amount_adj')}}" - name: PREDECESSOR_ID description: "{{ doc('predecessor_id')}}" @@ -36,32 +36,17 @@ models: - name: RECEIVER_ID description: "{{ doc('receiver_id')}}" - - name: DEPOSIT - description: "{{ doc('deposit')}}" - - - name: TRANSACTION_FEE - description: "{{ doc('transaction_fee')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: GAS_USED - description: "{{ doc('gas_used')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' - - - name: TX_SUCCEEDED - description: "{{ doc('tx_succeeded')}}" - - name: RECEIPT_SUCCEEDED description: "{{ doc('receipt_succeeded')}}" - - name: STATUS - description: "{{ doc('status')}}" - tests: - - not_null: - where: _inserted_timestamp <= current_timestamp - interval '1 hour' + - name: GAS_PRICE + description: "{{ doc('gas_price')}}" + + - name: GAS_BURNT + description: "{{ doc('gas_burnt')}}" + + - name: TOKENS_BURNT + description: "{{ doc('tokens_burnt')}}" - name: _PARTITION_BY_BLOCK_NUMBER description: "{{ doc('_partition_by_block_number')}}" @@ -72,7 +57,7 @@ models: - name: _MODIFIED_TIMESTAMP description: "{{ doc('_modified_timestamp')}}" - - name: TRANSFERS_ID + - name: TOKEN_TRANSFER_NATIVE_ID description: "{{doc('id')}}" tests: - not_null diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql index f3a2c88..186dd93 100644 --- a/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql @@ -30,7 +30,8 @@ WITH action_events AS( FROM {{ ref('silver__actions_events_s3') }} WHERE - action_name = 'Stake' {% if var("MANUAL_FIX") %} + action_name = 'Stake' + {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} {% else %} diff --git a/models/silver/curated/token_transfers/native/silver__transfers_s3.sql b/models/silver/curated/token_transfers/native/silver__transfers_s3.sql deleted file mode 100644 index 3f06796..0000000 --- a/models/silver/curated/token_transfers/native/silver__transfers_s3.sql +++ /dev/null @@ -1,134 +0,0 @@ -{{ config( - materialized = 'incremental', - merge_exclude_columns = ["inserted_timestamp"], - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - cluster_by = ['block_timestamp::DATE'], - unique_key = 'action_id', - incremental_strategy = 'merge', - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash,action_id,predecessor_id,receiver_id);", - tags = ['curated','scheduled_non_core'] -) }} -{# Note - multisource model #} -WITH action_events AS( - - SELECT - tx_hash, - block_id, - block_timestamp, - action_id, - action_data :deposit :: INT AS deposit, - predecessor_id, - receiver_id, - signer_id, - receipt_succeeded, - gas_price, - gas_burnt, - tokens_burnt, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__actions_events_s3') }} - WHERE - action_name = 'Transfer' - - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} -{% if is_incremental() %} -AND _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} -) -{% endif %} -{% endif %} -), -txs AS ( - SELECT - tx_hash, - tx :receipt :: ARRAY AS tx_receipt, - block_id, - block_timestamp, - tx_receiver, - tx_signer, - transaction_fee, - gas_used, - tx_succeeded, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - - {% if var("MANUAL_FIX") %} - WHERE {{ partition_load_manual('no_buffer') }} - {% else %} -{% if is_incremental() %} -WHERE _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} - ) -{% endif %} -{% endif %} -), -actions AS ( - SELECT - A.tx_hash, - A.action_id, - A.block_id, - A.block_timestamp, - t.tx_signer, - t.tx_receiver, - A.predecessor_id, - A.receiver_id, - A.signer_id, - A.deposit, - t.transaction_fee, - A.gas_burnt AS gas_used, - A.receipt_succeeded, - t.tx_succeeded, - A._partition_by_block_number, - A._inserted_timestamp, - A._modified_timestamp - FROM - action_events A - INNER JOIN txs t - ON A.tx_hash = t.tx_hash -), -FINAL AS ( - SELECT - block_id, - block_timestamp, - action_id, - deposit, - tx_hash, - tx_signer, - tx_receiver, - predecessor_id, - signer_id, - receiver_id, - transaction_fee, - gas_used, - tx_succeeded, - receipt_succeeded, - ARRAY_MIN([tx_succeeded, receipt_succeeded]) :: BOOLEAN AS status, - _partition_by_block_number, - _inserted_timestamp, - _modified_timestamp - FROM - actions -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['action_id'] - ) }} AS transfers_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL diff --git a/models/silver/curated/token_transfers/silver__token_transfers_complete.sql b/models/silver/curated/token_transfers/silver__token_transfers_complete.sql index f840be2..bd4e4e3 100644 --- a/models/silver/curated/token_transfers/silver__token_transfers_complete.sql +++ b/models/silver/curated/token_transfers/silver__token_transfers_complete.sql @@ -21,15 +21,15 @@ WITH native_transfers AS ( predecessor_id AS from_address, receiver_id AS to_address, NULL AS memo, - IFF(REGEXP_LIKE(deposit, '^[0-9]+$'), deposit, NULL) AS amount_unadj, + amount_unadj, 'native' AS transfer_type, _inserted_timestamp, modified_timestamp AS _modified_timestamp, _partition_by_block_number FROM - {{ ref('silver__transfers_s3') }} + {{ ref('silver__token_transfer_native') }} WHERE - status = TRUE AND deposit != 0 + receipt_succeeded {% if var("MANUAL_FIX") %} AND {{ partition_load_manual('no_buffer') }} From a4d173b9d04af9455049be5ed2bb81b9c871e2ff Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:51:38 -0600 Subject: [PATCH 3/6] rm stake and state from this branch --- models/silver/core/silver__state_changes.sql | 69 ----------------- .../native/silver__token_transfer_stake.sql | 74 ------------------- 2 files changed, 143 deletions(-) delete mode 100644 models/silver/core/silver__state_changes.sql delete mode 100644 models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql diff --git a/models/silver/core/silver__state_changes.sql b/models/silver/core/silver__state_changes.sql deleted file mode 100644 index ab8e982..0000000 --- a/models/silver/core/silver__state_changes.sql +++ /dev/null @@ -1,69 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_predicates = ['DBT_INTERNAL_DEST._partition_by_block_number >= (select min(_partition_by_block_number) from ' ~ generate_tmp_view_name(this) ~ ')'], - incremental_strategy = 'merge', - merge_exclude_columns = ['inserted_timestamp'], - unique_key = 'state_change_id', - cluster_by = ['modified_timestamp::date', '_partition_by_block_number'] -) }} --- https://app.snowflake.com/zsniary/exa10207/#/compute/history/queries/01b73a95-0411-29e0-3d4f-83027386d7bb/detail --- Took 3h16m on 2XL -WITH shards AS ( - - SELECT - block_id, - shard_id, - state_changes, - _partition_by_block_number, - COALESCE( - modified_timestamp, - _inserted_timestamp - ) AS _modified_timestamp - FROM - {{ ref('silver__streamline_shards') }} - WHERE - ARRAY_SIZE(state_changes) > 0 - -{% if var('MANUAL_FIX') %} - AND {{ partition_load_manual('no_buffer') }} -{% else %} - - {% if is_incremental() %} - AND _modified_timestamp >= ( - SELECT - MAX(modified_timestamp) - FROM - {{ this }} - ) - {% endif %} -{% endif %} -), -flatten_state_change AS ( - SELECT - block_id, - shard_id, - VALUE, - INDEX, - VALUE :cause :: variant AS state_change_cause, - VALUE :change :: variant AS state_changes, - VALUE :type :: STRING AS state_change_type, - _partition_by_block_number - FROM - shards, - LATERAL FLATTEN (state_changes) -) -SELECT - block_id, - shard_id, - value, - index, - state_change_cause, - state_changes, - state_change_type, - _partition_by_block_number, - {{ dbt_utils.generate_surrogate_key(['shard_id', 'index::STRING']) }} AS state_change_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - flatten_state_change diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql deleted file mode 100644 index 186dd93..0000000 --- a/models/silver/curated/token_transfers/native/silver__token_transfer_stake.sql +++ /dev/null @@ -1,74 +0,0 @@ -{{ config( - materialized = 'incremental', - merge_exclude_columns = ["inserted_timestamp"], - incremental_predicates = ["COALESCE(DBT_INTERNAL_DEST.block_timestamp::DATE,'2099-12-31') >= (select min(block_timestamp::DATE) from " ~ generate_tmp_view_name(this) ~ ")"], - cluster_by = ['block_timestamp::DATE', 'modified_timestamp::DATE'], - unique_key = 'token_transfer_stake_id', - incremental_strategy = 'merge', - tags = ['curated','scheduled_non_core'] -) }} - -WITH action_events AS( - - SELECT - block_id, - block_timestamp, - action_id, - tx_hash, - action_data :public_key :: STRING AS public_key, - action_data :stake :: STRING AS amount_unadj, - predecessor_id, - receiver_id, - signer_id, - receipt_succeeded, - gas_price, - gas_burnt, - tokens_burnt, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp - FROM - {{ ref('silver__actions_events_s3') }} - WHERE - action_name = 'Stake' - {% if var("MANUAL_FIX") %} - AND {{ partition_load_manual('no_buffer') }} - {% else %} - -{% if is_incremental() %} -AND _modified_timestamp >= ( - SELECT - MAX(_modified_timestamp) - FROM - {{ this }} -) -{% endif %} -{% endif %} -) -SELECT - action_id, - tx_hash, - block_id, - block_timestamp, - public_key, - amount_unadj, - amount_unadj :: DOUBLE / pow( - 10, - 24 - ) AS amount_adj, - predecessor_id, - receiver_id, - signer_id, - receipt_succeeded, - gas_price, - gas_burnt, - tokens_burnt, - _partition_by_block_number, - {{ dbt_utils.generate_surrogate_key( - ['action_id', 'predecessor_id', 'receiver_id', 'amount_unadj'] - ) }} AS token_transfer_stake_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - action_events From c8be8853c2bb4ffc6d8959fcfaf7c53ae95afde9 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:14:23 -0400 Subject: [PATCH 4/6] rm _ from incr block and add tests on all --- models/descriptions/tokens_burnt.md | 5 ++++ .../native/silver__token_transfer_native.sql | 4 +-- .../native/silver__token_transfer_native.yml | 26 ++++++++++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 models/descriptions/tokens_burnt.md diff --git a/models/descriptions/tokens_burnt.md b/models/descriptions/tokens_burnt.md new file mode 100644 index 0000000..8d72763 --- /dev/null +++ b/models/descriptions/tokens_burnt.md @@ -0,0 +1,5 @@ +{% docs tokens_burnt %} + +The amount of tokens burnt in a transaction or receipt execution. + +{% enddocs %} diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql index 2ecbddf..4311471 100644 --- a/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql @@ -37,7 +37,7 @@ WITH action_events AS( {% if is_incremental() %} AND _modified_timestamp >= ( SELECT - MAX(_modified_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -65,7 +65,7 @@ SELECT _partition_by_block_number, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['action_id', 'predecessor_id', 'receiver_id', 'amount_unadj'] + ['action_id', 'predecessor_id', 'receiver_id'] ) }} AS token_transfer_native_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml b/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml index 1459924..97d09f1 100644 --- a/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_native.yml @@ -8,12 +8,18 @@ models: columns: - name: TX_HASH description: "{{ doc('tx_hash')}}" + tests: + - not_null - name: ACTION_ID description: "{{ doc('action_id')}}" + tests: + - not_null - name: BLOCK_ID description: "{{ doc('block_id')}}" + tests: + - not_null - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp')}}" @@ -23,30 +29,48 @@ models: - name: AMOUNT_UNADJ description: "{{ doc('amount_unadj')}}" + tests: + - not_null - name: AMOUNT_ADJ description: "{{ doc('amount_adj')}}" + tests: + - not_null - name: PREDECESSOR_ID description: "{{ doc('predecessor_id')}}" + tests: + - not_null - name: SIGNER_ID description: "{{ doc('signer_id')}}" - + tests: + - not_null + - name: RECEIVER_ID description: "{{ doc('receiver_id')}}" + tests: + - not_null - name: RECEIPT_SUCCEEDED description: "{{ doc('receipt_succeeded')}}" + tests: + - not_null - name: GAS_PRICE description: "{{ doc('gas_price')}}" + tests: + - not_null - name: GAS_BURNT description: "{{ doc('gas_burnt')}}" + tests: + - not_null - name: TOKENS_BURNT description: "{{ doc('tokens_burnt')}}" + tests: + - not_null - name: _PARTITION_BY_BLOCK_NUMBER description: "{{ doc('_partition_by_block_number')}}" From 56bc6565f986bcff117ea6d0a618819ddc157db5 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:25:09 -0400 Subject: [PATCH 5/6] addback amount_unadj to pk --- .../token_transfers/native/silver__token_transfer_native.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql index 4311471..6a4e922 100644 --- a/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql +++ b/models/silver/curated/token_transfers/native/silver__token_transfer_native.sql @@ -65,7 +65,7 @@ SELECT _partition_by_block_number, _inserted_timestamp, {{ dbt_utils.generate_surrogate_key( - ['action_id', 'predecessor_id', 'receiver_id'] + ['action_id', 'predecessor_id', 'receiver_id', 'amount_unadj'] ) }} AS token_transfer_native_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, From e68c18c039fd5da3a472c26790f12bd68b7b64bc Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Wed, 25 Sep 2024 16:42:31 -0400 Subject: [PATCH 6/6] rm change to pool events --- .../curated/staking/silver__pool_events.sql | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/models/silver/curated/staking/silver__pool_events.sql b/models/silver/curated/staking/silver__pool_events.sql index ed280a2..cd2cc73 100644 --- a/models/silver/curated/staking/silver__pool_events.sql +++ b/models/silver/curated/staking/silver__pool_events.sql @@ -8,19 +8,7 @@ WITH receipts AS ( SELECT - tx_hash, - block_id, - block_timestamp, - receipt_object_id, - predecessor_id, - receiver_id, - signer_id, - status_value, - logs, - receipt_succeeded, - _partition_by_block_number, - _inserted_timestamp, - modified_timestamp AS _modified_timestamp + * FROM {{ ref('silver__streamline_receipts_final') }} WHERE @@ -37,7 +25,6 @@ FINAL AS ( block_id, block_timestamp, receipt_object_id, - predecessor_id, receiver_id, signer_id, status_value,