diff --git a/models/silver/silver__actions_events.sql b/models/silver/legacy/silver__actions_events.sql similarity index 100% rename from models/silver/silver__actions_events.sql rename to models/silver/legacy/silver__actions_events.sql diff --git a/models/silver/silver__actions_events.yml b/models/silver/legacy/silver__actions_events.yml similarity index 100% rename from models/silver/silver__actions_events.yml rename to models/silver/legacy/silver__actions_events.yml diff --git a/models/silver/silver__actions_events_addkey.sql b/models/silver/legacy/silver__actions_events_addkey.sql similarity index 100% rename from models/silver/silver__actions_events_addkey.sql rename to models/silver/legacy/silver__actions_events_addkey.sql diff --git a/models/silver/silver__actions_events_addkey.yml b/models/silver/legacy/silver__actions_events_addkey.yml similarity index 100% rename from models/silver/silver__actions_events_addkey.yml rename to models/silver/legacy/silver__actions_events_addkey.yml diff --git a/models/silver/silver__actions_events_function_call.sql b/models/silver/legacy/silver__actions_events_function_call.sql similarity index 100% rename from models/silver/silver__actions_events_function_call.sql rename to models/silver/legacy/silver__actions_events_function_call.sql diff --git a/models/silver/silver__actions_events_function_call.yml b/models/silver/legacy/silver__actions_events_function_call.yml similarity index 100% rename from models/silver/silver__actions_events_function_call.yml rename to models/silver/legacy/silver__actions_events_function_call.yml diff --git a/models/silver/silver__blocks.sql b/models/silver/legacy/silver__blocks.sql similarity index 100% rename from models/silver/silver__blocks.sql rename to models/silver/legacy/silver__blocks.sql diff --git a/models/silver/silver__blocks.yml b/models/silver/legacy/silver__blocks.yml similarity index 100% rename from models/silver/silver__blocks.yml rename to models/silver/legacy/silver__blocks.yml diff --git a/models/silver/silver__receipts.sql b/models/silver/legacy/silver__receipts.sql similarity index 100% rename from models/silver/silver__receipts.sql rename to models/silver/legacy/silver__receipts.sql diff --git a/models/silver/silver__receipts.yml b/models/silver/legacy/silver__receipts.yml similarity index 100% rename from models/silver/silver__receipts.yml rename to models/silver/legacy/silver__receipts.yml diff --git a/models/silver/silver__transactions.sql b/models/silver/legacy/silver__transactions.sql similarity index 100% rename from models/silver/silver__transactions.sql rename to models/silver/legacy/silver__transactions.sql diff --git a/models/silver/silver__transactions.yml b/models/silver/legacy/silver__transactions.yml similarity index 100% rename from models/silver/silver__transactions.yml rename to models/silver/legacy/silver__transactions.yml diff --git a/models/silver/silver__load_blocks.sql b/models/silver/streamline/silver__load_blocks.sql similarity index 100% rename from models/silver/silver__load_blocks.sql rename to models/silver/streamline/silver__load_blocks.sql diff --git a/models/silver/silver__load_shards.sql b/models/silver/streamline/silver__load_shards.sql similarity index 100% rename from models/silver/silver__load_shards.sql rename to models/silver/streamline/silver__load_shards.sql diff --git a/models/silver/silver__streamline_blocks.sql b/models/silver/streamline/silver__streamline_blocks.sql similarity index 100% rename from models/silver/silver__streamline_blocks.sql rename to models/silver/streamline/silver__streamline_blocks.sql diff --git a/models/silver/silver__streamline_blocks.yml b/models/silver/streamline/silver__streamline_blocks.yml similarity index 100% rename from models/silver/silver__streamline_blocks.yml rename to models/silver/streamline/silver__streamline_blocks.yml diff --git a/models/silver/streamline/silver__streamline_chunks.sql b/models/silver/streamline/silver__streamline_chunks.sql new file mode 100644 index 0000000..adeaeea --- /dev/null +++ b/models/silver/streamline/silver__streamline_chunks.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + unique_key = 'chunk_hash', + cluster_by = ['_load_timestamp::date','height_created','height_included'] +) }} + +WITH shards AS ( + + SELECT + * + FROM + {{ ref('silver__streamline_shards') }} + WHERE + chunk != 'null' + AND {{ incremental_load_filter('_load_timestamp') }} + -- sample for dev testing TODO remove before prod merge + AND block_id BETWEEN 52800000 AND 53000000 +), +FINAL AS ( + SELECT + block_id, + shard_id, + _load_timestamp, + chunk, + chunk :header :height_created :: NUMBER AS height_created, + chunk :header :height_included :: NUMBER AS height_included, + chunk :author :: STRING AS author, + chunk :header :chunk_hash :: STRING AS chunk_hash, + chunk :header :: OBJECT AS header, + chunk :receipts :: ARRAY AS receipts, + chunk :transactions :: ARRAY AS chunk_transactions + FROM + shards +) +SELECT + * +FROM + FINAL diff --git a/models/silver/streamline/silver__streamline_receipt_execution_outcome.sql b/models/silver/streamline/silver__streamline_receipt_execution_outcome.sql new file mode 100644 index 0000000..e973e06 --- /dev/null +++ b/models/silver/streamline/silver__streamline_receipt_execution_outcome.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + unique_key = 'receipt_execution_outcome_id', + cluster_by = ['_load_timestamp::date','block_id','chunk_hash'] +) }} + +WITH shards AS ( + + SELECT + * + FROM + {{ ref('silver__streamline_shards') }} + WHERE + ARRAY_SIZE(receipt_execution_outcomes) > 0 + AND {{ incremental_load_filter('_load_timestamp') }} + -- sample for dev testing TODO remove before prod merge + AND block_id BETWEEN 52800000 AND 53000000 +), +FINAL AS ( + SELECT + block_id, + shard_id, + INDEX AS receipt_outcome_execution_index, + CONCAT_WS('-',shard_id,INDEX) as receipt_execution_outcome_id, + _load_timestamp, + chunk :header :chunk_hash :: STRING AS chunk_hash, + VALUE :execution_outcome :: OBJECT AS execution_outcome, + VALUE :receipt :: OBJECT AS receipt + FROM + shards, + LATERAL FLATTEN( + input => receipt_execution_outcomes + ) +) +SELECT + * +FROM + FINAL diff --git a/models/silver/streamline/silver__streamline_receipts.sql b/models/silver/streamline/silver__streamline_receipts.sql new file mode 100644 index 0000000..3b9a25c --- /dev/null +++ b/models/silver/streamline/silver__streamline_receipts.sql @@ -0,0 +1,84 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + unique_key = '', + cluster_by = ['_load_timestamp::date'] +) }} + +WITH chunks AS ( + + SELECT + * + FROM + {{ ref('silver__streamline_chunks') }} + WHERE + ARRAY_SIZE(receipts) > 0 + AND {{ incremental_load_filter('_load_timestamp') }} +), +receipt_execution_outcomes AS ( + SELECT + * + FROM + {{ ref('silver__streamline_receipt_execution_outcome') }} + WHERE + {{ incremental_load_filter('_load_timestamp') }} +), +chunk_receipts AS ( + SELECT + block_id, + shard_id, + 'chunk' AS source_object, + INDEX AS object_receipt_index, + _load_timestamp, + chunk_hash, + VALUE AS receipt + FROM + chunks, + LATERAL FLATTEN( + input => receipts + ) + WHERE + ARRAY_SIZE(receipts) > 0 +), +reo_receipts AS ( + SELECT + block_id, + shard_id, + 'receipt_execution_outcomes' AS source_object, + receipt_outcome_execution_index AS object_receipt_index, + _load_timestamp, + chunk_hash, + receipt + FROM + receipt_execution_outcomes +), +receipts AS ( + SELECT + * + FROM + chunk_receipts + UNION + SELECT + * + FROM + reo_receipts +), +FINAL AS ( + SELECT + block_id, + shard_id, + source_object, + object_receipt_index AS receipt_index, + _load_timestamp, + chunk_hash, + receipt, + receipt :receipt_id :: STRING AS receipt_id, + receipt :receiver_id :: STRING AS receiver_id, + receipt :receipt :Action :signer_id :: STRING AS signer_id + FROM + receipts +) +SELECT + * +FROM + FINAL diff --git a/models/silver/silver__streamline_shards.sql b/models/silver/streamline/silver__streamline_shards.sql similarity index 100% rename from models/silver/silver__streamline_shards.sql rename to models/silver/streamline/silver__streamline_shards.sql diff --git a/models/silver/silver__streamline_shards.yml b/models/silver/streamline/silver__streamline_shards.yml similarity index 100% rename from models/silver/silver__streamline_shards.yml rename to models/silver/streamline/silver__streamline_shards.yml