diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index e4cda7e..8d8f84a 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -20,10 +20,17 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables (`aptos`.`CORE`.``) **Dimension Tables:** +- [dim_labels](#!/model/model.aptos.core__dim_labels) +- [dim_tokens](#!/model/model.aptos.core__dim_tokens) **Fact Tables:** - +- [fact_blocks](#!/model/model.aptos.core__fact_blocks) +- [fact_transactions](#!/model/model.aptos.core__fact_transactions) +- [fact_transactions_block_metadata](#!/model/model.aptos.core__fact_transactions_block_metadata) +- [fact_transactions_state_checkpoint](#!/model/model.aptos.core__fact_transactions_state_checkpoint) +- [fact_events](#!/model/model.aptos.core__fact_events) +- [fact_changes](#!/model/model.aptos.core__fact_changes) **Convenience Tables:** diff --git a/models/descriptions/accumulator_root_hash.md b/models/descriptions/accumulator_root_hash.md new file mode 100644 index 0000000..912fbd5 --- /dev/null +++ b/models/descriptions/accumulator_root_hash.md @@ -0,0 +1,5 @@ +{% docs accumulator_root_hash %} + +The root hash of a Merkle accumulator. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_hash.md b/models/descriptions/block_hash.md new file mode 100644 index 0000000..1658154 --- /dev/null +++ b/models/descriptions/block_hash.md @@ -0,0 +1,5 @@ +{% docs block_hash %} + +The hash of the block header for a given block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/block_number.md b/models/descriptions/block_number.md new file mode 100644 index 0000000..ba7fa4f --- /dev/null +++ b/models/descriptions/block_number.md @@ -0,0 +1,5 @@ +{% docs block_number %} + +Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block. + +{% enddocs %} diff --git a/models/descriptions/block_timestamp.md b/models/descriptions/block_timestamp.md new file mode 100644 index 0000000..0c1a1b8 --- /dev/null +++ b/models/descriptions/block_timestamp.md @@ -0,0 +1,5 @@ +{% docs block_timestamp %} + +The date and time at which the block was produced. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/changes.md b/models/descriptions/changes.md new file mode 100644 index 0000000..8b718ad --- /dev/null +++ b/models/descriptions/changes.md @@ -0,0 +1,5 @@ +{% docs changes %} + +The changes that the transaction executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/epoch.md b/models/descriptions/epoch.md new file mode 100644 index 0000000..1befe29 --- /dev/null +++ b/models/descriptions/epoch.md @@ -0,0 +1,5 @@ +{% docs epoch %} + +An epoch in the Aptos blockchain is defined as a duration of time, in seconds, during which a number of blocks are voted on by the validators, the validator set is updated, and the rewards are distributed to the validators. The Aptos mainnet epoch is set as 7200 seconds (two hours). + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/event_root_hash.md b/models/descriptions/event_root_hash.md new file mode 100644 index 0000000..496edae --- /dev/null +++ b/models/descriptions/event_root_hash.md @@ -0,0 +1,5 @@ +{% docs event_root_hash %} + +The root hash for the event. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/events.md b/models/descriptions/events.md new file mode 100644 index 0000000..c96082b --- /dev/null +++ b/models/descriptions/events.md @@ -0,0 +1,5 @@ +{% docs events %} + +The events that the transaction executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/expiration_timestamp_secs.md b/models/descriptions/expiration_timestamp_secs.md new file mode 100644 index 0000000..0386f71 --- /dev/null +++ b/models/descriptions/expiration_timestamp_secs.md @@ -0,0 +1,5 @@ +{% docs expiration_timestamp_secs %} + +The time at which the transaction ceases to valid. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/first_version.md b/models/descriptions/first_version.md new file mode 100644 index 0000000..1a5825b --- /dev/null +++ b/models/descriptions/first_version.md @@ -0,0 +1,5 @@ +{% docs first_version %} + +The version number of the first transaction in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_unit_price.md b/models/descriptions/gas_unit_price.md new file mode 100644 index 0000000..0a53fd5 --- /dev/null +++ b/models/descriptions/gas_unit_price.md @@ -0,0 +1,5 @@ +{% docs gas_unit_price %} + +The cost per unit of gas, determining the transaction fee paid by the sender for each unit of computational resource consumed + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/gas_used.md b/models/descriptions/gas_used.md new file mode 100644 index 0000000..785880e --- /dev/null +++ b/models/descriptions/gas_used.md @@ -0,0 +1,5 @@ +{% docs gas_used %} + +The amount of gas used for the transaction + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/inserted_timestamp.md b/models/descriptions/inserted_timestamp.md new file mode 100644 index 0000000..14703c3 --- /dev/null +++ b/models/descriptions/inserted_timestamp.md @@ -0,0 +1,5 @@ +{% docs inserted_timestamp %} + +The utc timestamp at which the row was inserted into the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/last_version.md b/models/descriptions/last_version.md new file mode 100644 index 0000000..a56d7b6 --- /dev/null +++ b/models/descriptions/last_version.md @@ -0,0 +1,5 @@ +{% docs last_version %} + +The version number of the last transaction in the block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/max_gas_amount.md b/models/descriptions/max_gas_amount.md new file mode 100644 index 0000000..45da165 --- /dev/null +++ b/models/descriptions/max_gas_amount.md @@ -0,0 +1,5 @@ +{% docs max_gas_amount %} + +The maximum amount of gas allocated for the execution of a transaction + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/modified_timestamp.md b/models/descriptions/modified_timestamp.md new file mode 100644 index 0000000..c4d18ee --- /dev/null +++ b/models/descriptions/modified_timestamp.md @@ -0,0 +1,5 @@ +{% docs modified_timestamp %} + +The utc timestamp at which the row was last modified. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/payload.md b/models/descriptions/payload.md new file mode 100644 index 0000000..bc8f12b --- /dev/null +++ b/models/descriptions/payload.md @@ -0,0 +1,5 @@ +{% docs payload %} + +The data that is being carried by a transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/pk.md b/models/descriptions/pk.md new file mode 100644 index 0000000..8dfdfb5 --- /dev/null +++ b/models/descriptions/pk.md @@ -0,0 +1,5 @@ +{% docs pk %} + +The unique identifier for each row in the table. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/proposer.md b/models/descriptions/proposer.md new file mode 100644 index 0000000..d66b281 --- /dev/null +++ b/models/descriptions/proposer.md @@ -0,0 +1,5 @@ +{% docs proposer %} + +The block proposer. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/round.md b/models/descriptions/round.md new file mode 100644 index 0000000..f24284a --- /dev/null +++ b/models/descriptions/round.md @@ -0,0 +1,5 @@ +{% docs round %} + +A round number is a shared counter used to select leaders during an epoch of the consensus protocol. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/sender.md b/models/descriptions/sender.md new file mode 100644 index 0000000..b86343b --- /dev/null +++ b/models/descriptions/sender.md @@ -0,0 +1,5 @@ +{% docs sender %} + +Sender is the address of the originator account for a transaction. A transaction must be signed by the originator. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/signature.md b/models/descriptions/signature.md new file mode 100644 index 0000000..121914a --- /dev/null +++ b/models/descriptions/signature.md @@ -0,0 +1,5 @@ +{% docs signature %} + +A signature is the result of hashing the signing message with the client's private key. By default Aptos uses the Ed25519 scheme to generate the signature of the raw transaction. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/success.md b/models/descriptions/success.md new file mode 100644 index 0000000..6b20598 --- /dev/null +++ b/models/descriptions/success.md @@ -0,0 +1,5 @@ +{% docs success %} + +The boolean value indicating whether the transaction was successful or not. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tables/core__fact_blocks.md b/models/descriptions/tables/core__fact_blocks.md new file mode 100644 index 0000000..b98a007 --- /dev/null +++ b/models/descriptions/tables/core__fact_blocks.md @@ -0,0 +1,8 @@ +{% docs core__fact_blocks %} + +This table contains "block" level data for the Aptos blockchain. This table can be used to analyze trends at a block level, for example total transactions over time. +"The Aptos blockchain doesn't have an explicit notion of a block — it only uses blocks for batching and executing transactions. +A transaction at height 0 is the first transaction (genesis transaction), and a transaction at height 100 is the 101st transaction in the transaction store." +from [aptos.dev docs] https://aptos.dev/reference/glossary/#version + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_transactions.md b/models/descriptions/tables/core__fact_transactions.md new file mode 100644 index 0000000..c691119 --- /dev/null +++ b/models/descriptions/tables/core__fact_transactions.md @@ -0,0 +1,6 @@ +{% docs core__fact_transactions %} + +This table contains transaction level data for the Aptos blockchain. Each transaction will have a unique transaction hash and version. +For more information see [aptos.dev docs] Each transaction will have a unique transaction hash + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_transactions_block_metadata.md b/models/descriptions/tables/core__fact_transactions_block_metadata.md new file mode 100644 index 0000000..819eeac --- /dev/null +++ b/models/descriptions/tables/core__fact_transactions_block_metadata.md @@ -0,0 +1,5 @@ +{% docs core__fact_transactions_block_metadata %} + +These transactions are inserted at the beginning of the block. A BlockMetadata transaction can also mark the end of an epoch and trigger reward distribution to validators. + +{% enddocs %} diff --git a/models/descriptions/tables/core__fact_transactions_state_checkpoint.md b/models/descriptions/tables/core__fact_transactions_state_checkpoint.md new file mode 100644 index 0000000..3b4b995 --- /dev/null +++ b/models/descriptions/tables/core__fact_transactions_state_checkpoint.md @@ -0,0 +1,5 @@ +{% docs core__fact_transactions_state_checkpoint %} + +These transactions are appended at the end of the block and is used as a checkpoint milestone. + +{% enddocs %} diff --git a/models/descriptions/tx_count.md b/models/descriptions/tx_count.md new file mode 100644 index 0000000..74978b9 --- /dev/null +++ b/models/descriptions/tx_count.md @@ -0,0 +1,5 @@ +{% docs tx_count %} + +The count of transactions in this block. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_hash.md b/models/descriptions/tx_hash.md new file mode 100644 index 0000000..a848c23 --- /dev/null +++ b/models/descriptions/tx_hash.md @@ -0,0 +1,5 @@ +{% docs tx_hash %} + +Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/tx_type.md b/models/descriptions/tx_type.md new file mode 100644 index 0000000..528e1bd --- /dev/null +++ b/models/descriptions/tx_type.md @@ -0,0 +1,5 @@ +{% docs tx_type %} + +The type of the transaction. Values will be one of "block_metadata_transaction","state_checkpoint_transaction","user_transaction". + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/version.md b/models/descriptions/version.md new file mode 100644 index 0000000..ac9836e --- /dev/null +++ b/models/descriptions/version.md @@ -0,0 +1,5 @@ +{% docs version %} + +The version also know as the height of the transactions that have been executed on the Aptos blockchain. The first transaction in the blockchain has a version of 0. The version number is incremented by 1 for each transaction that is executed on the blockchain. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/vm_status.md b/models/descriptions/vm_status.md new file mode 100644 index 0000000..1942480 --- /dev/null +++ b/models/descriptions/vm_status.md @@ -0,0 +1,5 @@ +{% docs vm_status %} + +For failed transactions, this fields provides context to why the transaction failed. For successful transactions, this field will be set to `Executed successfully`. + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__fact_blocks.yml b/models/gold/core/core__fact_blocks.yml index bfa080f..0cc900d 100644 --- a/models/gold/core/core__fact_blocks.yml +++ b/models/gold/core/core__fact_blocks.yml @@ -1,6 +1,26 @@ version: 2 models: - name: core__fact_blocks - tests: - - dbt_constraints.primary_key: - column_name: FACT_BLOCKS_ID + description: '{{ doc("core__fact_blocks") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: BLOCK_HASH + description: '{{ doc("block_hash") }}' + - name: FIRST_VERSION + description: '{{ doc("first_version") }}' + - name: LAST_VERSION + description: '{{ doc("last_version") }}' + - name: TX_COUNT + description: '{{ doc("tx_count") }}' + - name: FACT_BLOCKS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' + + diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index 74627c9..0dd933e 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -8,7 +8,7 @@ SELECT version, tx_hash, success, - tx_TYPE, + tx_type, sender, signature, payload, diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml new file mode 100644 index 0000000..5f5dbf7 --- /dev/null +++ b/models/gold/core/core__fact_transactions.yml @@ -0,0 +1,49 @@ +version: 2 +models: + - name: core__fact_transactions + description: '{{ doc("core__fact_transactions") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: SENDER + description: '{{ doc("sender") }}' + - name: SIGNATURE + description: '{{ doc("signature") }}' + - name: PAYLOAD + description: '{{ doc("payload") }}' + - name: CHANGES + description: '{{ doc("changes") }}' + - name: EVENTS + description: '{{ doc("events") }}' + - name: GAS_UNIT_PRICE + description: '{{ doc("gas_unit_price") }}' + - name: GAS_USED + description: '{{ doc("gas_used") }}' + - name: MAX_GAS_AMOUNT + description: '{{ doc("max_gas_amount") }}' + - name: EXPIRATION_TIMESTAMP_SECS + description: '{{ doc("expiration_timestamp_secs") }}' + - name: VM_STATUS + description: '{{ doc("vm_status") }}' + - name: STATE_CHANGE_HASH + - name: ACCUMULATOR_ROOT_HASH + description: '{{ doc("accumulator_root_hash") }}' + - name: EVENT_ROOT_HASH + description: '{{ doc("event_root_hash") }}' + - name: FACT_TRANSACTIONS_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_transactions_block_metadata.yml b/models/gold/core/core__fact_transactions_block_metadata.yml new file mode 100644 index 0000000..94180bc --- /dev/null +++ b/models/gold/core/core__fact_transactions_block_metadata.yml @@ -0,0 +1,44 @@ +version: 2 +models: + - name: core__fact_transactions_block_metadata + description: '{{ doc("core__fact_transactions_block_metadata") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: EPOCH + description: '{{ doc("epoch") }}' + - name: EVENTS + description: '{{ doc("events") }}' + - name: CHANGES + description: '{{ doc("changes") }}' + - name: FAILED_PROPOSER_INDICES + - name: ID + - name: PREVIOUS_BLOCK_VOTES_BITVEC + - name: PROPOSER + description: '{{ doc("proposer") }}' + - name: ROUND + description: '{{ doc("round") }}' + - name: VM_STATUS + description: '{{ doc("vm_status") }}' + - name: STATE_CHANGE_HASH + - name: ACCUMULATOR_ROOT_HASH + description: '{{ doc("accumulator_root_hash") }}' + - name: EVENT_ROOT_HASH + description: '{{ doc("event_root_hash") }}' + - name: FACT_TRANSACTIONS_BLOCK_METADATA_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/core/core__fact_transactions_state_checkpoint.yml b/models/gold/core/core__fact_transactions_state_checkpoint.yml new file mode 100644 index 0000000..a087b39 --- /dev/null +++ b/models/gold/core/core__fact_transactions_state_checkpoint.yml @@ -0,0 +1,31 @@ +version: 2 +models: + - name: core__fact_transactions_state_checkpoint + description: '{{ doc("core__fact_transactions_state_checkpoint") }}' + + columns: + - name: BLOCK_NUMBER + description: '{{ doc("block_number") }}' + - name: BLOCK_TIMESTAMP + description: '{{ doc("block_timestamp") }}' + - name: TX_HASH + description: '{{ doc("tx_hash") }}' + - name: VERSION + description: '{{ doc("version") }}' + - name: SUCCESS + description: '{{ doc("success") }}' + - name: TX_TYPE + description: '{{ doc("tx_type") }}' + - name: VM_STATUS + description: '{{ doc("vm_status") }}' + - name: STATE_CHECKPOINT_HASH + - name: ACCUMULATOR_ROOT_HASH + description: '{{ doc("accumulator_root_hash") }}' + - name: EVENT_ROOT_HASH + description: '{{ doc("event_root_hash") }}' + - name: FACT_TRANSACTIONS_STATE_CHECKPOINT_ID + description: '{{ doc("pk") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index c0e6243..6f40d8c 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -3,15 +3,18 @@ unique_key = "block_number", incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], + tags = ['core'] ) }} SELECT block_height AS block_number, DATA :data :block_hash :: STRING AS block_hash, -- DATA :data :block_height :: INT AS block_height, - DATA :data :block_timestamp :: STRING AS block_timestamp_num, - TO_TIMESTAMP(block_timestamp_num) AS block_timestamp, + DATA :data :block_timestamp :: bigint AS block_timestamp_num, + TO_TIMESTAMP( + block_timestamp_num :: STRING + ) AS block_timestamp, DATA :data :first_version :: bigint AS first_version, DATA :data :last_version :: bigint AS last_version, ARRAY_SIZE( diff --git a/models/silver/core/silver__blocks.yml b/models/silver/core/silver__blocks.yml new file mode 100644 index 0000000..79cb7c6 --- /dev/null +++ b/models/silver/core/silver__blocks.yml @@ -0,0 +1,50 @@ +version: 2 +models: + - name: silver__blocks + + tests: + - dbt_constraints.primary_key: + column_name: BLOCKS_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP_NUM + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - name: BLOCK_HASH + tests: + - not_null + - name: FIRST_VERSION + tests: + - not_null + - name: LAST_VERSION + tests: + - not_null + - name: TX_COUNT_FROM_TRANSACTIONS_ARRAY + tests: + - not_null + - name: TX_COUNT_FROM_VERSIONS + tests: + - not_null + - name: BLOCKS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_silver__blocks_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_silver__blocks_INVOCATION_ID + test_name: not_null diff --git a/models/silver/core/silver__changes.sql b/models/silver/core/silver__changes.sql index 673c377..dd663df 100644 --- a/models/silver/core/silver__changes.sql +++ b/models/silver/core/silver__changes.sql @@ -4,7 +4,8 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, change_type,inner_change_type,change_address,change_module,change_resource);" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, change_type,inner_change_type,change_address,change_module,change_resource);", + tags = ['core'] ) }} SELECT diff --git a/models/silver/core/silver__changes.yml b/models/silver/core/silver__changes.yml new file mode 100644 index 0000000..968349a --- /dev/null +++ b/models/silver/core/silver__changes.yml @@ -0,0 +1,62 @@ +version: 2 +models: + - name: silver__changes + + tests: + - dbt_constraints.primary_key: + column_name: CHANGES_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - name: VERSION + tests: + - not_null + - name: SUCCESS + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: CHANGE_INDEX + tests: + - not_null + - name: CHANGE_DATA + - name: CHANGE_TYPE + tests: + - not_null + - name: ADDRESS + - name: HANDLE + - name: INNER_CHANGE_TYPE + - name: CHANGE_ADDRESS + - name: CHANGE_MODULE + - name: CHANGE_RESOURCE + - name: KEY + - name: VALUE + - name: STATE_KEY_HASH + tests: + - not_null + - name: CHANGES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_silver__changes_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_silver__changes_INVOCATION_ID + test_name: not_null diff --git a/models/silver/core/silver__events.sql b/models/silver/core/silver__events.sql index 4f55b61..a059056 100644 --- a/models/silver/core/silver__events.sql +++ b/models/silver/core/silver__events.sql @@ -4,7 +4,8 @@ incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE'], - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, event_type,event_address,event_module,event_resource);" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_hash, event_type,event_address,event_module,event_resource);", + tags = ['core'] ) }} SELECT @@ -34,7 +35,7 @@ SELECT b.value :data AS event_data, -- b.value :guid :: STRING AS event_guid, -- extract into account_address + creation_number b.value :guid :account_address :: STRING AS account_address, - b.value :guid :creation_number :: STRING AS creation_number, + b.value :guid :creation_number :: bigint AS creation_number, b.value :sequence_number :: bigint AS sequence_number, {{ dbt_utils.generate_surrogate_key( ['tx_hash','event_index'] diff --git a/models/silver/core/silver__events.yml b/models/silver/core/silver__events.yml new file mode 100644 index 0000000..9e51fa2 --- /dev/null +++ b/models/silver/core/silver__events.yml @@ -0,0 +1,69 @@ +version: 2 +models: + - name: silver__events + + tests: + - dbt_constraints.primary_key: + column_name: EVENTS_ID + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - name: VERSION + tests: + - not_null + - name: SUCCESS + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: EVENT_INDEX + tests: + - not_null + - name: EVENT_TYPE + tests: + - not_null + - name: EVENT_ADDRESS + tests: + - not_null + - name: EVENT_MODULE + tests: + - not_null + - name: EVENT_RESOURCE + tests: + - not_null + - name: EVENT_DATA + - name: ACCOUNT_ADDRESS + tests: + - not_null + - name: CREATION_NUMBER + tests: + - not_null + - name: SEQUENCE_NUMBER + tests: + - not_null + - name: EVENTS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_silver__events_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_silver__eventss_INVOCATION_ID + test_name: not_null \ No newline at end of file diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 5930de0..6eef4d8 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -3,7 +3,8 @@ unique_key = "tx_hash", incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','tx_type'] + cluster_by = ['block_timestamp::DATE','_inserted_timestamp::DATE','tx_type'], + tags = ['core'] ) }} SELECT @@ -12,6 +13,8 @@ SELECT DATA :data :block_timestamp :: STRING ) AS block_timestamp, b.value :hash :: STRING AS tx_hash, + b.value :version :: INT AS version, + b.value :success :: BOOLEAN AS success, b.value :type :: STRING AS tx_type, -- DATA :data :block_height :: INT AS block_height, b.value :accumulator_root_hash :: STRING AS accumulator_root_hash, @@ -32,10 +35,8 @@ SELECT b.value :sender :: STRING AS sender, --only ut b.value :signature :: STRING AS signature, --only ut b.value :state_change_hash :: STRING AS state_change_hash, - b.value :state_checkpoint_hash :: STRING AS state_checkpoint_hash, --only type state_checkpoint_transaction (sch) is not null - b.value :success :: BOOLEAN AS success, - b.value :timestamp :: bigint AS TIMESTAMP, -- same as block_timestamp - b.value :version :: INT AS version, + b.value :state_checkpoint_hash :: STRING AS state_checkpoint_hash, --only type state_checkpoint_transaction (sch) is not null + b.value :timestamp :: bigint AS TIMESTAMP, -- same as block_timestamp b.value :vm_status :: STRING AS vm_status, --same as succeeded {{ dbt_utils.generate_surrogate_key( ['tx_hash'] diff --git a/models/silver/core/silver__transactions.yml b/models/silver/core/silver__transactions.yml new file mode 100644 index 0000000..cc2bf7a --- /dev/null +++ b/models/silver/core/silver__transactions.yml @@ -0,0 +1,79 @@ +version: 2 +models: + - name: silver__transactions + + tests: + - dbt_constraints.primary_key: + column_name: TRANSACTIONS_ID + + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - name: TX_HASH + tests: + - not_null + - name: VERSION + tests: + - not_null + - name: SUCCESS + tests: + - not_null + - name: TX_TYPE + tests: + - not_null + - name: ACCUMULATOR_ROOT_HASH + tests: + - not_null + - name: CHANGES + - name: EPOCH + - name: EVENT_ROOT_HASH + tests: + - not_null + - name: EVENTS + - name: EXPIRATION_TIMESTAMP_SECS + - name: FAILED_PROPOSER_INDICES + - name: GAS_UNIT_PRICE + - name: GAS_USED + tests: + - not_null + - name: ID + - name: MAX_GAS_AMOUNT + - name: PAYLOAD + - name: PREVIOUS_BLOCK_VOTES_BITVEC + - name: PROPOSER + - name: ROUND + - name: SENDER + - name: SIGNATURE + - name: STATE_CHANGE_HASH + tests: + - not_null + - name: STATE_CHECKPOINT_HASH + - name: TIMESTAMP + tests: + - not_null + - name: VM_STATUS + tests: + - not_null + - name: TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_silver__transactions_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_silver__transactions_INVOCATION_ID + test_name: not_null + \ No newline at end of file diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.sql b/models/silver/core/tests/blocks/test_silver__blocks_full.sql new file mode 100644 index 0000000..89acf46 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'silver__blocks' + ) }} diff --git a/models/silver/core/tests/blocks/test_silver__blocks_full.yml b/models/silver/core/tests/blocks/test_silver__blocks_full.yml new file mode 100644 index 0000000..2cdc737 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_full.yml @@ -0,0 +1,89 @@ +version: 2 +models: + - name: test_silver__blocks_full + + tests: + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + columns: + - name: BLOCK_NUMBER + tests: + - unique + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP_NUM + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: FIRST_VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: LAST_VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_COUNT_FROM_TRANSACTIONS_ARRAY + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TX_COUNT_FROM_VERSIONS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCKS_ID + tests: + - not_null + - unique + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_test_silver__blocks_INSERTED_TIMESTAMP + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_test_silver__blocks_INVOCATION_ID + test_name: not_null diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.sql b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql new file mode 100644 index 0000000..c27a8a9 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__blocks') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/blocks/test_silver__blocks_recent.yml b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml new file mode 100644 index 0000000..aba65e4 --- /dev/null +++ b/models/silver/core/tests/blocks/test_silver__blocks_recent.yml @@ -0,0 +1,21 @@ +version: 2 +models: + - name: test_silver__blocks_recent + + tests: + - fsc_utils.sequence_gaps: + column_name: BLOCK_NUMBER + config: + severity: error + error_if: ">10" + columns: + - name: BLOCK_NUMBER + tests: + - unique + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 \ No newline at end of file diff --git a/models/silver/core/tests/changes/test_silver__changes_full.sql b/models/silver/core/tests/changes/test_silver__changes_full.sql new file mode 100644 index 0000000..0f071ec --- /dev/null +++ b/models/silver/core/tests/changes/test_silver__changes_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'silver__changes' + ) }} diff --git a/models/silver/core/tests/changes/test_silver__changes_full.yml b/models/silver/core/tests/changes/test_silver__changes_full.yml new file mode 100644 index 0000000..12e2827 --- /dev/null +++ b/models/silver/core/tests/changes/test_silver__changes_full.yml @@ -0,0 +1,148 @@ +version: 2 +models: + - name: test_silver__changes_full + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - CHANGE_INDEX + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGE_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: CHANGE_DATA + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: CHANGE_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: ADDRESS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: HANDLE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: INNER_CHANGE_TYPE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGE_ADDRESS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGE_MODULE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGE_RESOURCE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: KEY + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: VALUE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: STATE_KEY_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGES_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_test_silver__changes_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_test_silver__changes_INVOCATION_ID + test_name: not_null \ No newline at end of file diff --git a/models/silver/core/tests/changes/test_silver__changes_recent.sql b/models/silver/core/tests/changes/test_silver__changes_recent.sql new file mode 100644 index 0000000..a966bd4 --- /dev/null +++ b/models/silver/core/tests/changes/test_silver__changes_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__changes') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/changes/test_silver__changes_recent.yml b/models/silver/core/tests/changes/test_silver__changes_recent.yml new file mode 100644 index 0000000..5ff1232 --- /dev/null +++ b/models/silver/core/tests/changes/test_silver__changes_recent.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: test_silver__changes_recent + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - CHANGE_INDEX + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - name: TX_HASH + tests: + - not_null + - name: CHANGE_INDEX + tests: + - not_null \ No newline at end of file diff --git a/models/silver/core/tests/events/test_silver__events_full.sql b/models/silver/core/tests/events/test_silver__events_full.sql new file mode 100644 index 0000000..8301458 --- /dev/null +++ b/models/silver/core/tests/events/test_silver__events_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'silver__events' + ) }} diff --git a/models/silver/core/tests/events/test_silver__events_full.yml b/models/silver/core/tests/events/test_silver__events_full.yml new file mode 100644 index 0000000..8e581a5 --- /dev/null +++ b/models/silver/core/tests/events/test_silver__events_full.yml @@ -0,0 +1,136 @@ +version: 2 +models: + - name: test_silver__events_full + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - EVENT_INDEX + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENT_INDEX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENT_MODULE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENT_RESOURCE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENT_DATA + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: ACCOUNT_ADDRESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CREATION_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SEQUENCE_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENTS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_test_silver__events_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_test_silver__events_INVOCATION_ID + test_name: not_null \ No newline at end of file diff --git a/models/silver/core/tests/events/test_silver__events_recent.sql b/models/silver/core/tests/events/test_silver__events_recent.sql new file mode 100644 index 0000000..1d31098 --- /dev/null +++ b/models/silver/core/tests/events/test_silver__events_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__events') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/events/test_silver__events_recent.yml b/models/silver/core/tests/events/test_silver__events_recent.yml new file mode 100644 index 0000000..ebf0858 --- /dev/null +++ b/models/silver/core/tests/events/test_silver__events_recent.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: test_silver__events_recent + + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TX_HASH + - EVENT_INDEX + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - name: TX_HASH + tests: + - not_null + - name: EVENT_INDEX + tests: + - not_null \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.sql b/models/silver/core/tests/transactions/test_silver__transactions_full.sql new file mode 100644 index 0000000..dcbd02c --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.sql @@ -0,0 +1,11 @@ +{{ config( + materialized = 'view', + tags = ['full_test'] +) }} + +SELECT + * +FROM + {{ ref( + 'silver__transactions' + ) }} diff --git a/models/silver/core/tests/transactions/test_silver__transactions_full.yml b/models/silver/core/tests/transactions/test_silver__transactions_full.yml new file mode 100644 index 0000000..822f0c8 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_full.yml @@ -0,0 +1,201 @@ +version: 2 +models: + - name: test_silver__transactions_full + + tests: + - fsc_utils.sequence_gaps: + column_name: VERSION + where: BLOCK_TIMESTAMP < CURRENT_DATE - 1 + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TX_HASH + tests: + - not_null + - unique + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - dbt_expectations.expect_column_values_to_match_regex: + regex: 0[xX][0-9a-fA-F]+ + - name: VERSION + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - BOOLEAN + - name: TX_TYPE + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: ACCUMULATOR_ROOT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: CHANGES + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: EPOCH + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: EVENT_ROOT_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: EVENTS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: EXPIRATION_TIMESTAMP_SECS + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: FAILED_PROPOSER_INDICES + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: GAS_UNIT_PRICE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: GAS_USED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: ID + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: MAX_GAS_AMOUNT + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: PAYLOAD + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: PREVIOUS_BLOCK_VOTES_BITVEC + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - VARIANT + - name: PROPOSER + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: ROUND + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: SENDER + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: SIGNATURE + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: STATE_CHANGE_HASH + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: STATE_CHECKPOINT_HASH + tests: + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: VM_STATUS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + - name: TRANSACTIONS_ID + tests: + - not_null + - name: INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - name: not_null_test_silver__transactions_INSERTED_TIMESTAMP_ + test_name: not_null + - name: _INVOCATION_ID + tests: + - name: not_null_test_silver__transactions_INVOCATION_ID + test_name: not_null \ No newline at end of file diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.sql b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql new file mode 100644 index 0000000..a7b0802 --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['recent_test'] +) }} + +WITH last_3_days AS ( + + SELECT + block_number + FROM + {{ ref("_max_block_by_date") }} + qualify ROW_NUMBER() over ( + ORDER BY + block_number DESC + ) = 3 +) +SELECT + * +FROM + {{ ref('silver__transactions') }} +WHERE + block_number >= ( + SELECT + block_number + FROM + last_3_days + ) diff --git a/models/silver/core/tests/transactions/test_silver__transactions_recent.yml b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml new file mode 100644 index 0000000..fc99f4b --- /dev/null +++ b/models/silver/core/tests/transactions/test_silver__transactions_recent.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: test_silver__transactions_recent + + tests: + - fsc_utils.sequence_gaps: + column_name: VERSION + config: + severity: error + error_if: ">10" + + columns: + - name: BLOCK_NUMBER + tests: + - not_null + - name: BLOCK_TIMESTAMP + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 3 + - name: TX_HASH + tests: + - not_null + - unique + - name: VERSION + tests: + - not_null \ No newline at end of file diff --git a/models/streamline/silver/_max_block_by_date.sql b/models/streamline/silver/_max_block_by_date.sql new file mode 100644 index 0000000..a56cf82 --- /dev/null +++ b/models/streamline/silver/_max_block_by_date.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "ephemeral", + unique_key = "block_number", +) }} + +WITH base AS ( + + SELECT + block_timestamp :: DATE AS block_date, + MAX(block_number) block_number + FROM + {{ ref("silver__blocks") }} + GROUP BY + block_timestamp :: DATE +) +SELECT + block_date, + block_number +FROM + base +WHERE + block_date <> ( + SELECT + MAX(block_date) + FROM + base + ) \ No newline at end of file