Merge pull request #4 from antonyip/chainwalker_dedupe

feat: Chainwalker blocks+txs and blocks

models/core/blocks.sql

@@ -2,30 +2,59 @@
     config(
         materialized='incremental',
         unique_key='block_id',
+        incremental_strategy = 'delete+insert',
         tags=['core'],
         cluster_by=['block_timestamp']
     )
 }}
 
--- WIP -- below serves as example
-with base_blocks as (
+with
 
-    select * from {{ deduped_blocks("near_blocks") }}
+base_blocks as (
+
+    select
+        *
+    from {{ ref("stg_blocks") }}
+    where {{ incremental_load_filter("block_timestamp") }}
 
 ),
 final as (
-    select
-        block_id,
-        block_timestamp,
-        header:hash::string as block_hash,
-        header:parent_hash::string as block_parent_hash,
-        header:miner::string as miner,
-        header:size as size,
-        -- Build out more columns here from `header`
-        -- ...
 
+    select
+        header:height::integer as block_id,
+        div0(header:timestamp::integer,1000000000)::timestamp as block_timestamp,
+        header:hash::string as block_hash,
+        header:tx_count::integer as tx_count,
+        header:author::string as block_author,
+        header:challenges_result as block_challenges_result,
+        header:challenges_root::string as block_challenges_root,
+        header:chunk_headers_root::string as chunk_headers_root,
+        header:chunk_mask as chunk_mask,
+        header:chunk_receipts_root::string as chunk_receipts_root,
+        header:chunk_tx_root::string as chunk_tx_root,
+        header:chunks as chunks,
+        header:chunks_included::integer as chunks_included,
+        header:epoch_id::string as epoch_id,
+        header:epoch_sync_data_hash::string as epoch_sync_data_hash,
+        header:events as events,
+        header:gas_price::float as gas_price,
+        header:last_ds_final_block::string as last_ds_final_block,
+        header:last_final_block::string as last_final_block,
+        header:latest_protocol_version::integer as latest_protocol_version,
+        header:next_bp_hash::string as next_bp_hash,
+        header:next_epoch_id::string as next_epoch_id,
+        header:outcome_root::string as outcome_root,
+        header:prev_hash::string as prev_hash,
+        header:prev_height::integer as prev_height,
+        header:prev_state_root::string as prev_state_root,
+        header:random_value::string as random_value,
+        header:rent_paid::float as rent_paid,
+        header:signature::string as signature,
+        header:total_supply::float as total_supply,
+        header:validator_proposals as validator_proposals,
+        header:validator_reward::float as validator_reward
     from base_blocks
-    where {{ incremental_load_filter("block_timestamp") }}
+
 )
 
 select * from final

models/core/blocks.yml

@@ -7,64 +7,161 @@ models:
 
     columns:
       - name: block_id
-        description: The block number.
+        description: The `block_id` taken from block headers. The block number that represent the length of the blockchain in blocks.
         tests:
-          - unique
           - not_null
 
       - name: block_timestamp
-        description: The timestamp for when the block was collated.
+        description: The `block_timestamp` taken from block headers.  The time when the block was created.
         tests:
           - not_null
 
       - name: block_hash
-        description: Hash of the block (32 Bytes).
-        tests:
-          - unique
-          - not_null
-
-      - name: block_parent_hash
-        description: Hash of the parent block (32 Bytes).
-        tests:
-          - unique
-          - not_null
-
-      - name: gas_limit
-        description: The maximum gas allowed in this block.
-        tests:
-          - not_null
-
-      - name: gas_used
-        description: The total used gas by all transactions in this block.
-        tests:
-          - not_null
-
-      - name: miner
-        description: The address of the beneficiary to whom the mining rewards were given.
-        tests:
-          - not_null
-
-      - name: nonce
-        description: Hash of the generated proof-of-work (8 Bytes).
-        tests:
-          - not_null
-
-      - name: size
-        description: Integer the size of this block in bytes.
+        description: The `block_hash` taken from block headers. A unique identifier for that block.
         tests:
           - not_null
 
       - name: tx_count
-        description: The number of transactions in the given block.
+        description: The `tx_count` taken from block headers. The number of transactions included in the block.
         tests:
           - not_null
 
-      - name: state_root
-        description: The root hash that stores the entire state of the system - account balances, contract storage, contract code and account nonces are inside
+      - name: block_author
+        description: The `block_author` taken from block headers. Validators of the blockchain.
         tests:
           - not_null
 
-      - name: receipts_root
-        description: The root hash that stores receipts of all transactions included in the block.
+      - name: block_challenges_result
+        description: The block_challenges_result taken from block headers.
+        tests:
+          - not_null
+
+      - name: block_challenges_root
+        description: The block_challenges_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_headers_root
+        description: The chunk_headers_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_mask
+        description: The chunk_mask taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_receipts_root
+        description: The chunk_receipts_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_tx_root
+        description: The chunk_tx_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunks
+        description: The chunks taken from block headers. Chunk is an aggregation of transactions which are executed within a particular shard.
+        tests:
+          - not_null
+
+      - name: chunks_included
+        description: The chunks_included taken from block headers.
+        tests:
+          - not_null
+
+      - name: epoch_id
+        description: The epoch_id taken from block headers.
+        tests:
+          - not_null
+
+      - name: epoch_sync_data_hash
+        description: The epoch_sync_data_hash taken from block headers.
+        tests:
+          - not_null
+
+      - name: events
+        description: The events taken from block headers.
+        tests:
+          - not_null
+
+      - name: gas_price
+        description: The gas_price taken from block headers.
+        tests:
+          - not_null
+
+      - name: last_ds_final_block
+        description: The last_ds_final_block taken from block headers.
+        tests:
+          - not_null
+
+      - name: last_final_block
+        description: The last_final_block taken from block headers.
+        tests:
+          - not_null
+
+      - name: latest_protocol_version
+        description: The latest_protocol_version taken from block headers.
+        tests:
+          - not_null
+
+      - name: next_bp_hash
+        description: The next_bp_hash taken from block headers.
+        tests:
+          - not_null
+
+      - name: next_epoch_id
+        description: The next_epoch_id taken from block headers.
+        tests:
+          - not_null
+
+      - name: outcome_root
+        description: The outcome_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: prev_hash
+        description: The prev_hash taken from block headers.
+        tests:
+          - not_null
+
+      - name: prev_height
+        description: The prev_height taken from block headers.
+        tests:
+          
+
+      - name: prev_state_root
+        description: The prev_state_root taken from block headers.
+        tests:
+          
+
+      - name: random_value
+        description: The random_value taken from block headers.
+        tests:
+          - not_null
+
+      - name: rent_paid
+        description: The rent_paid taken from block headers.
+        tests:
+          - not_null
+
+      - name: signature
+        description: The signature taken from block headers.
+        tests:
+          - not_null
+
+      - name: total_supply
+        description: The total_supply taken from block headers.
+        tests:
+          - not_null
+
+      - name: validator_proposals
+        description: The validator_proposals taken from block headers.
+        tests:
+          - not_null
+
+      - name: validator_reward
+        description: The validator_reward taken from block headers.
         tests:
           - not_null

models/staging/stg_blocks.sql

@@ -0,0 +1,23 @@
+{{
+    config(
+        materialized='incremental',
+        unique_key='tx_id',
+        incremental_strategy = 'delete+insert',
+        tags=['core'],
+        cluster_by=['block_timestamp']
+        )
+}}
+
+with
+
+final as (
+
+    select
+        *
+    from {{ source("chainwalkers","near_blocks") }}
+    where {{ incremental_load_filter("block_timestamp") }}
+    qualify row_number() over (partition by block_id order by ingested_at desc) = 1
+
+)
+
+select * from final

models/staging/stg_blocks.yml

@@ -0,0 +1,35 @@
+version: 2
+
+models:
+  - name: stg_blocks
+    description: |-
+      This table records all the blocks of the Near blockchain.
+
+    columns:
+      - name: record_id
+        description: A unique id for the record generated by Chainwalkers.
+
+      - name: offset_id
+        description: Synonmous with block_id for Near.
+
+      - name: block_id
+        description: The height of the chain this block corresponds with.
+
+      - name: block_timestamp
+        description: The time the block was minted.
+
+      - name: network
+        description: The blockchain network. (i.e. mainnet, testnet, etc.)
+
+      - name: chain_id
+        description: Synonmous with blockchain name for Near.
+
+      - name: tx_count
+        description: The number of transactions in the block.
+
+      - name: header
+        description: A json column containing the blocks header information.
+
+      - name: ingested_at
+        description: The time this row was ingested by Chainwalkers.
+

models/staging/stg_txs.sql

@@ -0,0 +1,23 @@
+{{
+    config(
+        materialized='incremental',
+        unique_key='tx_id',
+        incremental_strategy = 'delete+insert',
+        tags=['core', 'transactions'],
+        cluster_by=['block_timestamp']
+        )
+}}
+
+with
+
+final as (
+
+    select
+        *
+    from {{ source("chainwalkers","near_txs") }}
+    where {{ incremental_load_filter("block_timestamp") }}
+    qualify row_number() over (partition by block_id order by ingested_at desc) = 1
+
+)
+
+select * from final

models/staging/stg_txs.yml

@@ -0,0 +1,37 @@
+version: 2
+
+models:
+  - name: stg_txs
+    description: |-
+      This table records all the transactions of the Near blockchain.
+
+    columns:
+      - name: record_id
+        description: A unique id for the record generated by Chainwalkers
+
+      - name: tx_id
+        description: Synonmous with transaction hash, a unique on chain identifier for the transaction
+
+      - name: tx_block_index
+        description: The index of the transaction within the block. Starts at 0.
+
+      - name: offset_id
+        description: Synonmous with `block_id` for Near
+
+      - name: block_id
+        description: The height of the chain this block corresponds with
+
+      - name: block_timestamp
+        description: The time the block was minted
+
+      - name: network
+        description: The blockchain network (i.e. mainnet, testnet, etc.)
+
+      - name: chain_id
+        description: Synonmous with blockchain name for Near
+
+      - name: tx
+        description: A json object containing the tx and any decoded logs
+
+      - name: ingested_at
+        description: The time this row was ingested by Chainwalkers