From 054aedecc42a767d1303ac8007a07d714fae3ede Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Thu, 27 Jan 2022 05:58:55 +0800
Subject: [PATCH 1/7] feat: Chainwalker blocks+txs and blocks

---
 models/core/blocks.sql                |  55 ++++++--
 models/core/blocks.yml                | 179 ++++++++++++++++++++------
 models/staging/chainwalker_blocks.sql |  23 ++++
 models/staging/chainwalker_blocks.yml |  26 ++++
 models/staging/chainwalker_txs.sql    |  23 ++++
 models/staging/chainwalker_txs.yml    |  32 +++++
 6 files changed, 284 insertions(+), 54 deletions(-)
 create mode 100644 models/staging/chainwalker_blocks.sql
 create mode 100644 models/staging/chainwalker_blocks.yml
 create mode 100644 models/staging/chainwalker_txs.sql
 create mode 100644 models/staging/chainwalker_txs.yml

diff --git a/models/core/blocks.sql b/models/core/blocks.sql
index 41b160a..5caeb8b 100644
--- a/models/core/blocks.sql
+++ b/models/core/blocks.sql
@@ -2,30 +2,59 @@
     config(
         materialized='incremental',
         unique_key='block_id',
+        incremental_strategy = 'delete+insert',
         tags=['core'],
         cluster_by=['block_timestamp']
     )
 }}
 
--- WIP -- below serves as example
-with base_blocks as (
+with
 
-    select * from {{ deduped_blocks("near_blocks") }}
+base_blocks as (
+
+    select
+        *
+    from {{ ref("chainwalker_blocks") }}
+    where {{ incremental_load_filter("block_timestamp") }}
 
 ),
 final as (
-    select
-        block_id,
-        block_timestamp,
-        header:hash::string as block_hash,
-        header:parent_hash::string as block_parent_hash,
-        header:miner::string as miner,
-        header:size as size,
-        -- Build out more columns here from `header`
-        -- ...
 
+    select
+        header:height::integer as block_id,
+        header:timestamp::timestamp as block_timestamp,
+        header:hash::string as block_hash,
+        header:tx_count::integer as tx_count,
+        header:author::string as block_author,
+        header:challenges_result as block_challenges_result,
+        header:challenges_root::string as block_challenges_root,
+        header:chunk_headers_root::string as chunk_headers_root,
+        header:chunk_mask as chunk_mask,
+        header:chunk_receipts_root::string as chunk_receipts_root,
+        header:chunk_tx_root::string as chunk_tx_root,
+        header:chunks::string as chunks,
+        header:chunks_included::string as chunks_included,
+        header:epoch_id::string as epoch_id,
+        header:epoch_sync_data_hash::string as epoch_sync_data_hash,
+        header:events as events,
+        header:gas_price::float as gas_price,
+        header:last_ds_final_block::string as last_ds_final_block,
+        header:last_final_block::string as last_final_block,
+        header:latest_protocol_version::integer as latest_protocol_version,
+        header:next_bp_hash::string as next_bp_hash,
+        header:next_epoch_id::string as next_epoch_id,
+        header:outcome_root::string as outcome_root,
+        header:prev_hash::string as prev_hash,
+        header:prev_height::integer as prev_height,
+        header:prev_state_root::string as prev_state_root,
+        header:random_value::string as random_value,
+        header:rent_paid::float as rent_paid,
+        header:signature::string as signature,
+        header:total_supply::float as total_supply,
+        header:validator_proposals as validator_proposals,
+        header:validator_reward::float as validator_reward
     from base_blocks
-    where {{ incremental_load_filter("block_timestamp") }}
+
 )
 
 select * from final
diff --git a/models/core/blocks.yml b/models/core/blocks.yml
index 84623c3..964fc7a 100644
--- a/models/core/blocks.yml
+++ b/models/core/blocks.yml
@@ -7,64 +7,161 @@ models:
 
     columns:
       - name: block_id
-        description: The block number.
+        description: The block_id taken from block headers.
         tests:
-          - unique
           - not_null
 
       - name: block_timestamp
-        description: The timestamp for when the block was collated.
+        description: The block_timestamp taken from block headers.
         tests:
           - not_null
 
       - name: block_hash
-        description: Hash of the block (32 Bytes).
-        tests:
-          - unique
-          - not_null
-
-      - name: block_parent_hash
-        description: Hash of the parent block (32 Bytes).
-        tests:
-          - unique
-          - not_null
-
-      - name: gas_limit
-        description: The maximum gas allowed in this block.
-        tests:
-          - not_null
-
-      - name: gas_used
-        description: The total used gas by all transactions in this block.
-        tests:
-          - not_null
-
-      - name: miner
-        description: The address of the beneficiary to whom the mining rewards were given.
-        tests:
-          - not_null
-
-      - name: nonce
-        description: Hash of the generated proof-of-work (8 Bytes).
-        tests:
-          - not_null
-
-      - name: size
-        description: Integer the size of this block in bytes.
+        description: The block_hash taken from block headers.
         tests:
           - not_null
 
       - name: tx_count
-        description: The number of transactions in the given block.
+        description: The tx_count taken from block headers.
         tests:
           - not_null
 
-      - name: state_root
-        description: The root hash that stores the entire state of the system - account balances, contract storage, contract code and account nonces are inside
+      - name: block_author
+        description: The block_author taken from block headers.
         tests:
           - not_null
 
-      - name: receipts_root
-        description: The root hash that stores receipts of all transactions included in the block.
+      - name: block_challenges_result
+        description: The block_challenges_result taken from block headers.
+        tests:
+          - not_null
+
+      - name: block_challenges_root
+        description: The block_challenges_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_headers_root
+        description: The chunk_headers_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_mask
+        description: The chunk_mask taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_receipts_root
+        description: The chunk_receipts_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunk_tx_root
+        description: The chunk_tx_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunks
+        description: The chunks taken from block headers.
+        tests:
+          - not_null
+
+      - name: chunks_included
+        description: The chunks_included taken from block headers.
+        tests:
+          - not_null
+
+      - name: epoch_id
+        description: The epoch_id taken from block headers.
+        tests:
+          - not_null
+
+      - name: epoch_sync_data_hash
+        description: The epoch_sync_data_hash taken from block headers.
+        tests:
+          - not_null
+
+      - name: events
+        description: The events taken from block headers.
+        tests:
+          - not_null
+
+      - name: gas_price
+        description: The gas_price taken from block headers.
+        tests:
+          - not_null
+
+      - name: last_ds_final_block
+        description: The last_ds_final_block taken from block headers.
+        tests:
+          - not_null
+
+      - name: last_final_block
+        description: The last_final_block taken from block headers.
+        tests:
+          - not_null
+
+      - name: latest_protocol_version
+        description: The latest_protocol_version taken from block headers.
+        tests:
+          - not_null
+
+      - name: next_bp_hash
+        description: The next_bp_hash taken from block headers.
+        tests:
+          - not_null
+
+      - name: next_epoch_id
+        description: The next_epoch_id taken from block headers.
+        tests:
+          - not_null
+
+      - name: outcome_root
+        description: The outcome_root taken from block headers.
+        tests:
+          - not_null
+
+      - name: prev_hash
+        description: The prev_hash taken from block headers.
+        tests:
+          - not_null
+
+      - name: prev_height
+        description: The prev_height taken from block headers.
+        tests:
+          
+
+      - name: prev_state_root
+        description: The prev_state_root taken from block headers.
+        tests:
+          
+
+      - name: random_value
+        description: The random_value taken from block headers.
+        tests:
+          - not_null
+
+      - name: rent_paid
+        description: The rent_paid taken from block headers.
+        tests:
+          - not_null
+
+      - name: signature
+        description: The signature taken from block headers.
+        tests:
+          - not_null
+
+      - name: total_supply
+        description: The total_supply taken from block headers.
+        tests:
+          - not_null
+
+      - name: validator_proposals
+        description: The validator_proposals taken from block headers.
+        tests:
+          - not_null
+
+      - name: validator_reward
+        description: The validator_reward taken from block headers.
         tests:
           - not_null
diff --git a/models/staging/chainwalker_blocks.sql b/models/staging/chainwalker_blocks.sql
new file mode 100644
index 0000000..a4ab94c
--- /dev/null
+++ b/models/staging/chainwalker_blocks.sql
@@ -0,0 +1,23 @@
+{{
+    config(
+        materialized='incremental',
+        unique_key='tx_id',
+        incremental_strategy = 'delete+insert',
+        tags=['core', 'transactions'],
+        cluster_by=['block_timestamp']
+        )
+}}
+
+with
+
+final as (
+
+    select
+        *
+    from {{ source("chainwalkers","near_blocks") }}
+    where {{ incremental_load_filter("block_timestamp") }}
+    qualify row_number() over (partition by block_id order by ingested_at desc) = 1
+
+)
+
+select * from final
diff --git a/models/staging/chainwalker_blocks.yml b/models/staging/chainwalker_blocks.yml
new file mode 100644
index 0000000..b049bf8
--- /dev/null
+++ b/models/staging/chainwalker_blocks.yml
@@ -0,0 +1,26 @@
+version: 2
+
+models:
+  - name: chainwalker_blocks
+    description: |-
+      This table records all the blocks of the Near blockchain.
+
+    columns:
+      - name: record_id
+        description: A unique id for the record generated by Chainwalkers.
+      - name: offset_id
+        description: Synonmous with block_id for Near.
+      - name: block_id
+        description: The height of the chain this block corresponds with.
+      - name: block_timestamp
+        description: The time the block was minted.
+      - name: network
+        description: The blockchain network. (i.e. mainnet, testnet, etc.)
+      - name: chain_id
+        description: Synonmous with blockchain name for Near.
+      - name: tx_count
+        description: The number of transactions in the block.
+      - name: header
+        description: A json column containing the blocks header information.
+      - name: ingested_at
+        description: The time this row was ingested by Chainwalkers.
diff --git a/models/staging/chainwalker_txs.sql b/models/staging/chainwalker_txs.sql
new file mode 100644
index 0000000..648b8c9
--- /dev/null
+++ b/models/staging/chainwalker_txs.sql
@@ -0,0 +1,23 @@
+{{
+    config(
+        materialized='incremental',
+        unique_key='tx_id',
+        incremental_strategy = 'delete+insert',
+        tags=['core', 'transactions'],
+        cluster_by=['block_timestamp']
+        )
+}}
+
+with
+
+final as (
+
+    select
+        *
+    from {{ source("chainwalkers","near_txs") }}
+    where {{ incremental_load_filter("block_timestamp") }}
+    qualify row_number() over (partition by block_id order by ingested_at desc) = 1
+
+)
+
+select * from final
diff --git a/models/staging/chainwalker_txs.yml b/models/staging/chainwalker_txs.yml
new file mode 100644
index 0000000..55b84c6
--- /dev/null
+++ b/models/staging/chainwalker_txs.yml
@@ -0,0 +1,32 @@
+version: 2
+
+models:
+  - name: chainwalker_txs
+    description: |-
+      This table records all the transactions of the Near blockchain.
+
+    columns:
+      - name: record_id
+        description: A unique id for the record generated by Chainwalkers
+      - name: tx_id
+        description: Synonmous with transaction hash, a unique on chain identifier for the transaction
+      - name: tx_block_index
+        description: The index of the transaction within the block. Starts at 0.
+      - name: offset_id
+        description: Synonmous with block_id for Near
+      - name: block_id
+        description: The height of the chain this block corresponds with
+      - name: block_timestamp
+        description: The time the block was minted
+      - name: network
+        description: The blockchain network (i.e. mainnet, testnet, etc.)
+      - name: chain_id
+        description: Synonmous with blockchain name for Near
+      - name: tx_count
+        description: The number of transactions in the block
+      - name: header
+        description: A json column containing the blocks header information
+      - name: tx
+        description: A json object containing the tx and any decoded logs
+      - name: ingested_at
+        description: The time this row was ingested by Chainwalkers

From 460788639d2a66b0a3b6e8064ce49e27b8f33fdb Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Thu, 27 Jan 2022 22:34:18 +0800
Subject: [PATCH 2/7] fix: rename files

---
 models/core/blocks.sql                                    | 2 +-
 models/staging/{chainwalker_blocks.sql => stg_blocks.sql} | 0
 models/staging/{chainwalker_blocks.yml => stg_blocks.yml} | 2 +-
 models/staging/{chainwalker_txs.sql => stg_txs.sql}       | 0
 models/staging/{chainwalker_txs.yml => stg_txs.yml}       | 2 +-
 5 files changed, 3 insertions(+), 3 deletions(-)
 rename models/staging/{chainwalker_blocks.sql => stg_blocks.sql} (100%)
 rename models/staging/{chainwalker_blocks.yml => stg_blocks.yml} (96%)
 rename models/staging/{chainwalker_txs.sql => stg_txs.sql} (100%)
 rename models/staging/{chainwalker_txs.yml => stg_txs.yml} (97%)

diff --git a/models/core/blocks.sql b/models/core/blocks.sql
index 5caeb8b..055a6ef 100644
--- a/models/core/blocks.sql
+++ b/models/core/blocks.sql
@@ -14,7 +14,7 @@ base_blocks as (
 
     select
         *
-    from {{ ref("chainwalker_blocks") }}
+    from {{ ref("stg_blocks") }}
     where {{ incremental_load_filter("block_timestamp") }}
 
 ),
diff --git a/models/staging/chainwalker_blocks.sql b/models/staging/stg_blocks.sql
similarity index 100%
rename from models/staging/chainwalker_blocks.sql
rename to models/staging/stg_blocks.sql
diff --git a/models/staging/chainwalker_blocks.yml b/models/staging/stg_blocks.yml
similarity index 96%
rename from models/staging/chainwalker_blocks.yml
rename to models/staging/stg_blocks.yml
index b049bf8..209becd 100644
--- a/models/staging/chainwalker_blocks.yml
+++ b/models/staging/stg_blocks.yml
@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: chainwalker_blocks
+  - name: stg_blocks
     description: |-
       This table records all the blocks of the Near blockchain.
 
diff --git a/models/staging/chainwalker_txs.sql b/models/staging/stg_txs.sql
similarity index 100%
rename from models/staging/chainwalker_txs.sql
rename to models/staging/stg_txs.sql
diff --git a/models/staging/chainwalker_txs.yml b/models/staging/stg_txs.yml
similarity index 97%
rename from models/staging/chainwalker_txs.yml
rename to models/staging/stg_txs.yml
index 55b84c6..248bda3 100644
--- a/models/staging/chainwalker_txs.yml
+++ b/models/staging/stg_txs.yml
@@ -1,7 +1,7 @@
 version: 2
 
 models:
-  - name: chainwalker_txs
+  - name: stg_txs
     description: |-
       This table records all the transactions of the Near blockchain.
 

From bfd0c113bc4ee4764da45a2832cd139d43166fd9 Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Mon, 31 Jan 2022 09:25:37 +0800
Subject: [PATCH 3/7] fix: for timestamp as snowflake is not able to handle
 nanosecs

---
 models/core/blocks.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/models/core/blocks.sql b/models/core/blocks.sql
index 055a6ef..4441cae 100644
--- a/models/core/blocks.sql
+++ b/models/core/blocks.sql
@@ -22,7 +22,7 @@ final as (
 
     select
         header:height::integer as block_id,
-        header:timestamp::timestamp as block_timestamp,
+        div0(header:timestamp::integer,1000000000)::timestamp as block_timestamp,
         header:hash::string as block_hash,
         header:tx_count::integer as tx_count,
         header:author::string as block_author,

From 4c3ecad0b157f1879d8c27ab3d4b4167e3a9f74b Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Sat, 5 Feb 2022 09:46:11 +0800
Subject: [PATCH 4/7] Apply suggestions from code review

Co-authored-by: Chuxin Huang <30974572+chuxinh@users.noreply.github.com>
---
 models/staging/stg_blocks.sql | 2 +-
 models/staging/stg_txs.yml    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/models/staging/stg_blocks.sql b/models/staging/stg_blocks.sql
index a4ab94c..bd65268 100644
--- a/models/staging/stg_blocks.sql
+++ b/models/staging/stg_blocks.sql
@@ -3,7 +3,7 @@
         materialized='incremental',
         unique_key='tx_id',
         incremental_strategy = 'delete+insert',
-        tags=['core', 'transactions'],
+        tags=['core'],
         cluster_by=['block_timestamp']
         )
 }}
diff --git a/models/staging/stg_txs.yml b/models/staging/stg_txs.yml
index 248bda3..fa497f4 100644
--- a/models/staging/stg_txs.yml
+++ b/models/staging/stg_txs.yml
@@ -13,7 +13,7 @@ models:
       - name: tx_block_index
         description: The index of the transaction within the block. Starts at 0.
       - name: offset_id
-        description: Synonmous with block_id for Near
+        description: Synonmous with `block_id` for Near
       - name: block_id
         description: The height of the chain this block corresponds with
       - name: block_timestamp

From 382ffa370d35a58ed74bbf4ac688c05e911e03cd Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Sat, 5 Feb 2022 10:03:26 +0800
Subject: [PATCH 5/7] Apply suggestions from code review

---
 models/core/blocks.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/models/core/blocks.sql b/models/core/blocks.sql
index 4441cae..961fdd3 100644
--- a/models/core/blocks.sql
+++ b/models/core/blocks.sql
@@ -32,8 +32,8 @@ final as (
         header:chunk_mask as chunk_mask,
         header:chunk_receipts_root::string as chunk_receipts_root,
         header:chunk_tx_root::string as chunk_tx_root,
-        header:chunks::string as chunks,
-        header:chunks_included::string as chunks_included,
+        header:chunks as chunks,
+        header:chunks_included::integer as chunks_included,
         header:epoch_id::string as epoch_id,
         header:epoch_sync_data_hash::string as epoch_sync_data_hash,
         header:events as events,

From a874350ec609bbaf1d353709dd37a68ddbaa51fc Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Sat, 5 Feb 2022 10:06:30 +0800
Subject: [PATCH 6/7] fix: review remove missing columns and add lines

---
 models/staging/stg_blocks.yml |  9 +++++++++
 models/staging/stg_txs.yml    | 13 +++++++++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/models/staging/stg_blocks.yml b/models/staging/stg_blocks.yml
index 209becd..f9e4c95 100644
--- a/models/staging/stg_blocks.yml
+++ b/models/staging/stg_blocks.yml
@@ -8,19 +8,28 @@ models:
     columns:
       - name: record_id
         description: A unique id for the record generated by Chainwalkers.
+
       - name: offset_id
         description: Synonmous with block_id for Near.
+
       - name: block_id
         description: The height of the chain this block corresponds with.
+
       - name: block_timestamp
         description: The time the block was minted.
+
       - name: network
         description: The blockchain network. (i.e. mainnet, testnet, etc.)
+
       - name: chain_id
         description: Synonmous with blockchain name for Near.
+
       - name: tx_count
         description: The number of transactions in the block.
+
       - name: header
         description: A json column containing the blocks header information.
+
       - name: ingested_at
         description: The time this row was ingested by Chainwalkers.
+
diff --git a/models/staging/stg_txs.yml b/models/staging/stg_txs.yml
index fa497f4..1afe4ea 100644
--- a/models/staging/stg_txs.yml
+++ b/models/staging/stg_txs.yml
@@ -8,25 +8,30 @@ models:
     columns:
       - name: record_id
         description: A unique id for the record generated by Chainwalkers
+
       - name: tx_id
         description: Synonmous with transaction hash, a unique on chain identifier for the transaction
+
       - name: tx_block_index
         description: The index of the transaction within the block. Starts at 0.
+
       - name: offset_id
         description: Synonmous with `block_id` for Near
+
       - name: block_id
         description: The height of the chain this block corresponds with
+
       - name: block_timestamp
         description: The time the block was minted
+
       - name: network
         description: The blockchain network (i.e. mainnet, testnet, etc.)
+
       - name: chain_id
         description: Synonmous with blockchain name for Near
-      - name: tx_count
-        description: The number of transactions in the block
-      - name: header
-        description: A json column containing the blocks header information
+
       - name: tx
         description: A json object containing the tx and any decoded logs
+
       - name: ingested_at
         description: The time this row was ingested by Chainwalkers

From 7c4e75697e039702c707339a1897846216d89c58 Mon Sep 17 00:00:00 2001
From: Anton Yip <anton_yip@hotmail.com>
Date: Sat, 5 Feb 2022 18:48:42 +0800
Subject: [PATCH 7/7] Apply suggestions from code review

Co-authored-by: Chuxin Huang <30974572+chuxinh@users.noreply.github.com>
---
 models/core/blocks.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/models/core/blocks.yml b/models/core/blocks.yml
index 964fc7a..c109c1a 100644
--- a/models/core/blocks.yml
+++ b/models/core/blocks.yml
@@ -7,27 +7,27 @@ models:
 
     columns:
       - name: block_id
-        description: The block_id taken from block headers.
+        description: The `block_id` taken from block headers. The block number that represent the length of the blockchain in blocks.
         tests:
           - not_null
 
       - name: block_timestamp
-        description: The block_timestamp taken from block headers.
+        description: The `block_timestamp` taken from block headers.  The time when the block was created.
         tests:
           - not_null
 
       - name: block_hash
-        description: The block_hash taken from block headers.
+        description: The `block_hash` taken from block headers. A unique identifier for that block.
         tests:
           - not_null
 
       - name: tx_count
-        description: The tx_count taken from block headers.
+        description: The `tx_count` taken from block headers. The number of transactions included in the block.
         tests:
           - not_null
 
       - name: block_author
-        description: The block_author taken from block headers.
+        description: The `block_author` taken from block headers. Validators of the blockchain.
         tests:
           - not_null
 
@@ -62,7 +62,7 @@ models:
           - not_null
 
       - name: chunks
-        description: The chunks taken from block headers.
+        description: The chunks taken from block headers. Chunk is an aggregation of transactions which are executed within a particular shard.
         tests:
           - not_null