core models (#4)

* core models

* add type column

* gold models

* added docs workflow

* added gold transactions view and docs

* overview docs

* yml typo

* changed network name

---------

Co-authored-by: drethereum <trevor.wenokur@gmail.com>

.github/workflows/dbt_docs_update.yml

@@ -0,0 +1,68 @@
+name: docs_update
+
+on:
+  push:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  scheduled_run:
+    name: docs_update
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v1
+        with:
+          python-version: "3.7.x"
+
+      - name: install dependencies
+        run: |
+          pip install dbt-snowflake==${{ vars.DBT_VERSION }}
+          dbt deps
+      - name: checkout docs branch
+        run: |
+          git checkout -b docs origin/main
+          
+      - name: generate dbt docs
+        run: dbt docs generate -t prod
+
+      - name: move files to docs directory
+        run: |
+          mkdir -p ./docs
+          cp target/{catalog.json,manifest.json,index.html} docs/
+      - name: clean up target directory
+        run: dbt clean
+
+      - name: check for changes
+        run: git status
+
+      - name: stage changed files
+        run: git add .
+
+      - name: commit changed files
+        run: |
+          git config user.email "abc@xyz"
+          git config user.name "github-actions"
+          git commit -am "Auto-update docs"
+      - name: push changes to docs
+        run: |
+          git push -f --set-upstream origin docs

models/doc_descriptions/general/__overview__.md

@@ -0,0 +1,67 @@
+{% docs __overview__ %}
+
+# Welcome to the Flipside Crypto Base Models Documentation!
+
+## **What does this documentation cover?**
+The documentation included here details the design of the Base tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/base-models)
+
+## **How do I use these docs?**
+The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information.
+
+If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them.
+
+There is more information on how to use dbt docs in the last section of this document.
+
+## **Quick Links to Table Documentation**
+
+**Click on the links below to jump to the documentation for each schema.**
+
+### Core Tables (base.core)
+
+**Fact Tables:**
+- [testnet.fact_blocks](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.testnet__fact_blocks)
+- [testnet.fact_transactions](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.testnet__fact_transactions)
+
+## **Data Model Overview**
+
+The Base models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).**
+
+- Bronze: Data is loaded in from the source as a view
+- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here
+- Gold (or core): Final views and tables that are available publicly
+
+The dimension tables are sourced from a variety of on-chain and off-chain sources.
+
+Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data.
+
+NOTE: Base is currently operating in it's Testnet phase. Flipside will provide Mainnet data tables once Base Mainnet is deployed. 
+
+## **Using dbt docs**
+### Navigation
+
+You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project.
+
+### Database Tab
+
+This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database.
+
+### Graph Exploration
+
+You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.
+
+On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring.
+
+Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).
+
+Note that you can also right-click on models to interactively filter and explore the graph.
+
+
+### **More information**
+- [Flipside](https://flipsidecrypto.xyz/)
+- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover)
+- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials)
+- [Github](https://github.com/FlipsideCrypto/base-models)
+- [What is dbt?](https://docs.getdbt.com/docs/introduction)
+
+
+{% enddocs %}

models/gold/testnet/testnet__fact_blocks.sql

@@ -0,0 +1,27 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    A.block_number AS block_number,
+    block_timestamp,
+    network,
+    blockchain,
+    tx_count,
+    difficulty,
+    total_difficulty,
+    extra_data,
+    gas_limit,
+    gas_used,
+    HASH,
+    parent_hash,
+    receipts_root,
+    sha3_uncles,
+    SIZE,
+    uncle_blocks,
+    block_header_json
+FROM
+    {{ ref('silver_testnet__blocks') }} A
+

models/gold/testnet/testnet__fact_blocks.yml

@@ -0,0 +1,40 @@
+version: 2
+models:
+  - name: testnet__fact_blocks
+    description: This table contains block level data for the Base Blockchain Goerli Testnet. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on Base, please see [Base Documentation](https://docs.base.org/). For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/).
+
+    columns:
+      - name: BLOCK_NUMBER
+        description: Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block.
+      - name: BLOCK_TIMESTAMP
+        description: The date and time at which the block was produced.
+      - name: NETWORK
+        description: The network of the blockchain used by a transaction.
+      - name: BLOCKCHAIN
+        description: The blockchain on which transactions are being confirmed.
+      - name: TX_COUNT
+        description: Total number of transactions within a block.
+      - name: DIFFICULTY
+        description: The effort required to mine the block.
+      - name: TOTAL_DIFFICULTY
+        description: Total difficulty of the chain at a given block.
+      - name: EXTRA_DATA
+        description: Any data included by the validator for a given block.
+      - name: GAS_LIMIT
+        description: Total gas limit provided by all transactions in the block.
+      - name: GAS_USED
+        description: Total gas used in the block.
+      - name: HASH
+        description: The hash of the block header for a given block.
+      - name: PARENT_HASH
+        description: The hash of the block from which a given block is generated. Also known as the parent block.
+      - name: RECEIPTS_ROOT
+        description: The root of the state trie.
+      - name: SHA3_UNCLES
+        description: The mechanism which Ethereum Javascript RLP encodes an empty string.
+      - name: SIZE
+        description: Block size, which is determined by a given block's gas limit.
+      - name: UNCLE_BLOCKS
+        description: Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks.
+      - name: BLOCK_HEADER_JSON
+        description: This JSON column contains the block header details.

models/gold/testnet/testnet__fact_transactions.sql

@@ -0,0 +1,26 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true }
+) }}
+
+SELECT
+    A.block_number AS block_number,
+    block_timestamp,
+    tx_hash,
+    nonce,
+    POSITION,
+    origin_function_signature,
+    from_address,
+    to_address,
+    eth_value,
+    block_hash,
+    gas_price,
+    gas_limit,
+    input_data,
+    -- need receipts for tx status, gas used, L1 gas prices
+    tx_type,
+    is_system_tx,
+    tx_json
+FROM
+    {{ ref('silver_testnet__transactions') }} A

models/gold/testnet/testnet__fact_transactions.yml

@@ -0,0 +1,38 @@
+version: 2
+models:
+  - name: testnet__fact_transactions
+    description: This table contains transaction level data for the Base Blockchain Goerli Testnet. Each transaction will have a unique transaction hash, along with transactions fees and an ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information on Base, please see [Base Documentation](https://docs.base.org/). For more information on EVM transactions, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/)
+
+    columns:
+      - name: BLOCK_NUMBER
+        description: Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block.
+      - name: BLOCK_TIMESTAMP
+        description: The date and time at which the block was produced.
+      - name: TX_HASH
+        description: Transaction hash is a unique 66-character identifier that is generated when a transaction is executed.
+      - name: NONCE
+        description: The number of transactions sent from a given address.
+      - name: POSITION
+        description: The position of the transaction within the block.
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        description: The function signature of the contract call.
+      - name: FROM_ADDRESS
+        description: The sending address of this transaction.
+      - name: TO_ADDRESS
+        description: The receiving address of this transaction. This can be a contract address.
+      - name: ETH_VALUE
+        description: The value transacted in ETH.
+      - name: BLOCK_HASH
+        description: Block hash is a unique 66-character identifier that is generated when a block is produced.
+      - name: GAS_PRICE
+        description: Cost per unit of gas in Gwei.
+      - name: GAS_LIMIT
+        description: Maximum amount of gas allocated for the transaction.
+      - name: INPUT_DATA
+        description: This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient.
+      - name: TX_TYPE
+        description: This column indicates the transaction type.
+      - name: IS_SYSTEM_TX
+        description: This column indicates system transactions.
+      - name: TX_JSON
+        description: This JSON column contains the transaction details, including event logs.

models/silver/testnet/core/silver_testnet__blocks.sql

@@ -0,0 +1,69 @@
+{{ config (
+    materialized = "incremental",
+    unique_key = "block_number",
+    cluster_by = "block_timestamp::date",
+    merge_update_columns = ["block_number"]
+) }}
+
+SELECT
+    block_number,
+    TO_TIMESTAMP_NTZ(
+        ethereum.public.udf_hex_to_int(
+            TIMESTAMP :: STRING
+        )
+    ) AS block_timestamp,
+    'goerli' AS network,
+    'base' AS blockchain,
+    tx_count,
+    ethereum.public.udf_hex_to_int(
+        difficulty :: STRING
+    ) :: INTEGER AS difficulty,
+    ethereum.public.udf_hex_to_int(
+        totalDifficulty :: STRING
+    ) :: INTEGER AS total_difficulty,
+    extraData AS extra_data,
+    ethereum.public.udf_hex_to_int(
+        gasLimit :: STRING
+    ) :: INTEGER AS gas_limit,
+    ethereum.public.udf_hex_to_int(
+        gasUsed :: STRING
+    ) :: INTEGER AS gas_used,
+    block_hash AS HASH,
+    parentHash AS parent_hash,
+    receiptsRoot AS receipts_root,
+    sha3Uncles AS sha3_uncles,
+    ethereum.public.udf_hex_to_int(
+        SIZE :: STRING
+    ) :: INTEGER AS SIZE,
+    uncles AS uncle_blocks,
+    object_construct_keep_null(
+        'transactions',
+        transactions,
+        'transactions_root',
+        transactionsRoot,
+        'logs_bloom',
+        logsBloom,
+        'miner',
+        miner,
+        'mix_hash',
+        mixHash,
+        'nonce',
+        ethereum.public.udf_hex_to_int(
+            nonce :: STRING
+        ) :: INTEGER
+    ) AS block_header_json,
+    _inserted_timestamp
+FROM
+    {{ ref('silver_testnet__blocks_method') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/silver/testnet/core/silver_testnet__blocks.yml

@@ -0,0 +1,116 @@
+version: 2
+models:
+  - name: silver_testnet__blocks
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_NUMBER
+      - sequence_gaps:
+          column_name: BLOCK_NUMBER
+          where: BLOCK_TIMESTAMP < CURRENT_DATE - 1
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: NETWORK
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: BLOCKCHAIN
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: TX_COUNT
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: DIFFICULTY
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: TOTAL_DIFFICULTY
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: EXTRA_DATA
+        tests:
+          - not_null
+      - name: GAS_LIMIT
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER  
+                - FLOAT        
+      - name: GAS_USED
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: PARENT_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: RECEIPTS_ROOT
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: SHA3_UNCLES
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: SIZE
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+      - name: BLOCK_HEADER_JSON
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - OBJECT
+
+
+
+
+

models/silver/testnet/core/silver_testnet__transactions.sql

@@ -0,0 +1,98 @@
+{{ config (
+    materialized = "incremental",
+    unique_key = "tx_hash",
+    cluster_by = "BLOCK_TIMESTAMP::DATE",
+    merge_update_columns = ["tx_hash"]
+) }}
+
+SELECT
+    block_number,
+    TO_TIMESTAMP_NTZ(
+        ethereum.public.udf_hex_to_int(
+            block_timestamp :: STRING
+        )
+    ) AS block_timestamp,
+    tx_hash,
+    ethereum.public.udf_hex_to_int(
+        nonce :: STRING
+    ) :: INTEGER AS nonce,
+    ethereum.public.udf_hex_to_int(
+        POSITION :: STRING
+    ) :: INTEGER AS POSITION,
+    SUBSTR(
+        input,
+        1,
+        10
+    ) AS origin_function_signature,
+    from_address,
+    to_address,
+    ethereum.public.udf_hex_to_int(
+        eth_value :: STRING
+    ) / pow(
+        10,
+        18
+    ) AS eth_value,
+    block_hash,
+    COALESCE(
+        ethereum.public.udf_hex_to_int(
+            gas_price :: STRING
+        ) / pow(
+            10,
+            9
+        ),
+        0
+    ) AS gas_price,
+    ethereum.public.udf_hex_to_int(
+        gas_limit :: STRING
+    ) :: INTEGER AS gas_limit,
+    input AS input_data,
+    -- need receipts for tx status, gas used, L1 gas prices
+    ethereum.public.udf_hex_to_int(
+        tx_type :: STRING
+    ) :: INTEGER AS tx_type,
+    is_system_tx,
+    object_construct_keep_null(
+        'chain_ID',
+        ethereum.public.udf_hex_to_int(
+            chainID :: STRING
+        ) :: INTEGER,
+        'r',
+        r,
+        's',
+        s,
+        'v',
+        ethereum.public.udf_hex_to_int(
+            v :: STRING
+        ) :: INTEGER,
+        'access_list',
+        accesslist,
+        'max_priority_fee_per_gas',
+        ethereum.public.udf_hex_to_int(
+            max_priority_fee_per_gas :: STRING
+        ) :: INTEGER,
+        'max_fee_per_gas',
+        ethereum.public.udf_hex_to_int(
+            max_fee_per_gas :: STRING
+        ),
+        'mint',
+        ethereum.public.udf_hex_to_int(
+            mint :: STRING
+        ),
+        'source_hash',
+        sourcehash
+    ) AS tx_json,
+    _INSERTED_TIMESTAMP
+FROM
+    {{ ref('silver_testnet__tx_method') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(
+                _inserted_timestamp
+            )
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/silver/testnet/core/silver_testnet__transactions.yml

@@ -0,0 +1,121 @@
+version: 2
+models:
+  - name: silver_testnet__transactions
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_HASH
+      - sequence_gaps:
+          partition_by:
+            - BLOCK_NUMBER
+          column_name: POSITION
+          where: BLOCK_TIMESTAMP < CURRENT_DATE
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null  
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER    
+                - FLOAT    
+      - name: BLOCK_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: TX_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: NONCE
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER 
+                - FLOAT
+      - name: POSITION
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER 
+      - name: FROM_ADDRESS
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TO_ADDRESS
+        tests:
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+              where: TO_ADDRESS IS NOT NULL
+      - name: ETH_VALUE
+        tests:
+          - not_null
+      - name: BLOCK_HASH
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: GAS_PRICE
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER    
+                - FLOAT
+      - name: GAS_LIMIT
+        tests:
+          - not_null
+      - name: INPUT_DATA
+        tests:
+          - not_null
+      # - name: STATUS
+      #   tests:
+      #     - not_null
+      #     - dbt_expectations.expect_column_values_to_be_in_set:
+      #         value_set: ['SUCCESS', 'FAIL']
+      # - name: GAS_USED
+      #   tests:
+      #     - not_null
+      #     - dbt_expectations.expect_column_values_to_be_in_type_list:
+      #         column_type_list:
+      #           - NUMBER    
+      #           - FLOAT
+      # - name: CUMULATIVE_GAS_USED
+      #   tests:
+      #     - not_null
+      #     - dbt_expectations.expect_column_values_to_be_in_type_list:
+      #         column_type_list:
+      #           - NUMBER    
+      #           - FLOAT
+      # - name: TX_FEE
+      #   tests:
+      #     - not_null
+      #     - dbt_expectations.expect_column_values_to_be_in_type_list:
+      #         column_type_list:
+      #           - NUMBER    
+      #           - FLOAT
+      - name: TX_JSON 
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - OBJECT
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        tests:
+          - not_null
+
+

models/silver/testnet/methods/silver_testnet__blocks_method.sql

@@ -0,0 +1,103 @@
+ {{ config (
+    materialized = "incremental",
+    unique_key = "block_number",
+    cluster_by = "ROUND(block_number, -3)",
+    merge_update_columns = ["block_number"]
+) }}
+
+WITH meta AS (
+
+    SELECT
+        registered_on,
+        last_modified,
+        file_name
+    FROM
+        TABLE(
+            information_schema.external_table_files(
+                table_name => '{{ source( "bronze_streamline", "blocks") }}'
+            )
+        ) A
+
+{% if is_incremental() %}
+WHERE
+    LEAST(
+        registered_on,
+        last_modified
+    ) >= (
+        SELECT
+            COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
+        FROM
+            {{ this }})
+    ),
+    partitions AS (
+        SELECT
+            DISTINCT CAST(
+                SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER
+            ) AS _partition_by_block_number
+        FROM
+            meta
+    )
+{% else %}
+)
+{% endif %},
+base AS (
+    SELECT
+        block_number,
+        DATA :result AS response,
+        registered_on AS _inserted_timestamp
+    FROM
+        {{ source(
+            "bronze_streamline",
+            "blocks"
+        ) }}
+        t
+        JOIN meta b
+        ON b.file_name = metadata$filename -- add better partitioning
+    WHERE
+        DATA :error :code IS NULL
+        OR DATA :error :code NOT IN (
+            '-32000',
+            '-32001',
+            '-32002',
+            '-32003',
+            '-32004',
+            '-32005',
+            '-32006',
+            '-32007',
+            '-32008',
+            '-32009',
+            '-32010'
+        ) qualify(ROW_NUMBER() over (PARTITION BY block_number
+    ORDER BY
+        _inserted_timestamp DESC)) = 1
+)
+SELECT
+    block_number,
+    response :baseFeePerGas :: STRING AS baseFeePerGas,
+    response :difficulty :: STRING AS difficulty,
+    response :extraData :: STRING AS extraData,
+    response :gasLimit :: STRING AS gasLimit,
+    response :gasUsed :: STRING AS gasUsed,
+    response :hash :: STRING AS block_hash,
+    response :logsBloom :: STRING AS logsBloom,
+    response :miner :: STRING AS miner,
+    response :mixHash :: STRING AS mixHash,
+    response :nonce :: STRING AS nonce,
+    response :number :: STRING AS NUMBER,
+    response :parentHash :: STRING AS parentHash,
+    response :receiptsRoot :: STRING AS receiptsRoot,
+    response :sha3Uncles :: STRING AS sha3Uncles,
+    response :size :: STRING AS SIZE,
+    response :stateRoot :: STRING AS stateRoot,
+    response :timestamp :: STRING TIMESTAMP,
+    response :totalDifficulty :: STRING AS totalDifficulty,
+    response :transactions AS transactions,
+    ARRAY_SIZE(
+        response :transactions
+    ) AS tx_count,
+    response :transactionsRoot :: STRING AS transactionsRoot,
+    response :uncles AS uncles,
+    response,
+    _inserted_timestamp
+FROM
+    base

models/silver/testnet/methods/silver_testnet__blocks_method.yml

@@ -0,0 +1,24 @@
+version: 2
+models:
+  - name: silver_testnet__blocks_method
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - BLOCK_NUMBER
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null  
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER    
+                - FLOAT    
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+
+
+

models/silver/testnet/methods/silver_testnet__tx_method.sql

@@ -0,0 +1,118 @@
+{{ config (
+    materialized = "incremental",
+    unique_key = "tx_hash",
+    cluster_by = "ROUND(block_number, -3)",
+    merge_update_columns = ["tx_hash"]
+) }}
+
+WITH meta AS (
+
+    SELECT
+        registered_on,
+        last_modified,
+        file_name
+    FROM
+        TABLE(
+            information_schema.external_table_files(
+                table_name => '{{ source( "bronze_streamline", "transactions") }}'
+            )
+        ) A
+
+{% if is_incremental() %}
+WHERE
+    LEAST(
+        registered_on,
+        last_modified
+    ) >= (
+        SELECT
+            COALESCE(MAX(_INSERTED_TIMESTAMP), '1970-01-01' :: DATE) max_INSERTED_TIMESTAMP
+        FROM
+            {{ this }})
+    ),
+    partitions AS (
+        SELECT
+            DISTINCT CAST(
+                SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER
+            ) AS _partition_by_block_number
+        FROM
+            meta
+    )
+{% else %}
+)
+{% endif %},
+base AS (
+    SELECT
+        block_number,
+        DATA :result AS block_response,
+        DATA :result :transactions AS tx_response,
+        registered_on AS _inserted_timestamp
+    FROM
+        {{ source(
+            "bronze_streamline",
+            "transactions"
+        ) }}
+        t
+        JOIN meta b
+        ON b.file_name = metadata$filename --needs better partitioning once Ryan fixes his version of the model
+    WHERE
+        DATA :error :code IS NULL
+        OR DATA :error :code NOT IN (
+            '-32000',
+            '-32001',
+            '-32002',
+            '-32003',
+            '-32004',
+            '-32005',
+            '-32006',
+            '-32007',
+            '-32008',
+            '-32009',
+            '-32010'
+        ) qualify(ROW_NUMBER() over (PARTITION BY block_number
+    ORDER BY
+        _inserted_timestamp DESC)) = 1
+),
+flat AS (
+    SELECT
+        block_number,
+        block_response :timestamp :: STRING AS block_timestamp,
+        VALUE :hash :: STRING AS tx_hash,
+        VALUE :blockHash :: STRING AS block_hash,
+        VALUE :blockNumber :: STRING AS blockNumber,
+        VALUE :chainId :: STRING AS chainId,
+        VALUE :from :: STRING AS from_address,
+        VALUE :gas :: STRING AS gas_limit,
+        VALUE :gasPrice :: STRING AS gas_price,
+        VALUE :input :: STRING AS input,
+        CASE
+            WHEN VALUE :isSystemTx :: STRING = 'true' THEN TRUE
+            ELSE FALSE
+        END AS is_system_tx,
+        VALUE :maxFeePerGas :: STRING AS max_fee_per_gas,
+        VALUE :mint :: STRING AS mint,
+        VALUE :maxPriorityFeePerGas :: STRING AS max_priority_fee_per_gas,
+        VALUE :nonce :: STRING AS nonce,
+        VALUE :r :: STRING AS r,
+        VALUE :s :: STRING AS s,
+        VALUE :sourceHash :: STRING AS sourceHash,
+        VALUE :to :: STRING AS to_address,
+        VALUE :transactionIndex :: STRING POSITION,
+        VALUE :type :: STRING AS tx_type,
+        VALUE :v :: STRING AS v,
+        VALUE :value :: STRING AS eth_value,
+        VALUE :accessList AS accessList,
+        VALUE,
+        block_response,
+        _inserted_timestamp
+    FROM
+        base,
+        LATERAL FLATTEN (
+            input => tx_response
+        )
+)
+SELECT
+    *
+FROM
+    flat qualify(ROW_NUMBER() over (PARTITION BY tx_hash
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/silver/testnet/methods/silver_testnet__tx_method.yml

@@ -0,0 +1,24 @@
+version: 2
+models:
+  - name: silver_testnet__tx_method
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - TX_HASH
+    columns:
+      - name: BLOCK_NUMBER
+        tests:
+          - not_null  
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER    
+                - FLOAT    
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+
+
+