mainnet core models (#34)

* mainnet core models

* docs ref to base

* workflow cron comment

* weird error

* exclude goerli tests

* overview

---------

Co-authored-by: drethereum <trevor.wenokur@gmail.com>
This commit is contained in:
Austin 2023-07-25 10:56:24 -04:00 committed by GitHub
parent 74af425872
commit 05651b9f50
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
199 changed files with 4138 additions and 192 deletions

View File

@ -0,0 +1,47 @@
name: dbt_run_full_observability
run-name: dbt_run_full_observability
on:
workflow_dispatch:
schedule:
# Runs “At 06:00 on day-of-month 1.” (see https://crontab.guru)
- cron: '0 18 1 * *'
env:
DBT_PROFILES_DIR: ./
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod_2xl
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v1
with:
python-version: "3.7.x"
- name: install dependencies
run: |
pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
dbt deps
- name: Run DBT Jobs
run: |
dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m models/silver/_observability

View File

@ -4,8 +4,8 @@ run-name: dbt_run_scheduled
on:
workflow_dispatch:
schedule:
# Runs "every 2 hours" (see https://crontab.guru)
- cron: '0 */2 * * *'
# Runs "At minute 40, every hour" (see https://crontab.guru)
- cron: '40 * * * *'
env:
DBT_PROFILES_DIR: ./
@ -41,4 +41,4 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt run --exclude models/silver/eth_goerli models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql models/silver/streamline
dbt run --exclude models/silver/goerli models/silver/streamline models/bronze/eth_goerli models/silver/_observability

View File

@ -4,8 +4,8 @@ run-name: dbt_run_streamline_realtime
on:
workflow_dispatch:
schedule:
# Runs "at minute 5 and 35, every hour" (see https://crontab.guru)
- cron: '5,35 * * * *'
# Runs "at minute 25 and 55, every hour" (see https://crontab.guru)
- cron: '25,55 * * * *'
env:
DBT_PROFILES_DIR: ./

View File

@ -1,5 +1,5 @@
name: dbt_test
run-name: dbt_test
name: dbt_test_daily
run-name: dbt_test_daily
on:
workflow_dispatch:
@ -41,7 +41,7 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt test
dbt test --exclude tag:full_test tag:recent_test models/silver/goerli

View File

@ -1,12 +1,12 @@
name: dbt_run_scheduled_30_mins
run-name: dbt_run_scheduled_30_mins
name: dbt_test_intraday
run-name: dbt_test_intraday
on:
workflow_dispatch:
schedule:
# Runs "every 30 mins" (see https://crontab.guru)
- cron: '*/30 * * * *'
# Runs “At minute 5 past every 4th hour.” (see https://crontab.guru)
- cron: '5 */4 * * *'
env:
DBT_PROFILES_DIR: ./
@ -21,11 +21,11 @@ env:
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
environment:
name: workflow_prod
steps:
@ -37,8 +37,11 @@ jobs:
- name: install dependencies
run: |
pip3 install dbt-snowflake~=${{ vars.DBT_VERSION }} cli_passthrough requests click
pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click
dbt deps
- name: Run DBT Jobs
run: |
dbt run -m models/silver/eth_goerli models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql
dbt test -m tag:recent_test

View File

@ -37,6 +37,9 @@ on-run-end:
# Configuring models
# Full documentation: https://docs.getdbt.com/docs/configuring-models
models:
+copy_grants: true
# In this example config, we tell dbt to build all models in the example/ directory
# as tables. These settings can be overridden in the individual model files
# using the `{{ config(...) }}` macro.
@ -46,4 +49,5 @@ vars:
STREAMLINE_INVOKE_STREAMS: False
STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
UPDATE_UDFS_AND_SPS: False
UPDATE_SNOWFLAKE_TAGS: True
UPDATE_SNOWFLAKE_TAGS: True
OBSERV_FULL_TEST: False

View File

@ -51,7 +51,8 @@
'-32007',
'-32008',
'-32009',
'-32010'
'-32010',
'-32608'
)
)
{% endmacro %}
@ -109,7 +110,8 @@ WHERE
'-32007',
'-32008',
'-32009',
'-32010'
'-32010',
'-32608'
)
)
{% endmacro %}

View File

@ -0,0 +1,103 @@
{% macro missing_txs(
model
) %}
WITH txs_base AS (
SELECT
block_number AS base_block_number,
tx_hash AS base_tx_hash
FROM
{{ ref('test_silver__transactions_full') }}
),
model_name AS (
SELECT
block_number AS model_block_number,
tx_hash AS model_tx_hash
FROM
{{ model }}
)
SELECT
base_block_number,
base_tx_hash,
model_block_number,
model_tx_hash
FROM
txs_base
LEFT JOIN model_name
ON base_block_number = model_block_number
AND base_tx_hash = model_tx_hash
WHERE
(
model_tx_hash IS NULL
OR model_block_number IS NULL
)
{% endmacro %}
{% macro recent_missing_txs(
model
) %}
WITH txs_base AS (
SELECT
block_number AS base_block_number,
tx_hash AS base_tx_hash
FROM
{{ ref('test_silver__transactions_recent') }}
),
model_name AS (
SELECT
block_number AS model_block_number,
tx_hash AS model_tx_hash
FROM
{{ model }}
)
SELECT
base_block_number,
base_tx_hash,
model_block_number,
model_tx_hash
FROM
txs_base
LEFT JOIN model_name
ON base_block_number = model_block_number
AND base_tx_hash = model_tx_hash
WHERE
model_tx_hash IS NULL
OR model_block_number IS NULL
{% endmacro %}
{% macro missing_confirmed_txs(
model1,
model2
) %}
WITH txs_base AS (
SELECT
block_number AS base_block_number,
block_hash AS base_block_hash,
tx_hash AS base_tx_hash
FROM
{{ model1 }}
),
model_name AS (
SELECT
block_number AS model_block_number,
block_hash AS model_block_hash,
tx_hash AS model_tx_hash
FROM
{{ model2 }}
)
SELECT
DISTINCT base_block_number AS block_number
FROM
txs_base
LEFT JOIN model_name
ON base_block_number = model_block_number
AND base_tx_hash = model_tx_hash
AND base_block_hash = model_block_hash
WHERE
model_tx_hash IS NULL
AND model_block_number <= (
SELECT
MAX(base_block_number)
FROM
txs_base
)
{% endmacro %}

View File

@ -0,0 +1,5 @@
{% docs base_max_block %}
The max block on Base this batch relates to.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_min_block %}
The min block on Base this batch relates to.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_batch_size %}
Total Base Txs included within batch.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_batch_root %}
Root of batch, either for sumbission or state.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_block_no %}
The Ethereum block number that contained the batch.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_block_time %}
The timestamp of the Ethereum block that contained this batch.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_fee_scalar %}
This value covers the change in L1 gas price between the time the transaction is submitted and when it is published.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_gas_price %}
The gas price for L1 transactions when the transaction was processed.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_gas_used %}
The gas used on L1 to publish the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_state_batch %}
The batch index of when this block was included in the Ethereum state root. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_state_tx_hash %}
The L1 tx hash of when this block was included in the Ethereum state root. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_sub_batch %}
The batch index of when this block was submitted to L1. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_l1_sub_tx_hash %}
The L1 tx hash of when this block was submitted to L1. This column will be deprecated 8/7 and will be consolidated into a array column consisting of all L1 submission details.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_prev_total_elements %}
Confirmed blocks prior to this batch.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_block_header_json %}
This JSON column contains the block header details.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_blockchain %}
The blockchain on which transactions are being confirmed.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_blocks_hash %}
The hash of the block header for a given block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_blocks_nonce %}
Block nonce is a value used during mining to demonstrate proof of work for a given block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_blocks_table_doc %}
This table contains block level data for the Base Blockchain. This table can be used to analyze trends at a block level, for example gas fees vs. total transactions over time. For more information on EVM transactions, please see [Etherscan Resources](https://etherscan.io/directory/Learning_Resources/Ethereum) or [The Ethereum Organization](https://ethereum.org/en/developers/docs/blocks/)
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_difficulty %}
The effort required to mine the block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_extra_data %}
Any data included by the validator for a given block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_gas_limit %}
Total gas limit provided by all transactions in the block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_gas_used %}
Total gas used in the block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_miner %}
Miner who successfully added a given block to the blockchain.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_network %}
The network on the blockchain used by a transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_parent_hash %}
The hash of the block from which a given block is generated. Also known as the parent block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_receipts_root %}
The root of the state trie.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_sha3_uncles %}
The mechanism which Ethereum Javascript RLP encodes an empty string.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_size %}
Block size, which is determined by a given block's gas limit.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_total_difficulty %}
Total difficulty of the chain at a given block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_count %}
Total number of transactions within a block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_uncle_blocks %}
Uncle blocks occur when two blocks are mined and broadcasted at the same time, with the same block number. The block validated across the most nodes will be added to the primary chain, and the other one becomes an uncle block. Miners do receive rewards for uncle blocks.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_event_index %}
Event number within a transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_event_inputs %}
The decoded event inputs for a given event.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_event_name %}
The decoded event name for a given event.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_event_removed %}
Whether the event has been removed from the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_origin_sig %}
The function signature of this transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_log_id_events %}
This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the event occurred.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_logs_contract_address %}
The address interacted with for a given event.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_logs_contract_name %}
The name of the contract or token, where possible.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_logs_data %}
The un-decoded event data.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_logs_table_doc %}
This table contains flattened event logs from transactions on the Base Blockchain. Transactions may have multiple events, which are denoted by the event index for a transaction hash. Therefore, this table is unique on the combination of transaction hash and event index.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_logs_tx_hash %}
Transaction hash is a unique 66-character identifier that is generated when a transaction is executed. This field will not be unique in this table, as a given transaction can include multiple events.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_origin_from %}
The from address of this transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_origin_to %}
The to address of this transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_topics %}
The un-decoded event input topics.
{% enddocs %}

View File

@ -19,10 +19,11 @@ There is more information on how to use dbt docs in the last section of this doc
### Core Tables
**Fact Tables:**
- [goerli.fact_blocks](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_blocks)
- [goerli.fact_transactions](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_transactions)
- [goerli.fact_event_logs](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_event_logs)
- [goerli.fact_traces](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.goerli__fact_traces)
- [fact_blocks](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_blocks)
- [fact_transactions](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_transactions)
- [fact_event_logs](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_event_logs)
- [fact_traces](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_traces)
- [fact_token_transfers](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.core__fact_token_transfers)
## **Helpful User-Defined Functions (UDFs)**

View File

@ -0,0 +1,5 @@
{% docs base_block_number %}
Also known as block height. The block number, which indicates the length of the blockchain, increases after the addition of each new block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_block_timestamp %}
The date and time at which the block was produced.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_decimals %}
The number of decimal places this contract needs adjusted where token values exist. For example, use the decimal field to correctly transform raw amounts in ```fact_token_transfers```.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_from_address %}
The sending address of this transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_ingested_at %}
Internal column.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_symbol %}
The symbol belonging to the address of the token
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_to_address %}
The receiving address of this transaction. This can be a contract address.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_block_no %}
The block number of this transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_blocktime %}
The block timestamp of this transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_call_data %}
The raw JSON data for this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_from %}
The sending address of this trace. This is not necessarily the from address of the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_gas %}
The gas supplied for this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_gas_used %}
The gas used for this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_identifier %}
This field represents the position and type of the trace within the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_input %}
The input data for this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_output %}
The output data for this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_sub %}
The amount of nested sub traces for this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_table_doc %}
This table contains flattened trace data for internal contract calls on the Base Blockchain. Hex encoded fields can be decoded to integers by using `utils.udf_hex_to_int()`.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_to %}
The receiving address of this trace. This is not necessarily the to address of the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_tx_hash %}
The transaction hash for the trace. Please note, this is not necessarily unique in this table as transactions frequently have multiple traces.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_type %}
The type of internal transaction. Common trace types are `CALL`, `DELEGATECALL`, and `STATICCALL`.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_traces_value %}
The amount of ETH transferred in this trace.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_cumulative_gas_used %}
The total amount of gas used when this transaction was executed in the block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_block_hash %}
Block hash is a unique 66-character identifier that is generate when a block is produced.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_fee %}
Amount paid to validate the transaction in ETH.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_gas_limit %}
Maximum amount of gas allocated for the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_gas_price %}
Cost per unit of gas in Gwei.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_gas_used %}
Gas used by transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_hash %}
Transaction hash is a unique 66-character identifier that is generated when a transaction is executed.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_input_data %}
This column contains additional data for this transaction, and is commonly used as part of a contract interaction or as a message to the recipient.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_json %}
This JSON column contains the transaction details, including event logs.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_nonce %}
The number of transactions sent from a given address.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_origin_sig %}
The function signature of the contract call.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_position %}
The position of the transaction within the block.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_status %}
Status of the transaction.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_tx_table_doc %}
This table contains transaction level data for the Base Blockchain. Each transaction will have a unique transaction hash, along with transactions fees and a ETH value transferred when applicable. Transactions may be native ETH transfers or interactions with contract addresses. For more information, please see [The Ethereum Organization - Transactions](https://ethereum.org/en/developers/docs/transactions/)
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_value %}
The value transacted in ETH.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_eth_amount %}
ETH value transferred.
{% enddocs %}

View File

@ -0,0 +1,6 @@
{% docs base_eth_amount_usd %}
ETH value transferred, in USD.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_ez_eth_transfers_table_doc %}
This table contains all native ETH transfers, including equivalent USD amounts. The origin addresses correspond to the to and from addresses from the `fact_transactions` table. The `identifier` and `tx_hash` columns relate this table back to `fact_traces`, which contains more details on the transfers.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_ez_transfers_table_doc %}
This table will contain all events in the ```fact_token_transfers table```, along with joined columns such as token price, symbol, and decimals where possible that allow for easier analysis of token transfer events. Please note Native ETH transfers are not included here.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_log_id_transfers %}
This is the primary key for this table. This is a concatenation of the transaction hash and the event index at which the transfer event occurred. This field can be used to find more details on the event within the ```fact_event_logs``` table.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_eth_origin_from %}
The from address at the transaction level.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_eth_origin_to %}
The to address at the transaction level.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs base_transfer_amount %}
The decimal transformed amount for this token. Tokens without a decimal adjustment will be nulled out here.
{% enddocs %}

Some files were not shown because too many files have changed in this diff Show More