flip/arbitrum-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/arbitrum-models.git synced 2026-02-06 13:36:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

An 4916/new columns (#174)

Browse Source 
* stash

* stash

* name

* command

* nft

* dates

* event index

* typos

* inserted

---------

Co-authored-by: sam <sam@flipsidecrypto.com>
This commit is contained in:
Austin 2023-12-04 15:17:58 -05:00 committed by GitHub
parent 236f28f8f1
commit f5e502b7ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
68 changed files with 982 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dbt_project.yml
View File

@ -26,6 +26,7 @@ clean-targets: # directories to be removed by `dbt clean`
models:
+copy_grants: true
+on_schema_change: "append_new_columns"
tests:
+store_failures: true # all tests

15
models/doc_descriptions/general/deprecation.md
View File

@ -1,12 +1,11 @@
{% docs deprecation %}
{% docs internal_column %}
Deprecating soon: This is a notice that we're only removing the below columns. Please migrate queries using these columns to `fact_decoded_event_logs`, `ez_decoded_event_logs` or use manual parsing of topics and data. The following columns will be deprecated on 7/01/23:
Deprecated. This column is no longer used. Please remove from your query by Jan. 31 2024.
`Fact_event_logs` Columns:
- `event_name`
- `event_inputs`
- `contract_name`
{% enddocs %}
{% docs amount_deprecation %}
This column is being deprecated for standardization purposes on Jan. 31 2024. Please use the equivalent column without the native asset prefix. For example, use `amount` instead of `eth_amount`.
`Fact_transactions` Columns:
- `tx_json`
{% enddocs %}

17
models/doc_descriptions/general/export_columns.md Normal file
View File

@ -0,0 +1,17 @@
{% docs pk %}
The unique identifier for each row in the table.
{% enddocs %}
{% docs inserted_timestamp %}
The utc timestamp at which the row was inserted into the table.
{% enddocs %}
{% docs modified_timestamp %}
The utc timestamp at which the row was last modified.
{% enddocs %}

5
models/doc_descriptions/nft/nft_intra_event_index.md Normal file
View File

@ -0,0 +1,5 @@
{% docs nft_intra_event_index %}
The order of events within a single event index. This is primarily used for ERC1155 NFT batch transfer events.
{% enddocs %}

5
models/doc_descriptions/traces/arb_traces_index.md Normal file
View File

@ -0,0 +1,5 @@
{% docs arb_trace_index %}
The index of the trace within the transaction.
{% enddocs %}

13
models/gold/core/core__dim_contracts.sql
View File

@ -12,7 +12,18 @@ SELECT
c0.block_number AS created_block_number,
c0.block_timestamp AS created_block_timestamp,
c0.tx_hash AS created_tx_hash,
c0.creator_address AS creator_address
c0.creator_address AS creator_address,
c0.created_contracts_id AS dim_contracts_id,
GREATEST(
c0.inserted_timestamp,
c1.inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
GREATEST(
c0.modified_timestamp,
c1.modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__created_contracts') }}
c0

6
models/gold/core/core__dim_contracts.yml
View File

@ -20,3 +20,9 @@ models:
description: 'The transaction hash when the contract was created'
- name: CREATOR_ADDRESS
description: 'The address of the creator of the contract'
- name: DIM_CONTRACTS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/core/core__dim_labels.sql
View File

@ -11,6 +11,20 @@ SELECT
address_name,
label_type,
label_subtype,
project_name
project_name,
COALESCE (
labels_id,
{{ dbt_utils.generate_surrogate_key(
['address']
) }}
) AS dim_labels_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__labels') }}

7
models/gold/core/core__dim_labels.yml
View File

@ -52,4 +52,9 @@ models:
column_type_list:
- STRING
- VARCHAR
- name: DIM_LABELS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

15
models/gold/core/core__ez_decoded_event_logs.sql
View File

@ -20,7 +20,16 @@ SELECT
topics,
DATA,
event_removed,
tx_status
tx_status,
COALESCE (
decoded_logs_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }}
) AS ez_decoded_event_logs_id,
GREATEST(COALESCE(l.inserted_timestamp, '2000-01-01'), COALESCE(C.inserted_timestamp, '2000-01-01')) AS inserted_timestamp,
GREATEST(COALESCE(l.modified_timestamp, '2000-01-01'), COALESCE(C.modified_timestamp, '2000-01-01')) AS modified_timestamp
FROM
{{ ref('silver__decoded_logs') }}
LEFT JOIN {{ ref('silver__contracts') }} using (contract_address)
{{ ref('silver__decoded_logs') }}
l
LEFT JOIN {{ ref('silver__contracts') }} C USING (contract_address)

8
models/gold/core/core__ez_decoded_event_logs.yml
View File

@ -56,4 +56,10 @@ models:
- name: EVENT_REMOVED
description: '{{ doc("arb_event_removed") }}'
- name: TX_STATUS
description: '{{ doc("arb_tx_status") }}'
description: '{{ doc("arb_tx_status") }}'
- name: EZ_DECODED_EVENT_LOGS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

98
models/gold/core/core__ez_eth_transfers.sql
View File

@ -1,104 +1,26 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
unique_key = 'block_number',
cluster_by = ['block_timestamp::DATE'],
tags = ['core','non_realtime','reorg'],
materialized = 'view',
persist_docs ={ "relation": true,
"columns": true }
) }}
WITH eth_base AS (
SELECT
tx_hash,
block_number,
block_timestamp,
identifier,
from_address,
to_address,
eth_value,
_call_id,
_inserted_timestamp,
eth_value_precise_raw,
eth_value_precise,
tx_position,
trace_index
FROM
{{ ref('silver__traces') }}
WHERE
eth_value > 0
AND tx_status = 'SUCCESS'
AND trace_status = 'SUCCESS'
AND TYPE NOT IN (
'DELEGATECALL',
'STATICCALL'
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '72 hours'
FROM
{{ this }}
)
{% endif %}
),
tx_table AS (
SELECT
block_number,
tx_hash,
from_address AS origin_from_address,
to_address AS origin_to_address,
origin_function_signature
FROM
{{ ref('silver__transactions') }}
WHERE
tx_hash IN (
SELECT
DISTINCT tx_hash
FROM
eth_base
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '72 hours'
FROM
{{ this }}
)
{% endif %}
)
SELECT
tx_hash AS tx_hash,
block_number AS block_number,
block_timestamp AS block_timestamp,
identifier AS identifier,
tx_hash,
block_number,
block_timestamp,
identifier,
origin_from_address,
origin_to_address,
origin_function_signature,
from_address AS eth_from_address,
to_address AS eth_to_address,
eth_value AS amount,
eth_value_precise_raw AS amount_precise_raw,
eth_value_precise AS amount_precise,
ROUND(
eth_value * price,
2
) AS amount_usd,
amount,
amount_precise_raw,
amount_precise,
amount_usd,
_call_id,
_inserted_timestamp,
tx_position,
trace_index
FROM
eth_base A
LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }}
ON DATE_TRUNC(
'hour',
A.block_timestamp
) = HOUR
JOIN tx_table USING (
tx_hash,
block_number
)
{{ ref('silver__native_transfers') }}

2
models/gold/core/core__ez_eth_transfers.yml
View File

@ -1,7 +1,7 @@
version: 2
models:
- name: core__ez_eth_transfers
description: '{{ doc("arb_ez_eth_transfers_table_doc") }}'
description: 'Deprecating soon! Migrate to `core.ez_native_transfers` by Jan. 31 2024.'
columns:
- name: BLOCK_NUMBER

38
models/gold/core/core__ez_native_transfers.sql Normal file
View File

@ -0,0 +1,38 @@
{{ config(
materialized = 'view',
persist_docs ={ "relation": true,
"columns": true }
) }}
SELECT
tx_hash,
block_number,
block_timestamp,
tx_position,
trace_index,
identifier,
origin_from_address,
origin_to_address,
origin_function_signature,
from_address,
to_address,
amount,
amount_precise_raw,
amount_precise,
amount_usd,
COALESCE (
native_transfers_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }}
) AS ez_native_transfers_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__native_transfers') }}

42
models/gold/core/core__ez_native_transfers.yml Normal file
View File

@ -0,0 +1,42 @@
version: 2
models:
- name: core__ez_native_transfers
description: '{{ doc("arb_ez_eth_transfers_table_doc") }}'
columns:
- name: TX_HASH
description: '{{ doc("arb_transfer_tx_hash") }}'
- name: BLOCK_NUMBER
description: '{{ doc("arb_block_number") }}'
- name: BLOCK_TIMESTAMP
description: '{{ doc("arb_block_timestamp") }}'
- name: TX_POSITION
description: '{{ doc("arb_tx_position") }}'
- name: TRACE_INDEX
description: '{{ doc("arb_trace_index") }}'
- name: IDENTIFIER
description: '{{ doc("arb_traces_identifier") }}'
- name: ORIGIN_FROM_ADDRESS
description: '{{ doc("arb_origin_from") }}'
- name: ORIGIN_TO_ADDRESS
description: '{{ doc("arb_origin_to") }}'
- name: ORIGIN_FUNCTION_SIGNATURE
description: '{{ doc("arb_origin_sig") }}'
- name: FROM_ADDRESS
description: '{{ doc("arb_transfer_from_address") }}'
- name: TO_ADDRESS
description: '{{ doc("arb_transfer_to_address") }}'
- name: AMOUNT
description: '{{ doc("arb_eth_amount") }}'
- name: AMOUNT_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
- name: AMOUNT_PRECISE
description: '{{ doc("precise_amount_adjusted") }}'
- name: AMOUNT_USD
description: '{{ doc("arb_eth_amount_usd") }}'
- name: EZ_NATIVE_TRANSFERS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/core/core__ez_token_transfers.sql
View File

@ -26,6 +26,20 @@ SELECT
has_decimal,
has_price,
_log_id,
_inserted_timestamp
_inserted_timestamp,
COALESCE (
transfers_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }}
) AS ez_token_transfers_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__transfers') }}

10
models/gold/core/core__ez_token_transfers.yml
View File

@ -47,4 +47,12 @@ models:
- name: HAS_PRICE
description: '{{ doc("arb_transfer_has_price") }}'
- name: _LOG_ID
description: '{{ doc("arb_log_id_transfers") }}'
description: '{{ doc("internal_column") }}'
- name: _INSERTED_TIMESTAMP
description: '{{ doc("internal_column") }}'
- name: EZ_TOKEN_TRANSFERS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

32
models/gold/core/core__fact_blocks.sql
View File

@ -60,7 +60,33 @@ SELECT
transactions_root,
'uncles',
uncles
) AS block_header_json
) AS block_header_json,
COALESCE (
blocks_id,
{{ dbt_utils.generate_surrogate_key(
['a.block_number']
) }}
) AS fact_blocks_id,
GREATEST(
COALESCE(
A.inserted_timestamp,
'2000-01-01'
),
COALESCE(
d.inserted_timestamp,
'2000-01-01'
)
) AS inserted_timestamp,
GREATEST(
COALESCE(
A.modified_timestamp,
'2000-01-01'
),
COALESCE(
d.modified_timestamp,
'2000-01-01'
)
) AS modified_timestamp
FROM
{{ ref('silver__blocks') }}
LEFT JOIN {{ ref('silver__tx_count') }} USING (block_number)
{{ ref('silver__blocks') }} a
LEFT JOIN {{ ref('silver__tx_count') }} d USING (block_number)

8
models/gold/core/core__fact_blocks.yml
View File

@ -37,4 +37,10 @@ models:
- name: UNCLE_BLOCKS
description: '{{ doc("arb_uncle_blocks") }}'
- name: BLOCK_HEADER_JSON
description: '{{ doc("arb_block_header_json") }}'
description: '{{ doc("arb_block_header_json") }}'
- name: FACT_BLOCKS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/core/core__fact_decoded_event_logs.sql
View File

@ -12,6 +12,20 @@ SELECT
contract_address,
event_name,
decoded_flat AS decoded_log,
decoded_data AS full_decoded_log
decoded_data AS full_decoded_log,
COALESCE (
decoded_logs_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }}
) AS fact_decoded_event_logs_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__decoded_logs') }}

6
models/gold/core/core__fact_decoded_event_logs.yml
View File

@ -42,3 +42,9 @@ models:
description: 'The flattened decoded log, where the keys are the names of the event parameters, and the values are the values of the event parameters.'
- name: FULL_DECODED_LOG
description: 'The full decoded log, including the event name, the event parameters, and the data type of the event parameters.'
- name: FACT_DECODED_EVENT_LOGS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/core/core__fact_event_logs.sql
View File

@ -17,6 +17,20 @@ SELECT
DATA,
event_removed,
tx_status,
_log_id
_log_id,
COALESCE (
logs_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }}
) AS fact_event_logs_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__logs') }}

8
models/gold/core/core__fact_event_logs.yml
View File

@ -21,7 +21,7 @@ models:
- name: EVENT_REMOVED
description: '{{ doc("arb_event_removed") }}'
- name: _LOG_ID
description: '{{ doc("arb_log_id_events") }}'
description: '{{ doc("internal_column") }}'
- name: TX_STATUS
description: '{{ doc("arb_tx_status") }}'
- name: ORIGIN_FUNCTION_SIGNATURE
@ -30,3 +30,9 @@ models:
description: '{{ doc("arb_origin_from") }}'
- name: ORIGIN_TO_ADDRESS
description: '{{ doc("arb_origin_to") }}'
- name: FACT_EVENT_LOGS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/core/core__fact_token_transfers.sql
View File

@ -17,6 +17,20 @@ SELECT
to_address,
raw_amount,
raw_amount_precise,
_log_id
_log_id,
COALESCE (
transfers_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }}
) AS fact_token_transfers_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__transfers') }}

8
models/gold/core/core__fact_token_transfers.yml
View File

@ -29,4 +29,10 @@ models:
- name: RAW_AMOUNT_PRECISE
description: '{{ doc("arb_transfer_raw_amount_precise") }}'
- name: _LOG_ID
description: '{{ doc("arb_log_id_transfers") }}'
description: '{{ doc("internal_column") }}'
- name: FACT_TOKEN_TRANSFERS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

25
models/gold/core/core__fact_traces.sql
View File

@ -10,9 +10,9 @@ SELECT
block_timestamp,
from_address,
to_address,
eth_value,
eth_value_precise_raw,
eth_value_precise,
eth_value AS VALUE,
eth_value_precise_raw AS value_precise_raw,
eth_value_precise AS value_precise,
gas,
gas_used,
input,
@ -26,6 +26,23 @@ SELECT
error_reason,
trace_index,
before_evm_transfers,
after_evm_transfers
after_evm_transfers,
COALESCE (
traces_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }}
) AS fact_traces_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp,
eth_value,
eth_value_precise_raw,
eth_value_precise
FROM
{{ ref('silver__traces') }}

20
models/gold/core/core__fact_traces.yml
View File

@ -15,10 +15,16 @@ models:
- name: TO_ADDRESS
description: '{{ doc("arb_traces_to") }}'
- name: ETH_VALUE
description: '{{ doc("arb_traces_value") }}'
description: '{{ doc("amount_deprecation") }}'
- name: ETH_VALUE_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
description: '{{ doc("amount_deprecation") }}'
- name: ETH_VALUE_PRECISE
description: '{{ doc("amount_deprecation") }}'
- name: VALUE
description: '{{ doc("arb_traces_value") }}'
- name: VALUE_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
- name: VALUE_PRECISE
description: '{{ doc("precise_amount_adjusted") }}'
- name: GAS
description: '{{ doc("arb_traces_gas") }}'
@ -48,7 +54,9 @@ models:
description: The state of all ETH transfers to be executed for a given transaction.
- name: AFTER_EVM_TRANSFERS
description: The state of all ETH transfers executed for a given transaction.
- name: FACT_TRACES_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

84
models/gold/core/core__fact_transactions.sql
View File

@ -14,74 +14,42 @@ SELECT
origin_function_signature,
from_address,
to_address,
eth_value,
eth_value_precise_raw,
eth_value_precise,
VALUE,
value_precise_raw,
value_precise,
tx_fee,
tx_fee_precise,
gas_price_bid,
gas_price_paid,
gas_limit,
gas_price AS gas_price_bid,
effective_gas_price AS gas_price_paid,
gas AS gas_limit,
gas_used,
cumulative_gas_used,
max_fee_per_gas,
max_priority_fee_per_gas,
input_data,
status,
tx_status AS status,
r,
s,
v,
tx_type,
l1_block_number,
gas_used_for_l1
gas_used_for_l1,
COALESCE (
transactions_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }}
) AS fact_transactions_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp,
VALUE AS eth_value,
value_precise_raw AS eth_value_precise_raw,
value_precise AS eth_value_precise
FROM
(
SELECT
block_number,
block_timestamp,
block_hash,
tx_hash,
nonce,
POSITION,
origin_function_signature,
from_address,
to_address,
VALUE AS eth_value,
tx_fee,
tx_fee_precise,
gas_price AS gas_price_bid,
effective_gas_price AS gas_price_paid,
gas AS gas_limit,
gas_used,
cumulative_gas_used,
max_fee_per_gas,
max_priority_fee_per_gas,
input_data,
tx_status AS status,
r,
s,
v,
tx_type,
l1_block_number,
gas_used_for_l1,
CASE
WHEN tx_hash <> '0xefc4b1845f162fd61b496766c69fc0da9ee1317f0153efa30b3cb30d8f7884ba' THEN to_varchar(
TO_NUMBER(REPLACE(DATA :value :: STRING, '0x'), REPEAT('X', LENGTH(REPLACE(DATA :value :: STRING, '0x'))))
)
ELSE NULL
END AS eth_value_precise_raw,
IFF(LENGTH(eth_value_precise_raw) > 18, LEFT(eth_value_precise_raw, LENGTH(eth_value_precise_raw) - 18) || '.' || RIGHT(eth_value_precise_raw, 18), '0.' || LPAD(eth_value_precise_raw, 18, '0')) AS rough_conversion,
IFF(
POSITION(
'.000000000000000000' IN rough_conversion
) > 0,
LEFT(rough_conversion, LENGTH(rough_conversion) - 19),
REGEXP_REPLACE(
rough_conversion,
'0*$',
''
)
) AS eth_value_precise
FROM
{{ ref('silver__transactions') }}
)
{{ ref('silver__transactions') }}

18
models/gold/core/core__fact_transactions.yml
View File

@ -21,10 +21,16 @@ models:
- name: TO_ADDRESS
description: '{{ doc("arb_to_address") }}'
- name: ETH_VALUE
description: '{{ doc("arb_value") }}'
description: '{{ doc("amount_deprecation") }}'
- name: ETH_VALUE_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
description: '{{ doc("amount_deprecation") }}'
- name: ETH_VALUE_PRECISE
description: '{{ doc("amount_deprecation") }}'
- name: VALUE
description: '{{ doc("arb_value") }}'
- name: VALUE_PRECISE_RAW
description: '{{ doc("precise_amount_unadjusted") }}'
- name: VALUE_PRECISE
description: '{{ doc("precise_amount_adjusted") }}'
- name: TX_FEE
description: '{{ doc("arb_tx_fee") }}'
@ -61,4 +67,10 @@ models:
- name: L1_BLOCK_NUMBER
description: The block number of the transaction on Ethereum Mainnet.
- name: GAS_USED_FOR_L1
description: The gas used by the transaction on Ethereum Mainnet.
description: The gas used by the transaction on Ethereum Mainnet.
- name: FACT_TRANSACTIONS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/defi/defi__dim_dex_liquidity_pools.sql
View File

@ -22,6 +22,20 @@ SELECT
pool_name,
tokens,
symbols,
decimals
decimals,
COALESCE (
complete_dex_liquidity_pools_id,
{{ dbt_utils.generate_surrogate_key(
['block_number','platform','version']
) }}
) AS dim_dex_liquidity_pools_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver_dex__complete_dex_liquidity_pools') }}

8
models/gold/defi/defi__dim_dex_liquidity_pools.yml
View File

@ -23,4 +23,10 @@ models:
- name: SYMBOLS
description: '{{ doc("eth_dex_lp_symbols") }}'
- name: DECIMALS
description: '{{ doc("eth_dex_lp_decimals") }}'
description: '{{ doc("eth_dex_lp_decimals") }}'
- name: DIM_DEX_LIQUIDITY_POOLS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/defi/defi__ez_dex_swaps.sql
View File

@ -36,5 +36,19 @@ SELECT
token_out,
symbol_in,
symbol_out,
_log_id
_log_id,
COALESCE (
complete_dex_swaps_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash','event_index']
) }}
) AS ez_dex_swaps_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM {{ ref('silver_dex__complete_dex_swaps') }}

9
models/gold/defi/defi__ez_dex_swaps.yml
View File

@ -39,7 +39,7 @@ models:
- name: EVENT_INDEX
description: '{{ doc("arb_event_index") }}'
- name: _LOG_ID
description: '{{ doc("arb_log_id_events") }}'
description: '{{ doc("internal_column") }}'
- name: ORIGIN_FUNCTION_SIGNATURE
description: '{{ doc("arb_tx_origin_sig") }}'
- name: ORIGIN_FROM_ADDRESS
@ -50,4 +50,9 @@ models:
description: '{{ doc("eth_dex_swaps_amount_in_unadj") }}'
- name: AMOUNT_OUT_UNADJ
description: '{{ doc("eth_dex_swaps_amount_out_unadj") }}'
- name: EZ_DEX_SWAPS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

17
models/gold/nft/nft__ez_nft_sales.sql
View File

@ -15,6 +15,7 @@ SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -37,6 +38,20 @@ SELECT
creator_fee_usd,
origin_from_address,
origin_to_address,
origin_function_signature
origin_function_signature,
COALESCE (
complete_nft_sales_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index', 'nft_address','tokenId','platform_exchange_version']
) }}
) AS ez_nft_sales_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__complete_nft_sales') }}

10
models/gold/nft/nft__ez_nft_sales.yml
View File

@ -9,7 +9,9 @@ models:
- name: BLOCK_TIMESTAMP
description: '{{ doc("nft_blocktime") }}'
- name: TX_HASH
description: '{{ doc("nft_tx_hash") }}'
description: '{{ doc("nft_tx_hash") }}'
- name: EVENT_INDEX
description: '{{ doc("nft_event_index") }}'
- name: EVENT_TYPE
description: '{{ doc("nft_event_type") }}'
- name: PLATFORM_ADDRESS
@ -54,3 +56,9 @@ models:
description: '{{ doc("nft_origin_to") }}'
- name: ORIGIN_FUNCTION_SIGNATURE
description: '{{ doc("nft_origin_sig") }}'
- name: EZ_NFT_SALES_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

17
models/gold/nft/nft__ez_nft_transfers.sql
View File

@ -13,13 +13,28 @@ SELECT
block_number,
tx_hash,
event_index,
intra_event_index,
event_type,
contract_address AS nft_address,
project_name,
from_address AS nft_from_address,
to_address AS nft_to_address,
tokenId,
erc1155_value
erc1155_value,
COALESCE (
nft_transfers_id,
{{ dbt_utils.generate_surrogate_key(
['tx_hash','event_index','intra_event_index']
) }}
) AS ez_nft_transfers_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__nft_transfers') }}

9
models/gold/nft/nft__ez_nft_transfers.yml
View File

@ -12,6 +12,8 @@ models:
description: '{{ doc("nft_tx_hash") }}'
- name: EVENT_INDEX
description: '{{ doc("nft_event_index") }}'
- name: INTRA_EVENT_INDEX
description: '{{ doc("nft_intra_event_index") }}'
- name: EVENT_TYPE
description: '{{ doc("nft_event_type") }}'
- name: NFT_ADDRESS
@ -26,4 +28,9 @@ models:
description: '{{ doc("nft_tokenid") }}'
- name: ERC1155_VALUE
description: '{{ doc("nft_erc1155_value") }}'
- name: EZ_NFT_TRANSFERS_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/price/price__dim_asset_metadata.sql
View File

@ -10,6 +10,20 @@ SELECT
symbol,
NAME,
decimals,
provider
provider,
COALESCE (
asset_metadata_all_providers_id,
{{ dbt_utils.generate_surrogate_key(
['token_address','symbol','id','provider']
) }}
) AS dim_asset_metadata_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__asset_metadata_all_providers') }}

6
models/gold/price/price__dim_asset_metadata.yml
View File

@ -23,3 +23,9 @@ models:
description: The specific address representing the asset in a specific platform.
- name: DECIMALS
description: The number of decimal places the token needs adjusted where token values exist.
- name: DIM_ASSET_METADATA_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/price/price__ez_asset_metadata.sql
View File

@ -9,6 +9,20 @@ SELECT
id,
symbol,
NAME,
decimals
decimals,
COALESCE (
asset_metadata_priority_id,
{{ dbt_utils.generate_surrogate_key(
['token_address']
) }}
) AS ez_asset_metadata_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__asset_metadata_priority') }}

6
models/gold/price/price__ez_asset_metadata.yml
View File

@ -18,3 +18,9 @@ models:
description: The specific address representing the asset in a specific platform.
- name: DECIMALS
description: The number of decimal places the token needs adjusted where token values exist.
- name: EZ_ASSET_METADATA_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/price/price__ez_hourly_token_prices.sql
View File

@ -10,6 +10,20 @@ SELECT
symbol,
decimals,
price,
is_imputed
is_imputed,
COALESCE (
hourly_prices_priority_id,
{{ dbt_utils.generate_surrogate_key(
['token_address', 'hour']
) }}
) AS ez_hourly_token_prices_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__hourly_prices_priority') }}

8
models/gold/price/price__ez_hourly_token_prices.yml
View File

@ -20,4 +20,10 @@ models:
- name: PRICE
description: Closing price of the recorded hour in USD
- name: IS_IMPUTED
description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens)
description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens)
- name: EZ_HOURLY_TOKEN_PRICES_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

16
models/gold/price/price__fact_hourly_token_prices.sql
View File

@ -9,6 +9,20 @@ SELECT
token_address,
price,
is_imputed,
provider
provider,
COALESCE (
hourly_prices_all_providers_id,
{{ dbt_utils.generate_surrogate_key(
['token_address', 'hour', 'provider']
) }}
) AS fact_hourly_token_prices_id,
COALESCE(
inserted_timestamp,
'2000-01-01'
) AS inserted_timestamp,
COALESCE(
modified_timestamp,
'2000-01-01'
) AS modified_timestamp
FROM
{{ ref('silver__hourly_prices_all_providers') }}

8
models/gold/price/price__fact_hourly_token_prices.yml
View File

@ -13,4 +13,10 @@ models:
- name: PRICE
description: Closing price of the recorded hour in USD
- name: IS_IMPUTED
description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens)
description: Whether the price was imputed from an earlier record (generally used for low trade volume tokens)
- name: FACT_HOURLY_TOKEN_PRICES_ID
description: '{{ doc("pk") }}'
- name: INSERTED_TIMESTAMP
description: '{{ doc("inserted_timestamp") }}'
- name: MODIFIED_TIMESTAMP
description: '{{ doc("modified_timestamp") }}'

12
models/silver/core/silver__blocks.sql
View File

@ -3,7 +3,9 @@
materialized = 'incremental',
unique_key = "block_number",
cluster_by = "block_timestamp::date",
tags = ['core','non_realtime']
tags = ['core','non_realtime'],
merge_exclude_columns = ["inserted_timestamp"],
full_refresh = false
) }}
SELECT
@ -49,7 +51,13 @@ SELECT
) :: INT AS total_difficulty,
DATA :transactionsRoot :: STRING AS transactions_root,
DATA :uncles AS uncles,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['block_number']
) }} AS blocks_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{% if is_incremental() %}

9
models/silver/core/silver__contracts.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = 'contract_address',
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -81,7 +82,13 @@ token_names AS (
token_name,
token_decimals :: INTEGER AS token_decimals,
token_symbol,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['c1.contract_address']
) }} AS contracts_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
contracts c1
LEFT JOIN token_names

9
models/silver/core/silver__created_contracts.sql
View File

@ -1,6 +1,7 @@
{{ config (
materialized = "incremental",
unique_key = "created_contract_address",
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -11,7 +12,13 @@ SELECT
to_address AS created_contract_address,
from_address AS creator_address,
input AS created_contract_input,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['to_address']
) }} AS created_contracts_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('silver__traces') }}
WHERE

17
models/silver/core/silver__decoded_logs.sql
View File

@ -6,6 +6,7 @@
incremental_predicates = ["dynamic_range", "block_number"],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
full_refresh = false,
merge_exclude_columns = ["inserted_timestamp"],
tags = ['decoded_logs','reorg']
) }}
@ -191,7 +192,13 @@ SELECT
DATA,
event_removed,
tx_status,
is_pending
is_pending,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }} AS decoded_logs_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
new_records
@ -216,7 +223,13 @@ SELECT
DATA,
event_removed,
tx_status,
is_pending
is_pending,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }} AS decoded_logs_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
missing_data
{% endif %}

8
models/silver/core/silver__logs.sql
View File

@ -176,7 +176,13 @@ FROM
{% endif %}
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }} AS logs_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL qualify(ROW_NUMBER() over (PARTITION BY block_number, event_index
ORDER BY

109
models/silver/core/silver__native_transfers.sql Normal file
View File

@ -0,0 +1,109 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
unique_key = 'block_number',
cluster_by = ['block_timestamp::DATE'],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
tags = ['core','non_realtime','reorg']
) }}
WITH eth_base AS (
SELECT
tx_hash,
block_number,
block_timestamp,
identifier,
from_address,
to_address,
eth_value,
_call_id,
_inserted_timestamp,
eth_value_precise_raw,
eth_value_precise,
tx_position,
trace_index
FROM
{{ ref('silver__traces') }}
WHERE
eth_value > 0
AND tx_status = 'SUCCESS'
AND trace_status = 'SUCCESS'
AND TYPE NOT IN (
'DELEGATECALL',
'STATICCALL'
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '72 hours'
FROM
{{ this }}
)
{% endif %}
),
tx_table AS (
SELECT
block_number,
tx_hash,
from_address AS origin_from_address,
to_address AS origin_to_address,
origin_function_signature
FROM
{{ ref('silver__transactions') }}
WHERE
tx_hash IN (
SELECT
DISTINCT tx_hash
FROM
eth_base
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp) - INTERVAL '72 hours'
FROM
{{ this }}
)
{% endif %}
)
SELECT
tx_hash AS tx_hash,
block_number AS block_number,
block_timestamp AS block_timestamp,
identifier AS identifier,
origin_from_address,
origin_to_address,
origin_function_signature,
from_address,
to_address,
eth_value AS amount,
eth_value_precise_raw AS amount_precise_raw,
eth_value_precise AS amount_precise,
ROUND(
eth_value * price,
2
) AS amount_usd,
_call_id,
_inserted_timestamp,
tx_position,
trace_index,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }} AS native_transfers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
eth_base A
LEFT JOIN {{ ref('silver__hourly_prices_priority_eth') }}
ON DATE_TRUNC(
'hour',
A.block_timestamp
) = HOUR
JOIN tx_table USING (
tx_hash,
block_number
)

5
models/silver/core/silver__receipts.sql
View File

@ -46,10 +46,7 @@ FINAL AS (
) :: INT AS cumulative_gas_used,
utils.udf_hex_to_int(
DATA :effectiveGasPrice :: STRING
) :: INT / pow(
10,
9
) AS effective_gas_price,
) :: bigint AS effective_gas_price,
DATA :from :: STRING AS from_address,
utils.udf_hex_to_int(
DATA :gasUsed :: STRING

8
models/silver/core/silver__traces.sql
View File

@ -554,7 +554,13 @@ FROM
{% endif %}
)
SELECT
*
*,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'trace_index']
) }} AS traces_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL qualify(ROW_NUMBER() over(PARTITION BY block_number, tx_position, trace_index
ORDER BY

54
models/silver/core/silver__transactions.sql
View File

@ -104,7 +104,12 @@ base_tx AS (
A.data :v :: STRING AS v,
utils.udf_hex_to_int(
A.data :value :: STRING
) :: FLOAT AS VALUE,
) AS value_precise_raw,
utils.udf_decimal_adjust(
value_precise_raw,
18
) AS value_precise,
value_precise :: FLOAT AS VALUE,
A._INSERTED_TIMESTAMP,
A.data
FROM
@ -130,6 +135,8 @@ new_records AS (
t.position,
t.type,
t.v,
t.value_precise_raw,
t.value_precise,
t.value,
block_timestamp,
CASE
@ -141,12 +148,18 @@ new_records AS (
tx_success,
tx_status,
cumulative_gas_used,
effective_gas_price,
utils.udf_decimal_adjust(
effective_gas_price * r.gas_used,
r.effective_gas_price,
9
) AS effective_gas_price,
utils.udf_decimal_adjust(
r.effective_gas_price * r.gas_used,
18
) AS tx_fee_precise,
tx_fee_precise :: FLOAT AS tx_fee,
COALESCE(
tx_fee_precise :: FLOAT,
0
) AS tx_fee,
r.type AS tx_type,
r.l1BlockNumber AS l1_block_number,
r.gas_used_for_l1,
@ -193,6 +206,8 @@ missing_data AS (
t.position,
t.type,
t.v,
t.value_precise_raw,
t.value_precise,
t.value,
b.block_timestamp,
FALSE AS is_pending,
@ -200,12 +215,18 @@ missing_data AS (
r.tx_success,
r.tx_status,
r.cumulative_gas_used,
r.effective_gas_price,
utils.udf_decimal_adjust(
r.effective_gas_price,
9
) AS effective_gas_price,
utils.udf_decimal_adjust(
r.effective_gas_price * r.gas_used,
9
18
) AS tx_fee_precise_heal,
tx_fee_precise_heal :: FLOAT AS tx_fee,
COALESCE(
tx_fee_precise_heal :: FLOAT,
0
) AS tx_fee,
r.type AS tx_type,
r.l1BlockNumber AS l1_block_number,
r.gas_used_for_l1,
@ -259,6 +280,8 @@ FINAL AS (
POSITION,
TYPE,
v,
value_precise_raw,
value_precise,
VALUE,
block_timestamp,
is_pending,
@ -298,6 +321,8 @@ SELECT
POSITION,
TYPE,
v,
value_precise_raw,
value_precise,
VALUE,
block_timestamp,
is_pending,
@ -336,10 +361,9 @@ SELECT
POSITION,
TYPE,
v,
VALUE / pow(
10,
18
) AS VALUE,
VALUE,
value_precise_raw,
value_precise,
block_timestamp,
CASE
WHEN CONCAT(
@ -365,7 +389,13 @@ SELECT
l1_block_number,
gas_used_for_l1,
_inserted_timestamp,
DATA
DATA,
{{ dbt_utils.generate_surrogate_key(
['tx_hash']
) }} AS transactions_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL
WHERE

16
models/silver/core/silver__transfers.sql
View File

@ -239,7 +239,13 @@ heal_model AS (
has_decimal,
has_price,
_log_id,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }} AS transfers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
token_transfers
@ -269,7 +275,13 @@ SELECT
has_decimal,
has_price,
_log_id,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index']
) }} AS transfers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
heal_model
{% endif %}

28
models/silver/core/silver__tx_count.sql
View File

@ -1,15 +1,18 @@
{{ config(
materialized = 'incremental',
unique_key = "block_number",
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
SELECT
block_number,
MIN(_inserted_timestamp) AS _inserted_timestamp,
COUNT(*) AS tx_count
FROM
{{ ref('silver__transactions') }}
WITH base AS (
SELECT
block_number,
MIN(_inserted_timestamp) AS _inserted_timestamp,
COUNT(*) AS tx_count
FROM
{{ ref('silver__transactions') }}
{% if is_incremental() %}
WHERE
@ -21,4 +24,15 @@ WHERE
)
{% endif %}
GROUP BY
block_number
block_number
)
SELECT
*,
{{ dbt_utils.generate_surrogate_key(
['block_number']
) }} AS tx_count_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
base

8
models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql
View File

@ -586,5 +586,11 @@ SELECT
symbols,
decimals,
_id,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['block_number','platform','version']
) }} AS complete_dex_liquidity_pools_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM FINAL

8
models/silver/dex/silver_dex__complete_dex_swaps.sql
View File

@ -1994,7 +1994,13 @@ SELECT
symbol_in,
symbol_out,
f._log_id,
f._inserted_timestamp
f._inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['f.tx_hash','f.event_index']
) }} AS complete_dex_swaps_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
FINAL f
LEFT JOIN {{ ref('silver_dex__complete_dex_liquidity_pools') }} p

9
models/silver/labels/silver__labels.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = 'address',
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -13,7 +14,13 @@ SELECT
label_type,
label_subtype,
address_name,
project_name
project_name,
{{ dbt_utils.generate_surrogate_key(
['address']
) }} AS labels_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('bronze__labels') }}
WHERE

46
models/silver/nft/silver__complete_nft_sales.sql
View File

@ -12,6 +12,7 @@ WITH nft_base_models AS (
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -51,6 +52,7 @@ SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -90,6 +92,7 @@ SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -129,6 +132,7 @@ SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -238,6 +242,7 @@ final_base AS (
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -347,6 +352,7 @@ label_fill_sales AS (
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -392,6 +398,30 @@ label_fill_sales AS (
WHERE
t.project_name IS NULL
AND C.token_name IS NOT NULL
),
blocks_fill AS (
SELECT
* exclude (
complete_nft_sales_id,
inserted_timestamp,
modified_timestamp,
_invocation_id
)
FROM
{{ this }}
WHERE
block_number IN (
SELECT
block_number
FROM
label_fill_sales
)
AND nft_log_id NOT IN (
SELECT
nft_log_id
FROM
label_fill_sales
)
)
{% endif %},
final_joins AS (
@ -401,17 +431,23 @@ final_joins AS (
final_base
{% if is_incremental() %}
UNION
UNION ALL
SELECT
*
FROM
label_fill_sales
UNION ALL
SELECT
*
FROM
blocks_fill
{% endif %}
)
SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
event_type,
platform_address,
platform_name,
@ -445,7 +481,13 @@ SELECT
nft_log_id,
input_data,
_log_id,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['tx_hash', 'event_index', 'nft_address','tokenId','platform_exchange_version']
) }} AS complete_nft_sales_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
final_joins qualify(ROW_NUMBER() over(PARTITION BY nft_log_id
ORDER BY

3
models/silver/nft/silver__complete_nft_sales.yml
View File

@ -28,6 +28,9 @@ models:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: EVENT_INDEX
tests:
- not_null
- name: PLATFORM_ADDRESS
tests:
- not_null

65
models/silver/nft/silver__nft_transfers.sql
View File

@ -43,7 +43,7 @@ AND TO_TIMESTAMP_NTZ(_inserted_timestamp) >= (
SELECT
MAX(
_inserted_timestamp
)
) - INTERVAL '24 hours'
FROM
{{ this }}
)
@ -234,6 +234,7 @@ all_transfers AS (
erc1155_value,
_inserted_timestamp,
event_index,
1 AS intra_event_index,
'erc721_Transfer' AS token_transfer_type,
CONCAT(
_log_id,
@ -256,6 +257,7 @@ all_transfers AS (
erc1155_value,
_inserted_timestamp,
event_index,
1 AS intra_event_index,
'erc1155_TransferSingle' AS token_transfer_type,
CONCAT(
_log_id,
@ -280,6 +282,7 @@ all_transfers AS (
erc1155_value,
_inserted_timestamp,
event_index,
intra_event_index,
'erc1155_TransferBatch' AS token_transfer_type,
CONCAT(
_log_id,
@ -301,6 +304,7 @@ transfer_base AS (
block_timestamp,
tx_hash,
event_index,
intra_event_index,
contract_address,
C.token_name AS project_name,
from_address,
@ -328,6 +332,7 @@ fill_transfers AS (
t.block_timestamp,
t.tx_hash,
t.event_index,
t.intra_event_index,
t.contract_address,
C.token_name AS project_name,
t.from_address,
@ -348,6 +353,30 @@ fill_transfers AS (
WHERE
t.project_name IS NULL
AND C.token_name IS NOT NULL
),
blocks_fill AS (
SELECT
* exclude (
nft_transfers_id,
inserted_timestamp,
modified_timestamp,
_invocation_id
)
FROM
{{ this }}
WHERE
block_number IN (
SELECT
block_number
FROM
fill_transfers
)
AND _log_id NOT IN (
SELECT
_log_id
FROM
fill_transfers
)
)
{% endif %},
final_base AS (
@ -356,6 +385,7 @@ final_base AS (
block_timestamp,
tx_hash,
event_index,
intra_event_index,
contract_address,
project_name,
from_address,
@ -370,12 +400,13 @@ final_base AS (
transfer_base
{% if is_incremental() %}
UNION
UNION ALL
SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
intra_event_index,
contract_address,
project_name,
from_address,
@ -388,13 +419,13 @@ SELECT
_inserted_timestamp
FROM
fill_transfers
{% endif %}
)
UNION ALL
SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
intra_event_index,
contract_address,
project_name,
from_address,
@ -405,6 +436,32 @@ SELECT
token_transfer_type,
_log_id,
_inserted_timestamp
FROM
blocks_fill
{% endif %}
)
SELECT
block_number,
block_timestamp,
tx_hash,
event_index,
intra_event_index,
contract_address,
project_name,
from_address,
to_address,
tokenId,
erc1155_value,
event_type,
token_transfer_type,
_log_id,
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['tx_hash','event_index','intra_event_index']
) }} AS nft_transfers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
final_base qualify ROW_NUMBER() over (
PARTITION BY _log_id

6
models/silver/nft/silver__nft_transfers.yml
View File

@ -28,6 +28,12 @@ models:
- not_null
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: EVENT_INDEX
tests:
- not_null
- name: INTRA_EVENT_INDEX
tests:
- not_null
- name: CONTRACT_ADDRESS
tests:
- not_null

9
models/silver/prices/silver__asset_metadata_all_providers.sql
View File

@ -1,5 +1,6 @@
{{ config(
materialized = 'incremental',
merge_exclude_columns = ["inserted_timestamp"],
unique_key = ['token_address','symbol','id','provider'],
tags = ['non_realtime']
) }}
@ -14,7 +15,13 @@ SELECT
token_name AS NAME,
token_decimals AS decimals,
provider,
p._inserted_timestamp AS _inserted_timestamp
p._inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['token_address','symbol','id','provider']
) }} AS asset_metadata_all_providers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('bronze__asset_metadata_all_providers') }}
p

9
models/silver/prices/silver__asset_metadata_priority.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = 'token_address',
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -18,7 +19,13 @@ SELECT
WHEN p.provider = 'coingecko' THEN 1
WHEN p.provider = 'coinmarketcap' THEN 2
END AS priority,
p._inserted_timestamp
p._inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['p.token_address']
) }} AS asset_metadata_priority_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('bronze__asset_metadata_priority') }}
p

9
models/silver/prices/silver__hourly_prices_all_providers.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = ['token_address', 'hour', 'provider'],
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -10,7 +11,13 @@ SELECT
provider,
price,
is_imputed,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['token_address', 'hour', 'provider']
) }} AS hourly_prices_all_providers_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('bronze__hourly_prices_all_providers') }}
WHERE

9
models/silver/prices/silver__hourly_prices_priority.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = ['token_address', 'hour'],
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -14,7 +15,13 @@ SELECT
C.token_symbol,
m.symbol
) AS symbol,
C.token_decimals AS decimals
C.token_decimals AS decimals,
{{ dbt_utils.generate_surrogate_key(
['p.token_address', 'p.hour']
) }} AS hourly_prices_priority_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('bronze__hourly_prices_priority') }}
p

9
models/silver/prices/silver__hourly_prices_priority_eth.sql
View File

@ -1,6 +1,7 @@
{{ config(
materialized = 'incremental',
unique_key = ['token_address', 'hour'],
merge_exclude_columns = ["inserted_timestamp"],
tags = ['non_realtime']
) }}
@ -9,7 +10,13 @@ SELECT
token_address,
price,
is_imputed,
_inserted_timestamp
_inserted_timestamp,
{{ dbt_utils.generate_surrogate_key(
['token_address', 'hour']
) }} AS hourly_prices_priority_eth_id,
SYSDATE() AS inserted_timestamp,
SYSDATE() AS modified_timestamp,
'{{ invocation_id }}' AS _invocation_id
FROM
{{ ref('bronze__hourly_prices_priority_eth') }}
WHERE