mirror of
https://github.com/FlipsideCrypto/flow-models.git
synced 2026-02-06 15:21:50 +00:00
AN-2178/events fix (#66)
* test excl bad events * del date filter * prices fix * nft tweaks * clean
This commit is contained in:
Jack Forgash
GitHub
parent
78927966ff
commit
b541b628cc
@ -49,5 +49,3 @@ SELECT
|
||||
*
|
||||
FROM
|
||||
combo
|
||||
WHERE
|
||||
block_timestamp >= '2022-04-20'
|
||||
|
||||
@ -17,8 +17,6 @@ WITH single_swaps AS (
|
||||
token_in_contract
|
||||
FROM
|
||||
{{ ref('silver__swaps_single_trade') }}
|
||||
WHERE
|
||||
block_timestamp >= '2022-04-20'
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
|
||||
@ -9,8 +9,6 @@ WITH silver_nfts AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__nft_sales') }}
|
||||
WHERE
|
||||
block_timestamp >= '2022-04-20'
|
||||
),
|
||||
gold_nfts AS (
|
||||
SELECT
|
||||
|
||||
@ -9,8 +9,6 @@ WITH staking_actions AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__staking_actions') }}
|
||||
WHERE
|
||||
block_timestamp :: DATE >= '2022-04-20'
|
||||
),
|
||||
FINAL AS (
|
||||
SELECT
|
||||
|
||||
@ -15,8 +15,4 @@ SELECT
|
||||
FROM
|
||||
{{ ref('silver__token_transfers') }}
|
||||
WHERE
|
||||
block_timestamp :: DATE >= '2022-04-20'
|
||||
AND (
|
||||
token_contract = 'A.c38aea683c0c4d38.ZelosAccountingToken'
|
||||
AND recipient IS NOT NULL
|
||||
)
|
||||
token_contract != 'A.c38aea683c0c4d38.ZelosAccountingToken'
|
||||
|
||||
@ -8,8 +8,6 @@ WITH silver_blocks AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__blocks') }}
|
||||
WHERE
|
||||
block_timestamp >= '2022-04-20'
|
||||
),
|
||||
gold_blocks AS (
|
||||
SELECT
|
||||
|
||||
@ -8,8 +8,6 @@ WITH events_final AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
WHERE
|
||||
block_timestamp >= '2022-04-20'
|
||||
),
|
||||
events AS (
|
||||
SELECT
|
||||
|
||||
@ -8,8 +8,6 @@ WITH silver_txs AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__transactions') }}
|
||||
WHERE
|
||||
block_timestamp >= '2022-04-20'
|
||||
),
|
||||
gold_txs AS (
|
||||
SELECT
|
||||
|
||||
@ -79,7 +79,6 @@ models:
|
||||
- name: AUTHORIZERS
|
||||
description: "{{ doc('authorizers') }}"
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- ARRAY
|
||||
@ -87,7 +86,6 @@ models:
|
||||
- name: COUNT_AUTHORIZERS
|
||||
description: "{{ doc('count_authorizers') }}"
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
|
||||
29
models/silver/silver__bad_events.sql
Normal file
29
models/silver/silver__bad_events.sql
Normal file
@ -0,0 +1,29 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
incremental_strategy = 'delete+insert',
|
||||
cluster_by = ['_inserted_timestamp::date'],
|
||||
unique_key = "CONCAT_WS('-', tx_id, event_index)"
|
||||
) }}
|
||||
|
||||
WITH events AS (
|
||||
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
WHERE
|
||||
event_data :: STRING = '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND _inserted_timestamp >= (
|
||||
SELECT
|
||||
MAX(_inserted_timestamp)
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
)
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
events
|
||||
8
models/silver/silver__bad_events.yml
Normal file
8
models/silver/silver__bad_events.yml
Normal file
@ -0,0 +1,8 @@
|
||||
version: 2
|
||||
|
||||
models:
|
||||
- name: silver__bad_events
|
||||
description: Tracks erroneous events.
|
||||
tests:
|
||||
- dbt_expectations.expect_table_row_count_to_be_between:
|
||||
max_value: 1
|
||||
@ -12,6 +12,8 @@ WITH events AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
-- WHERE
|
||||
-- event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
|
||||
@ -12,6 +12,8 @@ WITH events AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
-- WHERE
|
||||
-- event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
|
||||
@ -23,16 +23,13 @@ AND _inserted_timestamp >= (
|
||||
)
|
||||
{% endif %}
|
||||
),
|
||||
event_nulls AS (
|
||||
payload_method AS (
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
events
|
||||
WHERE
|
||||
COALESCE (
|
||||
_event_data_type :Fields,
|
||||
_event_data_type :fields
|
||||
) IS NULL
|
||||
_try_parse_payload IS NOT NULL
|
||||
),
|
||||
events_data AS (
|
||||
SELECT
|
||||
@ -52,6 +49,8 @@ events_data AS (
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
events
|
||||
WHERE
|
||||
_try_parse_payload IS NULL
|
||||
),
|
||||
attributes AS (
|
||||
SELECT
|
||||
@ -103,7 +102,7 @@ attributes_2 AS (
|
||||
VALUE :value :value :value :value,
|
||||
VALUE :value :value :value,
|
||||
VALUE :value :value
|
||||
) AS attribute_value,
|
||||
) :: STRING AS attribute_value,
|
||||
concat_ws(
|
||||
'-',
|
||||
event_id,
|
||||
@ -114,8 +113,10 @@ attributes_2 AS (
|
||||
_inserted_timestamp,
|
||||
'attributes_2' AS _cte
|
||||
FROM
|
||||
event_nulls,
|
||||
LATERAL FLATTEN(_event_data_fields)
|
||||
payload_method,
|
||||
LATERAL FLATTEN(
|
||||
_try_parse_payload :value :fields
|
||||
)
|
||||
),
|
||||
combo AS (
|
||||
SELECT
|
||||
|
||||
@ -62,6 +62,7 @@ events AS (
|
||||
tx_id,
|
||||
event_index
|
||||
) AS event_id,
|
||||
TRY_PARSE_JSON(BASE64_DECODE_STRING(VALUE :payload)) AS try_parse_payload,
|
||||
_ingested_at,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
@ -85,6 +86,7 @@ events AS (
|
||||
event_data,
|
||||
event_data_type AS _event_data_type,
|
||||
event_data_fields AS _event_data_fields,
|
||||
try_parse_payload AS _try_parse_payload,
|
||||
_ingested_at,
|
||||
_inserted_timestamp
|
||||
FROM
|
||||
|
||||
@ -92,7 +92,11 @@ objs AS (
|
||||
event_id,
|
||||
OBJECT_AGG(
|
||||
attribute_key,
|
||||
attribute_value :: variant
|
||||
IFF(
|
||||
attribute_value IS NULL,
|
||||
'null',
|
||||
attribute_value
|
||||
) :: variant
|
||||
) AS event_data
|
||||
FROM
|
||||
all_event_attributes
|
||||
|
||||
@ -95,6 +95,8 @@ models:
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
- DOUBLE
|
||||
- FLOAT
|
||||
|
||||
- name: currency
|
||||
description: "{{ doc('currency') }}"
|
||||
|
||||
@ -12,6 +12,8 @@ WITH silver_events AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
-- WHERE
|
||||
-- event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
|
||||
@ -4,12 +4,6 @@ models:
|
||||
- name: silver__nft_topshot_sales
|
||||
description: |-
|
||||
TopShot direct market sales on the Flow blockchain.
|
||||
tests:
|
||||
- dbt_utils.unique_combination_of_columns:
|
||||
combination_of_columns:
|
||||
- tx_id
|
||||
- buyer
|
||||
- nft_id
|
||||
|
||||
columns:
|
||||
- name: tx_id
|
||||
|
||||
@ -12,16 +12,17 @@ WITH silver_events AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
WHERE
|
||||
event_data :: STRING != '{}'
|
||||
-- WHERE
|
||||
-- event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND _inserted_timestamp >= (
|
||||
SELECT
|
||||
MAX(_inserted_timestamp)
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
WHERE
|
||||
_inserted_timestamp >= (
|
||||
SELECT
|
||||
MAX(_inserted_timestamp)
|
||||
FROM
|
||||
{{ this }}
|
||||
)
|
||||
{% endif %}
|
||||
),
|
||||
sale_trigger AS (
|
||||
|
||||
@ -21,7 +21,10 @@ prices AS (
|
||||
SELECT
|
||||
recorded_at,
|
||||
asset_id,
|
||||
NAME AS token,
|
||||
COALESCE(
|
||||
NAME,
|
||||
INITCAP(SPLIT(asset_id, '-') [0])
|
||||
) AS token,
|
||||
SPLIT(
|
||||
symbol,
|
||||
'$'
|
||||
@ -41,7 +44,15 @@ adj_token_names AS (
|
||||
WHEN token = 'Blocto Token' THEN 'Blocto'
|
||||
ELSE token
|
||||
END AS token,
|
||||
symbol,
|
||||
COALESCE(
|
||||
symbol,
|
||||
CASE
|
||||
WHEN token = 'Flow' THEN 'FLOW'
|
||||
WHEN token = 'Blocto' THEN 'BLT'
|
||||
WHEN token = 'Starly' THEN 'STARLY'
|
||||
ELSE 'Error'
|
||||
END
|
||||
) AS symbol,
|
||||
price,
|
||||
source
|
||||
FROM
|
||||
|
||||
@ -11,6 +11,8 @@ WITH silver_events AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
-- WHERE
|
||||
-- event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
|
||||
@ -20,7 +20,8 @@ swaps_txs AS (
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
WHERE
|
||||
event_contract IN (
|
||||
event_data :: STRING != '{}'
|
||||
AND event_contract IN (
|
||||
SELECT
|
||||
event_contract
|
||||
FROM
|
||||
@ -48,6 +49,7 @@ swap_events AS (
|
||||
FROM
|
||||
swaps_txs
|
||||
)
|
||||
AND event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND _inserted_timestamp >= (
|
||||
|
||||
@ -11,6 +11,8 @@ WITH events AS (
|
||||
*
|
||||
FROM
|
||||
{{ ref('silver__events_final') }}
|
||||
-- WHERE
|
||||
-- event_data :: STRING != '{}'
|
||||
|
||||
{% if is_incremental() %}
|
||||
WHERE
|
||||
|
||||
@ -75,7 +75,6 @@ models:
|
||||
- name: authorizers
|
||||
description: "{{ doc('authorizers') }}"
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- ARRAY
|
||||
@ -83,7 +82,6 @@ models:
|
||||
- name: count_authorizers
|
||||
description: "{{ doc('count_authorizers') }}"
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
|
||||
@ -75,7 +75,6 @@ models:
|
||||
- name: authorizers
|
||||
description: "{{ doc('authorizers') }}"
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- ARRAY
|
||||
@ -83,7 +82,6 @@ models:
|
||||
- name: count_authorizers
|
||||
description: "{{ doc('count_authorizers') }}"
|
||||
tests:
|
||||
- not_null
|
||||
- dbt_expectations.expect_column_values_to_be_in_type_list:
|
||||
column_type_list:
|
||||
- NUMBER
|
||||
|
||||
Loading…
Reference in New Issue
Block a user