An-2435/core liga (#92)

* core view

* restructure

* test

* unique key on metadata tbl

* la liga silver models

* la liga metadata

* final

* now its final

* final to table
This commit is contained in:
Jack Forgash 2022-12-01 10:15:11 -07:00 committed by GitHub
parent f12f90ede9
commit 6be5eea928
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 719 additions and 96 deletions

View File

@ -3,7 +3,7 @@ version: 2
models:
- name: core__dim_allday_metadata
description: |-
Data for NFL AllDay Moments, including player, team, stats and more.
Data for NFL AllDay Moments, including player, team, stats and more. This is produced via API and may differ in structure from metadata available on-chain in the `dim_moment_metadata` table.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:

View File

@ -0,0 +1,20 @@
{{ config (
materialized = 'view',
tags = ['nft', 'dapper']
) }}
SELECT
event_contract as nft_collection,
nft_id,
serial_number,
max_mint_size,
play_id,
series_id,
series_name,
set_id,
set_name,
edition_id,
tier,
metadata
FROM
{{ ref('silver__nft_moment_metadata_final') }}

View File

@ -0,0 +1,61 @@
version: 2
models:
- name: core__dim_moment_metadata
description: |-
NFT Moment Metadata scraped from on-chain activity, where available. This should be joinable on sales or mints using the nft_collection (event_contract) and nft id.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- nft_collection
- nft_id
columns:
- name: nft_collection
description: "{{ doc('nft_collection') }}"
tests:
- not_null
- name: nft_id
description: "{{ doc('nft_id') }}"
tests:
- not_null
- name: serial_number
description: "{{ doc('serial_number') }}"
tests:
- not_null
- name: max_mint_size
description: "{{ doc('max_mint_size') }}"
tests:
- not_null
- name: play_id
description: "{{ doc('play_id') }}"
tests:
- not_null
- name: series_id
description: "{{ doc('series_id') }}"
tests:
- not_null
- name: series_name
description: "{{ doc('series_name') }}"
tests:
- not_null
- name: set_id
description: "{{ doc('set_id') }}"
tests:
- not_null
- name: set_name
description: "{{ doc('set_name') }}"
tests:
- not_null
- name: edition_id
description: "{{ doc('edition_id') }}"
tests:
- not_null
- name: tier
description: "{{ doc('tier') }}"
tests:
- not_null
- name: metadata
description: "{{ doc('metadata') }}"
tests:
- not_null

View File

@ -3,7 +3,7 @@ version: 2
models:
- name: core__dim_topshot_metadata
description: |-
Data for TopShot Moments, including player, team, stats and more.
Data for TopShot Moments, including player, team, stats and more. This is produced via API and may differ in structure from metadata available on-chain in the `dim_moment_metadata` table.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:

View File

@ -0,0 +1,5 @@
{% docs edition_id %}
The numeric ID for the edition of moments. This is likely a bundle of moment NFTs within the same type of play, differentiated by some trait like rarity.
{% enddocs %}

View File

@ -0,0 +1,6 @@
{% docs max_mint_size %}
The gross NFTs minted for the batch.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs metadata %}
A JSON object containing moment or play metadata.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs moment_id %}
The ID for the moment as recorded on-chain.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs play_id %}
The numeric ID of the play for the moment. For example, with topshot moment ABC may be a version of Play 123. The play id is 123 and moment metadata is associated with the play, rather than the moment.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs series_id %}
The numeric ID for the series of moments or plays. This corresponds with the series column found in some metadata tables.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs series_name %}
The series for this particular set of drops.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs set_id %}
The numeric ID for the set. This corresponds with the set name found in some metadata columns.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs tier %}
The tiered rarity of the edition.
{% enddocs %}

View File

@ -1,35 +0,0 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_inserted_timestamp::DATE'],
unique_key = 'tx_id',
incremental_strategy = 'delete+insert'
) }}
WITH la_liga AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_contract ILIKE '%87ca73a41bb50ad5%'
AND event_type IN (
'PlayCreated',
'EditionCreated',
'SetCreated',
'SeriesCreated'
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
)
SELECT
*
FROM
la_liga

View File

@ -1,59 +0,0 @@
{{ config(
materialized = 'incremental',
cluster_by = ['nft_id'],
unique_key = 'nft_id',
incremental_strategy = 'delete+insert'
) }}
WITH liga_plays AS (
SELECT
*
FROM
{{ ref('silver__nft_la_liga_events') }}
WHERE
event_type = 'PlayCreated'
),
play_metadata AS (
SELECT
event_data :id :: NUMBER AS nft_id_raw,
VALUE :key :value :: STRING AS column_header,
VALUE :value :value :: STRING AS column_value
FROM
liga_plays,
LATERAL FLATTEN(input => TRY_PARSE_JSON(event_data :metadata))
WHERE
event_type = 'PlayCreated'
),
FINAL AS (
SELECT
*
FROM
play_metadata pivot(MAX(column_value) for column_header IN ('PlayerCountry', 'MatchHighlightedTeam', 'MatchSeason', 'PlayerPosition', 'PlayType', 'MatchDate', 'PlayerLastName', 'PlayDataID', 'MatchDay', 'PlayTime', 'PlayerNumber', 'PlayerFirstName', 'PlayerKnownName', 'MatchHomeTeam', 'MatchAwayTeam', 'MatchHomeScore', 'MatchAwayScore', 'PlayerJerseyName', 'PlayHalf', 'PlayerDataID')) AS p (
nft_id,
player_country,
match_highlighted_team,
match_season,
player_position,
play_type,
match_date,
player_last_name,
play_data_id,
match_day,
play_time,
player_number,
player_first_name,
player_known_name,
match_home_team,
match_away_team,
match_home_score,
match_away_score,
player_jersey_name,
play_half,
player_data_id
)
)
SELECT
*
FROM
FINAL

View File

@ -0,0 +1,50 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_inserted_timestamp'],
unique_key = "concat_ws('-', event_contract, edition_id)",
incremental_strategy = 'delete+insert',
tags = ['nft', 'dapper']
) }}
WITH events AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_type = 'EditionCreated'
AND event_contract IN (
'A.e4cf4bdc1751c65d.AllDay',
'A.b715b81853fef53f.AllDay',
'A.87ca73a41bb50ad5.Golazos'
)
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
),
org AS (
SELECT
tx_id,
block_timestamp,
event_contract,
event_data :id :: STRING AS edition_id,
event_data :maxMintSize :: STRING AS max_mint_size,
event_data :playID :: STRING AS play_id,
event_data :seriesID :: STRING AS series_id,
event_data :setID :: STRING AS set_id,
event_data :tier :: STRING AS tier,
_inserted_timestamp
FROM
events
)
SELECT
*
FROM
org

View File

@ -0,0 +1,32 @@
version: 2
models:
- name: silver__nft_moment_editions
description: |-
Cleaned EditionCreated events.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- event_contract
- edition_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
- name: edition_id
description: "{{ doc('edition_id') }}"
- name: max_mint_size
description: "{{ doc('max_mint_size') }}"
- name: play_id
description: "{{ doc('play_id') }}"
- name: series_id
description: "{{ doc('series_id') }}"
- name: set_id
description: "{{ doc('set_id') }}"
- name: tier
description: "{{ doc('tier') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"

View File

@ -0,0 +1,75 @@
{{ config(
materialized = 'incremental',
cluster_by = ['play_id'],
unique_key = "concat_ws('-', event_contract, play_id)",
incremental_strategy = 'delete+insert',
tags = ['nft', 'dapper']
) }}
WITH play_creation AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_type = 'PlayCreated'
{#
currently includes the following contracts
A.c38aea683c0c4d38.Eternal
A.b715b81853fef53f.AllDay
A.67af7ecf76556cd3.ABD
A.0b2a3299cc857e29.TopShot
A.5c0992b465832a94.TKNZ
A.e4cf4bdc1751c65d.AllDay
A.87ca73a41bb50ad5.Golazos
#}
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
),
play_metadata AS (
SELECT
tx_id,
block_timestamp,
event_contract,
event_data :id :: NUMBER AS play_id,
VALUE :key :value :: STRING AS column_header,
VALUE :value :value :: STRING AS column_value,
_inserted_timestamp
FROM
play_creation,
LATERAL FLATTEN(input => TRY_PARSE_JSON(event_data :metadata))
),
FINAL AS (
SELECT
tx_id,
block_timestamp,
event_contract,
play_id,
_inserted_timestamp,
OBJECT_AGG(
column_header :: variant,
column_value :: variant
) AS metadata
FROM
play_metadata
GROUP BY
1,
2,
3,
4,
5
)
SELECT
*
FROM
FINAL

View File

@ -0,0 +1,22 @@
version: 2
models:
- name: silver__nft_moment_metadata
description: |-
Cleaned PlayCreated events, which is what records the metadata on-chain.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- event_contract
- play_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
- name: play_id
description: "{{ doc('play_id') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"

View File

@ -0,0 +1,80 @@
{{ config(
materialized = 'table',
cluster_by = ['_inserted_timestamp'],
unique_key = "concat_ws('-',event_contract,edition_id,nft_id)",
tags = ['nft', 'dapper']
) }}
WITH moments AS (
SELECT
*
FROM
{{ ref('silver__nft_moment_minted') }}
),
metadata AS (
SELECT
*
FROM
{{ ref('silver__nft_moment_metadata') }}
),
editions AS (
SELECT
*
FROM
{{ ref('silver__nft_moment_editions') }}
),
series AS (
SELECT
*
FROM
{{ ref('silver__nft_moment_series') }}
),
set_nm AS (
SELECT
*
FROM
{{ ref('silver__nft_moment_set') }}
),
FINAL AS (
SELECT
-- tx id and block timestamp don't matter for the final table
m.tx_id,
m.block_timestamp,
m.event_contract,
m.nft_id,
m.serial_number,
e.max_mint_size,
e.play_id,
e.series_id,
s.series_name,
e.set_id,
sn.set_name,
e.edition_id,
e.tier,
pl.metadata,
m._inserted_timestamp,
sn._inserted_timestamp AS _inserted_timestamp_set
FROM
moments m
LEFT JOIN editions e USING (
event_contract,
edition_id
)
LEFT JOIN metadata pl USING (
event_contract,
play_id
)
LEFT JOIN series s USING (
event_contract,
series_id
)
LEFT JOIN set_nm sn USING (
event_contract,
set_id
)
)
SELECT
*
FROM
FINAL

View File

@ -0,0 +1,39 @@
version: 2
models:
- name: silver__nft_moment_metadata_final
description: |-
Cleaned PlayCreated events.
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
- name: nft_id
description: "{{ doc('nft_id') }}"
- name: serial_number
description: "{{ doc('serial_number') }}"
- name: max_mint_size
description: "{{ doc('max_mint_size') }}"
- name: play_id
description: "{{ doc('play_id') }}"
- name: series_id
description: "{{ doc('series_id') }}"
- name: series_name
description: "{{ doc('series_name') }}"
- name: set_id
description: "{{ doc('set_id') }}"
- name: set_name
description: "{{ doc('set_name') }}"
- name: edition_id
description: "{{ doc('edition_id') }}"
- name: tier
description: "{{ doc('tier') }}"
- name: metadata
description: "{{ doc('metadata') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"
tests:
- not_null

View File

@ -0,0 +1,42 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_inserted_timestamp'],
unique_key = "concat_ws('-', event_contract, edition_id)",
incremental_strategy = 'delete+insert',
tags = ['nft', 'dapper']
) }}
WITH events AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_type = 'MomentNFTMinted'
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
),
org AS (
SELECT
tx_id,
block_timestamp,
event_contract,
event_data :editionID :: STRING AS edition_id,
event_data :id :: STRING AS nft_id,
event_data :serialNumber :: STRING AS serial_number,
_inserted_timestamp
FROM
events
)
SELECT
*
FROM
org

View File

@ -0,0 +1,32 @@
version: 2
models:
- name: silver__nft_moment_minted
description: |-
Cleaned MomentNFTMinted events.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- event_contract
- nft_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
tests:
- accepted_values:
values:
- "A.e4cf4bdc1751c65d.AllDay"
- "A.b715b81853fef53f.AllDay"
- "A.87ca73a41bb50ad5.Golazos"
- name: serial_number
description: "{{ doc('serial_number') }}"
- name: edition_id
description: "{{ doc('edition_id') }}"
- name: nft_id
description: "{{ doc('nft_id') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"

View File

@ -0,0 +1,43 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_inserted_timestamp'],
unique_key = "concat_ws('-', event_contract, moment_id)",
incremental_strategy = 'delete+insert',
tags = ['nft', 'dapper']
) }}
WITH events AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_type = 'MomentMinted'
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
),
org AS (
SELECT
tx_id,
block_timestamp,
event_contract,
event_data :momentID :: STRING AS moment_id,
event_data :serialNumber :: STRING AS serial_number,
event_data :seriesID :: STRING AS series_id,
event_data :setID :: STRING AS set_id,
_inserted_timestamp
FROM
events
)
SELECT
*
FROM
org

View File

@ -0,0 +1,35 @@
version: 2
models:
- name: silver__nft_moment_minted_2
description: |-
Cleaned MomentNFTMinted events.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- event_contract
- moment_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
tests:
- accepted_values:
values:
- "A.c38aea683c0c4d38.Eternal"
- "A.d4ad4740ee426334.Moments"
- "A.67af7ecf76556cd3.ABD"
- "A.0b2a3299cc857e29.TopShot"
- name: moment_id
description: "{{ doc('moment_id') }}"
- name: serial_number
description: "{{ doc('serial_number') }}"
- name: set_id
description: "{{ doc('set_id') }}"
- name: series_id
description: "{{ doc('series_id') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"

View File

@ -0,0 +1,42 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_inserted_timestamp'],
unique_key = "concat_ws('-', event_contract, series_id)",
incremental_strategy = 'delete+insert',
tags = ['nft', 'dapper']
) }}
WITH events AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_type = 'SeriesCreated'
AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data))
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
),
org AS (
SELECT
tx_id,
block_timestamp,
event_contract,
event_data :id :: STRING AS series_id,
event_data :name :: STRING AS series_name,
_inserted_timestamp
FROM
events
)
SELECT
*
FROM
org

View File

@ -0,0 +1,30 @@
version: 2
models:
- name: silver__nft_moment_series
description: |-
Cleaned SeriesCreated events.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- event_contract
- series_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
tests:
- accepted_values:
values:
- "A.e4cf4bdc1751c65d.AllDay"
- "A.b715b81853fef53f.AllDay"
- "A.87ca73a41bb50ad5.Golazos"
- name: series_id
description: "{{ doc('series_id') }}"
- name: series_name
description: "{{ doc('series_name') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"

View File

@ -0,0 +1,44 @@
{{ config(
materialized = 'incremental',
cluster_by = ['_inserted_timestamp'],
unique_key = "concat_ws('-', event_contract, set_id)",
incremental_strategy = 'delete+insert',
tags = ['nft', 'dapper']
) }}
WITH events AS (
SELECT
*
FROM
{{ ref('silver__events_final') }}
WHERE
event_type = 'SetCreated'
AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data))
{% if is_incremental() %}
AND _inserted_timestamp >= (
SELECT
MAX(_inserted_timestamp)
FROM
{{ this }}
)
{% endif %}
),
org AS (
SELECT
tx_id,
block_timestamp,
event_contract,
event_data :id :: STRING AS set_id,
event_data :name :: STRING AS set_name,
_inserted_timestamp
FROM
events
WHERE
set_id IS NOT NULL
)
SELECT
*
FROM
org

View File

@ -0,0 +1,24 @@
version: 2
models:
- name: silver__nft_moment_set
description: |-
Cleaned SetCreated events.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- event_contract
- set_id
columns:
- name: tx_id
description: "{{ doc('tx_id') }}"
- name: block_timestamp
description: "{{ doc('block_timestamp') }}"
- name: event_contract
description: "{{ doc('event_contract') }}"
- name: set_id
description: "{{ doc('set_id') }}"
- name: set_name
description: "{{ doc('set_name') }}"
- name: _inserted_timestamp
description: "{{ doc('_inserted_timestamp') }}"