flip/solana-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/solana-models.git synced 2026-02-06 14:16:46 +00:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

An 1636/convert solana to use inserted timestamp (#60)

Browse Source 
* setting to append new columns on incremental for existing models

* add _inserted_timestamp to applicable models

* update yml descriptions
This commit is contained in:
desmond-hui 2022-06-22 06:13:44 -07:00 committed by GitHub
parent 47945f0412
commit 3a82774a82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 268 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dbt_project.yml
View File

@ -42,6 +42,7 @@ models:
+persist_docs:
relation: true
columns: true
+on_schema_change: "append_new_columns"
vars:
"dbt_date:time_zone": America/Los_Angeles

3
models/bronze/bronze__blocks.sql
View File

@ -11,7 +11,8 @@ SELECT
chain_id,
tx_count,
header,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ source(
'prod',

3
models/bronze/bronze__transactions.sql
View File

@ -12,7 +12,8 @@ SELECT
network,
chain_id,
tx,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ source(
'prod',

5
models/descriptions/_inserted_timestamp.md Normal file
View File

@ -0,0 +1,5 @@
{% docs _inserted_timestamp %}
Time the record was inserted into snowflake
{% enddocs %}

3
models/silver/silver___inner_instructions.sql
View File

@ -12,7 +12,8 @@ SELECT
e.index,
e.value :index :: NUMBER AS mapped_instruction_index,
e.value,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('silver__transactions') }}
t,

4
models/silver/silver___inner_instructions.yml
View File

@ -32,4 +32,6 @@ models:
- name: INGESTED_AT
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

3
models/silver/silver___instructions.sql
View File

@ -12,7 +12,8 @@ SELECT
succeeded,
e.index,
e.value,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('silver__transactions') }}
t,

4
models/silver/silver___instructions.yml
View File

@ -32,4 +32,6 @@ models:
- name: INGESTED_AT
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

3
models/silver/silver___post_token_balances.sql
View File

@ -18,7 +18,8 @@ SELECT
b.value :uiTokenAmount :decimals AS DECIMAL,
b.value :uiTokenAmount :uiAmount AS uiAmount,
b.value :uiTokenAmount :uiAmountString AS uiAmountString,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('silver__transactions') }}
t,

4
models/silver/silver___post_token_balances.yml
View File

@ -55,4 +55,6 @@ models:
- name: INGESTED_AT
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

3
models/silver/silver___pre_token_balances.sql
View File

@ -18,7 +18,8 @@ SELECT
b.value :uiTokenAmount :decimals AS DECIMAL,
b.value :uiTokenAmount :uiAmount AS uiAmount,
b.value :uiTokenAmount :uiAmountString AS uiAmountString,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('silver__transactions') }}
t,

4
models/silver/silver___pre_token_balances.yml
View File

@ -55,4 +55,6 @@ models:
- name: INGESTED_AT
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

3
models/silver/silver__blocks.sql
View File

@ -27,7 +27,8 @@ SELECT
header :blockhash :: VARCHAR AS block_hash,
header :parentSlot :: INTEGER AS previous_block_id,
header :previousBlockhash :: VARCHAR AS previous_block_hash,
ingested_at
ingested_at,
_inserted_timestamp
FROM
base_tables

2
models/silver/silver__blocks.yml
View File

@ -49,3 +49,5 @@ models:
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

9
models/silver/silver__events.sql
View File

@ -16,7 +16,8 @@ WITH base_i AS (
value:parsed:type AS event_type,
value:programId AS program_id,
value,
ingested_at
ingested_at,
_inserted_timestamp
FROM {{ ref('silver___instructions') }}
{% if is_incremental() %}
@ -30,7 +31,8 @@ base_ii AS (
tx_id,
mapped_instruction_index :: INTEGER AS mapped_instruction_index,
value,
ingested_at
ingested_at,
_inserted_timestamp
FROM {{ ref('silver___inner_instructions') }}
{% if is_incremental() %}
@ -48,7 +50,8 @@ SELECT
i.program_id :: STRING AS program_id,
i.value AS instruction,
ii.value AS inner_instruction,
i.ingested_at
i.ingested_at,
i._inserted_timestamp
FROM
base_i
i

4
models/silver/silver__events.yml
View File

@ -44,4 +44,6 @@ models:
- name: INGESTED_AT
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

6
models/silver/silver__gov_actions_marinade_tmp.sql
View File

@ -217,7 +217,8 @@ actions_tmp AS (
) THEN e.instruction :accounts [1] :: STRING
ELSE NULL
END AS locker_nft,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }}
e
@ -262,7 +263,8 @@ SELECT
a1.lock_amount,
a2.lock_amount
) AS amount,
a1.ingested_at
a1.ingested_at,
a1._inserted_timestamp
FROM
actions_tmp a1
LEFT OUTER JOIN actions_tmp a2

15
models/silver/silver__nft_mints.sql
View File

@ -85,7 +85,8 @@ txs_tmp AS (
WHEN e.inner_instruction :instructions [0] :programId :: STRING = 'metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s' THEN e.instruction :accounts [3] :: STRING
ELSE NULL
END AS update_authority,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }}
e
@ -193,7 +194,8 @@ pre_final AS (
nft,
potential_nft_mint
) AS mint,
ingested_at
ingested_at,
_inserted_timestamp
FROM
txs t
LEFT OUTER JOIN mint_currency p
@ -212,7 +214,8 @@ pre_final AS (
mint_currency,
mint,
update_authority,
ingested_at
ingested_at,
_inserted_timestamp
),
pre_pre_final AS (
SELECT
@ -229,7 +232,8 @@ pre_pre_final AS (
END AS mint_price,
pf.mint_currency,
pf.mint,
pf.ingested_at
pf.ingested_at,
pf._inserted_timestamp
FROM
pre_final pf
LEFT OUTER JOIN transfers tr
@ -259,7 +263,8 @@ SELECT
) AS mint_price,
mint_currency,
mint,
ingested_at
ingested_at,
_inserted_timestamp
FROM
pre_pre_final
WHERE

4
models/silver/silver__nft_mints.yml
View File

@ -53,4 +53,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

9
models/silver/silver__nft_sales_magic_eden_v1.sql
View File

@ -21,7 +21,8 @@ WITH sales_inner_instructions AS (
e.instruction :accounts [0] :: STRING AS purchaser,
e.instruction :accounts [1] :: STRING AS nft_account,
e.instruction :accounts [2] :: STRING AS nft_account_2,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }} e
INNER JOIN {{ ref('silver__transactions') }}
@ -70,7 +71,8 @@ SELECT
10,
9
) AS sales_amount,
s.ingested_at
s.ingested_at,
s._inserted_timestamp
FROM
sales_inner_instructions s
LEFT OUTER JOIN post_token_balances p
@ -90,4 +92,5 @@ GROUP BY
p2.mint
),
s.purchaser,
s.ingested_at
s.ingested_at,
s._inserted_timestamp

4
models/silver/silver__nft_sales_magic_eden_v1.yml
View File

@ -46,4 +46,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

9
models/silver/silver__nft_sales_magic_eden_v2.sql
View File

@ -61,7 +61,8 @@ base_tmp AS (
ORDER BY
inner_index
) AS nft_account_mint,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('silver__events') }}
e
@ -127,7 +128,8 @@ SELECT
10,
9
) AS sales_amount,
b.ingested_at
b.ingested_at,
b._inserted_timestamp
FROM
base b
LEFT OUTER JOIN post_token_balances p
@ -144,4 +146,5 @@ GROUP BY
p.mint
),
b.purchaser,
b.ingested_at
b.ingested_at,
b._inserted_timestamp

4
models/silver/silver__nft_sales_magic_eden_v2.yml
View File

@ -45,4 +45,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

9
models/silver/silver__nft_sales_opensea.sql
View File

@ -20,7 +20,8 @@ WITH sales_inner_instructions AS (
) AS amount,
e.instruction :accounts [0] :: STRING AS purchaser,
e.instruction :accounts [2] :: STRING AS nft_account,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }}
e
@ -89,7 +90,8 @@ SELECT
10,
9
) AS sales_amount,
s.ingested_at
s.ingested_at,
s._inserted_timestamp
FROM
sales_inner_instructions s
LEFT OUTER JOIN post_token_balances p
@ -105,4 +107,5 @@ GROUP BY
s.program_id,
p.mint,
s.purchaser,
s.ingested_at
s.ingested_at,
s._inserted_timestamp

4
models/silver/silver__nft_sales_opensea.yml
View File

@ -46,4 +46,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

6
models/silver/silver__nft_sales_smb.sql
View File

@ -14,7 +14,8 @@ WITH base_table AS (
e.program_id,
instruction :accounts[0] :: STRING AS acct_1,
instruction :accounts[1] :: STRING AS mint,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM {{ ref('silver__events') }} e
INNER JOIN {{ ref('silver__transactions') }} t
@ -53,7 +54,8 @@ SELECT
b.mint,
b.acct_1 AS purchaser,
p.amount / POW(10,9) AS sales_amount,
b.ingested_at
b.ingested_at,
b._inserted_timestamp
FROM base_table b
INNER JOIN price p

4
models/silver/silver__nft_sales_smb.yml
View File

@ -45,4 +45,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

137
models/silver/silver__nft_sales_solanart.sql
View File

@ -1,68 +1,89 @@
{{ config(
materialized = 'incremental',
unique_key = "CONCAT_WS('-', tx_id, mint)",
incremental_strategy = 'delete+insert',
cluster_by = ['block_timestamp::DATE'],
materialized = 'incremental',
unique_key = "CONCAT_WS('-', tx_id, mint)",
incremental_strategy = 'delete+insert',
cluster_by = ['block_timestamp::DATE'],
) }}
WITH sales_inner_instructions AS (
SELECT
e.block_timestamp,
e.block_id,
e.tx_id,
t.succeeded,
e.program_id,
e.index,
SELECT
e.block_timestamp,
e.block_id,
e.tx_id,
t.succeeded,
e.program_id,
e.index,
COALESCE(
i.value :parsed :info :lamports :: NUMBER,
i.value :parsed :info :lamports :: NUMBER,
0
) AS amount,
e.instruction :accounts [0] :: STRING AS purchaser,
e.instruction :accounts [1] :: STRING AS nft_account,
e.ingested_at
FROM {{ ref('silver__events') }} e
INNER JOIN {{ ref('silver__transactions') }} t
ON t.tx_id = e.tx_id
LEFT OUTER JOIN TABLE(FLATTEN(inner_instruction :instructions)) i
WHERE e.program_id = 'CJsLwbP1iu5DuUikHEJnLfANgKy6stB2uFgvBBHoyxwz' -- Solanart Program ID
{% if is_incremental() %}
AND e.ingested_at :: DATE >= CURRENT_DATE - 2
AND t.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
) AS amount,
e.instruction :accounts [0] :: STRING AS purchaser,
e.instruction :accounts [1] :: STRING AS nft_account,
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }}
e
INNER JOIN {{ ref('silver__transactions') }}
t
ON t.tx_id = e.tx_id
LEFT OUTER JOIN TABLE(FLATTEN(inner_instruction :instructions)) i
WHERE
e.program_id = 'CJsLwbP1iu5DuUikHEJnLfANgKy6stB2uFgvBBHoyxwz' -- Solanart Program ID
{% if is_incremental() %}
AND e.ingested_at :: DATE >= CURRENT_DATE - 2
AND t.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
post_token_balances AS (
SELECT
tx_id,
account,
mint
FROM {{ ref('silver___post_token_balances') }} p
WHERE amount <> 0 -- Removes random account transfers with no NFT
SELECT
tx_id,
account,
mint
FROM
{{ ref('silver___post_token_balances') }}
p
WHERE
amount <> 0 -- Removes random account transfers with no NFT
{% if is_incremental() %}
AND p.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
{% if is_incremental() %}
AND p.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
)
SELECT
s.block_timestamp,
s.block_id,
s.tx_id,
s.succeeded,
s.program_id,
p.mint AS mint,
s.purchaser,
SUM(s.amount) / POW(10,9) AS sales_amount,
s.ingested_at
FROM sales_inner_instructions s
INNER JOIN post_token_balances p
ON s.tx_id = p.tx_id AND s.nft_account = p.account
GROUP BY s.block_timestamp, s.block_id, s.tx_id, s.succeeded, s.program_id, p.mint, s.purchaser, s.ingested_at
HAVING SUM(s.amount) > 0 -- Removes transfers
SELECT
s.block_timestamp,
s.block_id,
s.tx_id,
s.succeeded,
s.program_id,
p.mint AS mint,
s.purchaser,
SUM(
s.amount
) / pow(
10,
9
) AS sales_amount,
s.ingested_at,
s._inserted_timestamp
FROM
sales_inner_instructions s
INNER JOIN post_token_balances p
ON s.tx_id = p.tx_id
AND s.nft_account = p.account
GROUP BY
s.block_timestamp,
s.block_id,
s.tx_id,
s.succeeded,
s.program_id,
p.mint,
s.purchaser,
s.ingested_at,
s._inserted_timestamp
HAVING
SUM(
s.amount
) > 0 -- Removes transfers

4
models/silver/silver__nft_sales_solanart.yml
View File

@ -46,4 +46,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

108
models/silver/silver__nft_sales_solport.sql
View File

@ -5,7 +5,8 @@
cluster_by = ['block_timestamp::DATE'],
) }}
WITH sales_inner_instructions AS (
WITH sales_inner_instructions AS (
SELECT
e.block_timestamp,
e.block_id,
@ -19,50 +20,67 @@ WITH sales_inner_instructions AS (
) AS amount,
e.instruction :accounts [0] :: STRING AS purchaser,
e.instruction :accounts [3] :: STRING AS nft_account,
e.ingested_at
FROM {{ ref('silver__events') }} e
INNER JOIN {{ ref('silver__transactions') }} t
ON t.tx_id = e.tx_id
LEFT OUTER JOIN TABLE(FLATTEN(inner_instruction :instructions)) i
WHERE program_id = 'SPf5WqNywtPrRXSU5enq5z9bPPhREaSYf2LhN5fUxcj' -- Solport Program ID
{% if is_incremental() %}
AND e.ingested_at :: DATE >= CURRENT_DATE - 2
AND t.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }}
e
INNER JOIN {{ ref('silver__transactions') }}
t
ON t.tx_id = e.tx_id
LEFT OUTER JOIN TABLE(FLATTEN(inner_instruction :instructions)) i
WHERE
program_id = 'SPf5WqNywtPrRXSU5enq5z9bPPhREaSYf2LhN5fUxcj' -- Solport Program ID
{% if is_incremental() %}
AND e.ingested_at :: DATE >= CURRENT_DATE - 2
AND t.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
),
post_token_balances AS (
SELECT
tx_id,
account,
amount,
mint
FROM {{ ref('silver___post_token_balances') }} p
SELECT
tx_id,
account,
amount,
mint
FROM
{{ ref('silver___post_token_balances') }}
p
WHERE
amount > 0
WHERE amount > 0
{% if is_incremental() %}
AND p.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
)
SELECT
s.block_timestamp,
s.block_id,
s.tx_id,
s.succeeded,
s.program_id,
p.mint AS mint,
s.purchaser,
SUM(s.amount) / POW(10,9) AS sales_amount,
s.ingested_at
FROM sales_inner_instructions s
INNER JOIN post_token_balances p
ON s.tx_id = p.tx_id AND s.nft_account = p.account
GROUP BY s.block_timestamp, s.block_id, s.tx_id, s.succeeded, s.program_id, p.mint, s.purchaser, s.ingested_at
{% if is_incremental() %}
AND p.ingested_at :: DATE >= CURRENT_DATE - 2
{% endif %}
)
SELECT
s.block_timestamp,
s.block_id,
s.tx_id,
s.succeeded,
s.program_id,
p.mint AS mint,
s.purchaser,
SUM(
s.amount
) / pow(
10,
9
) AS sales_amount,
s.ingested_at,
s._inserted_timestamp
FROM
sales_inner_instructions s
INNER JOIN post_token_balances p
ON s.tx_id = p.tx_id
AND s.nft_account = p.account
GROUP BY
s.block_timestamp,
s.block_id,
s.tx_id,
s.succeeded,
s.program_id,
p.mint,
s.purchaser,
s.ingested_at,
s._inserted_timestamp

4
models/silver/silver__nft_sales_solport.yml
View File

@ -46,4 +46,6 @@ models:
- name: INGESTED_AT
description: " {{ doc('ingested_at') }} "
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

8
models/silver/silver__transactions.sql
View File

@ -3,7 +3,7 @@
unique_key = "CONCAT_WS('-', block_id, tx_id)",
incremental_strategy = 'delete+insert',
cluster_by = ['ingested_at::DATE'],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
) }}
WITH base AS (
@ -28,7 +28,8 @@ WITH base AS (
tx :transaction :message :instructions :: ARRAY AS instructions,
tx :meta :innerInstructions :: ARRAY AS inner_instructions,
tx :meta :logMessages :: ARRAY AS log_messages,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('bronze__transactions') }}
t
@ -58,7 +59,8 @@ SELECT
instructions,
inner_instructions,
log_messages,
ingested_at
ingested_at,
_inserted_timestamp
FROM
base b
qualify(ROW_NUMBER() over(PARTITION BY b.block_id, b.tx_id

2
models/silver/silver__transactions.yml
View File

@ -57,3 +57,5 @@ models:
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

15
models/silver/silver__transfers.sql
View File

@ -14,7 +14,8 @@ WITH base_transfers AS (
e.tx_id,
e.index,
e.instruction,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
{{ ref('silver__events') }}
e
@ -100,7 +101,8 @@ spl_transfers AS (
p3.mint,
p4.mint
) AS mint,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
base_transfers e
LEFT OUTER JOIN base_pre_token_balances p
@ -131,7 +133,8 @@ sol_transfers AS (
9
) AS amount,
'So11111111111111111111111111111111111111112' AS mint,
e.ingested_at
e.ingested_at,
e._inserted_timestamp
FROM
base_transfers e
WHERE
@ -146,7 +149,8 @@ SELECT
tx_to,
amount,
mint,
ingested_at
ingested_at,
_inserted_timestamp
FROM
spl_transfers
UNION
@ -159,6 +163,7 @@ SELECT
tx_to,
amount,
mint,
ingested_at
ingested_at,
_inserted_timestamp
FROM
sol_transfers

2
models/silver/silver__transfers.yml
View File

@ -47,3 +47,5 @@ models:
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"

6
models/silver/silver__votes.sql
View File

@ -21,7 +21,8 @@ WITH base AS (
tx :transaction :message :instructions [0] :parsed :info :voteAuthority :: STRING AS vote_authority,
tx :transaction :message :instructions [0] :parsed :info :vote :hash :: STRING AS vote_hash,
tx :transaction :message :instructions [0] :parsed :info :vote :slots :: ARRAY AS vote_slots,
ingested_at
ingested_at,
_inserted_timestamp
FROM
{{ ref('bronze__transactions') }}
t
@ -44,7 +45,8 @@ SELECT
vote_authority,
vote_hash,
vote_slots,
ingested_at
ingested_at,
_inserted_timestamp
FROM
base qualify(ROW_NUMBER() over(PARTITION BY block_id, tx_id
ORDER BY

4
models/silver/silver__votes.yml
View File

@ -54,4 +54,6 @@ models:
- name: INGESTED_AT
description: "{{ doc('ingested_at') }}"
tests:
- not_null
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('_inserted_timestamp') }}"