flip/solana-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/solana-models.git synced 2026-02-06 11:47:08 +00:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

An 6361/cte logic txs and idls (#851)

Browse Source 
* update tx's logic

* set active col in idls

* same run and tag for idl tables

* fix tag name
This commit is contained in:
tarikceric 2025-06-23 11:30:33 -07:00 committed by GitHub
parent c949225fdf
commit 818fd59939
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 115 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/dbt_run_idls_history.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt run -s models/streamline/parser/history/streamline__idls_history.sql models/streamline/parser/history/streamline__complete_decoded_history.sql
dbt run -s "solana_models,tag:idls" models/streamline/parser/history/streamline__idls_history.sql models/streamline/parser/history/streamline__complete_decoded_history.sql
notify-failure:
needs: [run_dbt_jobs]

4
models/gold/core/core__dim_idls.sql
View File

@ -1,6 +1,6 @@
{{ config(
materialized='view',
tags = ['scheduled_non_core']
tags = ['idls']
)
}}
@ -9,6 +9,8 @@ SELECT
idl,
idl_hash,
is_valid,
is_active,
last_activity_timestamp,
submitted_by,
date_submitted,
first_block_id,

8
models/gold/core/core__dim_idls.yml
View File

@ -23,6 +23,14 @@ models:
description: "{{ doc('is_valid') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: IS_ACTIVE
description: "If the program has decoded instructions in the last 14 days"
tests:
- dbt_expectations.expect_column_to_exist
- name: LAST_ACTIVITY_TIMESTAMP
description: "Most recent date that the program has decoded instructions"
tests:
- dbt_expectations.expect_column_to_exist
- name: SUBMITTED_BY
description: "{{ doc('submitted_by') }}"
tests:

50
models/silver/core/silver__transactions.sql
View File

@ -14,8 +14,33 @@
{% set cutover_block_id = 307103862 %}
{% set cutover_partition_id = 150215 %}
WITH pre_final AS (
WITH base AS (
SELECT
block_timestamp,
block_id,
tx_index,
data,
_partition_id,
_inserted_timestamp,
COALESCE(
data:transaction:message:instructions[0]:programId::STRING,
''
) AS first_program_id,
array_size(data :transaction :message :instructions) as instruction_count
FROM
{{ ref('bronze__stage_block_txs_2') }} AS t
WHERE
block_id >= {{ cutover_block_id }}
{% if is_incremental() %}
AND t._partition_id >= (SELECT max(_partition_id)-1 FROM {{ this }})
AND t._partition_id <= (SELECT max(_partition_id) FROM {{ ref('streamline__complete_block_txs_2') }})
AND t._inserted_timestamp > (SELECT max(_inserted_timestamp) FROM {{ this }})
{% else %}
AND t._partition_id < 0 /* keep this here, if we ever do a full refresh this should select no data from streamline 2.0 data */
{% endif %}
AND t._partition_id >= {{ cutover_partition_id }}
),
pre_final AS (
SELECT
COALESCE(TO_TIMESTAMP_NTZ(t.value :block_time), b.block_timestamp) AS block_timestamp,
t.block_id,
@ -91,26 +116,11 @@ WITH pre_final AS (
t.tx_index,
t._partition_id,
t._inserted_timestamp
FROM
{{ ref('bronze__stage_block_txs_2') }} AS t
FROM base AS t
WHERE
t.block_id >= {{ cutover_block_id }}
first_program_id <> 'Vote111111111111111111111111111111111111111'
OR instruction_count > 1
AND tx_id IS NOT NULL
AND (
COALESCE(t.data :transaction :message :instructions [0] :programId :: STRING,'') <> 'Vote111111111111111111111111111111111111111'
OR
(
array_size(t.data :transaction :message :instructions) > 1
)
)
{% if is_incremental() %}
AND t._partition_id >= (SELECT max(_partition_id)-1 FROM {{ this }})
AND t._partition_id <= (SELECT max(_partition_id) FROM {{ ref('streamline__complete_block_txs_2') }})
AND t._inserted_timestamp > (SELECT max(_inserted_timestamp) FROM {{ this }})
{% else %}
AND t._partition_id < 0 /* keep this here, if we ever do a full refresh this should select no data from streamline 2.0 data */
{% endif %}
AND t._partition_id >= {{ cutover_partition_id }}
),
{% if is_incremental() %}
prev_null_block_timestamp_txs AS (

12
models/silver/idls/silver__idls.sql
View File

@ -2,7 +2,7 @@
materialized = 'incremental',
unique_key = "program_id",
merge_exclude_columns = ["inserted_timestamp"],
tags = ['scheduled_non_core']
tags = ['idls']
) }}
WITH submitted_idls AS (
@ -11,12 +11,14 @@ WITH submitted_idls AS (
A.program_id,
A.idl,
A.idl_hash,
A.is_valid,
TRUE as is_valid, --all idls from verified_idls are valid
A.discord_username,
A._inserted_timestamp,
a.is_active,
a.last_activity_timestamp,
b.first_block_id
FROM
{{ ref('silver__verified_user_idls') }} A
{{ ref('silver__verified_idls') }} A
LEFT JOIN {{ ref('streamline__idls_history') }}
b
ON A.program_id = b.program_id qualify(ROW_NUMBER() over(PARTITION BY A.program_id
@ -60,6 +62,8 @@ pre_final AS (
A.is_valid,
A.discord_username,
A._inserted_timestamp,
a.is_active,
a.last_activity_timestamp,
A.first_block_id,
{% if is_incremental() %}
iff(b.earliest_decoded_block < d.earliest_decoded_block, b.earliest_decoded_block, d.earliest_decoded_block) AS earliest_decoded_block,
@ -85,6 +89,8 @@ SELECT
idl,
idl_hash,
is_valid,
is_active,
last_activity_timestamp,
discord_username as submitted_by,
_inserted_timestamp as date_submitted,
first_block_id,

72
models/silver/idls/silver__verified_idls.sql
View File

@ -1,17 +1,18 @@
{{ config(
materialized = 'incremental',
unique_key = "program_id",
tags = ['idls','scheduled_non_core']
tags = ['idls']
) }}
WITH user_abis AS (
WITH new_user_idls AS (
SELECT
program_id,
idl,
discord_username,
_inserted_timestamp,
'user' AS idl_source,
idl_hash
idl_hash,
TRUE as is_new_record
FROM
{{ ref('silver__verified_user_idls') }}
WHERE
@ -32,13 +33,64 @@ WITH user_abis AS (
idl_source = 'user'
)
{% endif %}
),
all_idls AS (
{% if is_incremental() %}
-- Include existing records
SELECT
program_id,
idl,
discord_username,
_inserted_timestamp,
idl_source,
idl_hash,
CASE
WHEN _inserted_timestamp >= CURRENT_TIMESTAMP - INTERVAL '2 days' THEN TRUE
ELSE FALSE
END as is_new_record
FROM {{ this }}
UNION ALL
{% endif %}
-- Include new records
SELECT
program_id,
idl,
discord_username,
_inserted_timestamp,
idl_source,
idl_hash,
is_new_record
FROM new_user_idls
),
recent_activity AS (
SELECT
program_id,
MAX(_inserted_timestamp) as last_activity_ts,
COUNT(*) as activity_count
FROM
{{ ref('silver__decoded_instructions_combined') }}
WHERE
_inserted_timestamp >= CURRENT_DATE - 14
GROUP BY
program_id
)
SELECT
program_id,
idl,
_inserted_timestamp,
idl_source,
discord_username,
idl_hash
a.program_id,
a.idl,
a._inserted_timestamp,
a.idl_source,
a.discord_username,
a.idl_hash,
CASE
WHEN a.is_new_record THEN TRUE -- Records from last 2 days are always active
WHEN r.program_id IS NOT NULL AND r.activity_count > 0 THEN TRUE -- Existing records with recent activity
ELSE FALSE -- No recent activity
END as is_active,
COALESCE(r.last_activity_ts, a._inserted_timestamp) as last_activity_timestamp
FROM
user_abis
all_idls a
LEFT JOIN
recent_activity r
ON a.program_id = r.program_id

2
models/silver/idls/silver__verified_user_idls.sql
View File

@ -2,7 +2,7 @@
{{ config (
materialized = "incremental",
unique_key = "id",
tags = ['idls','scheduled_non_core']
tags = ['idls']
) }}
{% if execute %}

1
models/streamline/decode_instructions/streamline__decode_instructions_3_realtime.sql
View File

@ -28,6 +28,7 @@
{{ ref('silver__verified_idls') }}
WHERE
program_id <> 'FsJ3A3u2vn5cTVofAjvy6y5kwABJAqYWpe4975bi2epH'
and is_active
{% endset %}
{% set idls_to_decode = run_query(idls_to_decode_query)[0][0] %}
{% endif %}