From 1fbf84441f34d733ca129996f0e07b1390c04734 Mon Sep 17 00:00:00 2001 From: tarikceric <46071768+tarikceric@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:12:16 -0700 Subject: [PATCH] update idl table (#537) * update idl table * add union for idl_hash change * col desc * update per pr comments * rename --- models/descriptions/backfill_status.md | 5 + models/descriptions/date_submitted.md | 5 + models/descriptions/earliest_decoded_block.md | 5 + models/descriptions/first_block_id.md | 5 + models/descriptions/is_valid.md | 5 + models/descriptions/submitted_by.md | 5 + models/gold/core/core__dim_idl.yml | 4 +- models/gold/core/core__dim_idls.sql | 20 ++++ models/gold/core/core__dim_idls.yml | 53 +++++++++ models/silver/idls/silver__idl.sql | 2 +- models/silver/idls/silver__idl.yml | 3 +- models/silver/idls/silver__idls.sql | 104 ++++++++++++++++++ models/silver/idls/silver__idls.yml | 29 +++++ 13 files changed, 240 insertions(+), 5 deletions(-) create mode 100644 models/descriptions/backfill_status.md create mode 100644 models/descriptions/date_submitted.md create mode 100644 models/descriptions/earliest_decoded_block.md create mode 100644 models/descriptions/first_block_id.md create mode 100644 models/descriptions/is_valid.md create mode 100644 models/descriptions/submitted_by.md create mode 100644 models/gold/core/core__dim_idls.sql create mode 100644 models/gold/core/core__dim_idls.yml create mode 100644 models/silver/idls/silver__idls.sql create mode 100644 models/silver/idls/silver__idls.yml diff --git a/models/descriptions/backfill_status.md b/models/descriptions/backfill_status.md new file mode 100644 index 00000000..0ee551bb --- /dev/null +++ b/models/descriptions/backfill_status.md @@ -0,0 +1,5 @@ +{% docs backfill_status%} + +Status for the decoding of historical events - complete/in progress/not started + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/date_submitted.md b/models/descriptions/date_submitted.md new file mode 100644 index 00000000..4af6122a --- /dev/null +++ b/models/descriptions/date_submitted.md @@ -0,0 +1,5 @@ +{% docs date_submitted %} + +Date at which the IDL was submitted. + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/earliest_decoded_block.md b/models/descriptions/earliest_decoded_block.md new file mode 100644 index 00000000..5bdd9bf4 --- /dev/null +++ b/models/descriptions/earliest_decoded_block.md @@ -0,0 +1,5 @@ +{% docs earliest_decoded_block %} + +The oldest block where events has been decoded + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/first_block_id.md b/models/descriptions/first_block_id.md new file mode 100644 index 00000000..245f794c --- /dev/null +++ b/models/descriptions/first_block_id.md @@ -0,0 +1,5 @@ +{% docs first_block_id %} + +The block where the program was first called in a transaction + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/is_valid.md b/models/descriptions/is_valid.md new file mode 100644 index 00000000..16084438 --- /dev/null +++ b/models/descriptions/is_valid.md @@ -0,0 +1,5 @@ +{% docs is_valid %} + +Whether events are properly decoding with the IDL + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/submitted_by.md b/models/descriptions/submitted_by.md new file mode 100644 index 00000000..a2944937 --- /dev/null +++ b/models/descriptions/submitted_by.md @@ -0,0 +1,5 @@ +{% docs submitted_by%} + +The discord username for the individual that submitted the IDL, or 'flipside' if submitted by internal team + +{% enddocs %} \ No newline at end of file diff --git a/models/gold/core/core__dim_idl.yml b/models/gold/core/core__dim_idl.yml index 6358d3d0..031ce75f 100644 --- a/models/gold/core/core__dim_idl.yml +++ b/models/gold/core/core__dim_idl.yml @@ -1,7 +1,7 @@ version: 2 models: - name: core__dim_idl - description: The status of Program IDL's submitted for decoding events + description: Deprecating soon! This is a notice that we're replacing this table with `core__dim_idls`. Please migrate queries to the new table by 06/06/24. columns: - name: PROGRAM_ID description: "{{ doc('program_id') }}" @@ -30,4 +30,4 @@ models: - name: INSERTED_TIMESTAMP description: "{{ doc('inserted_timestamp') }}" tests: - - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/gold/core/core__dim_idls.sql b/models/gold/core/core__dim_idls.sql new file mode 100644 index 00000000..2f12b373 --- /dev/null +++ b/models/gold/core/core__dim_idls.sql @@ -0,0 +1,20 @@ +{{ config( + materialized='view', + tags = ['scheduled_non_core'] + ) +}} + +SELECT + program_id, + idl, + idl_hash, + is_valid, + submitted_by, + date_submitted, + first_block_id, + earliest_decoded_block, + backfill_status, + idls_id as dim_idls_id, + inserted_timestamp, + modified_timestamp +FROM {{ ref('silver__idls') }} \ No newline at end of file diff --git a/models/gold/core/core__dim_idls.yml b/models/gold/core/core__dim_idls.yml new file mode 100644 index 00000000..c1f93b8b --- /dev/null +++ b/models/gold/core/core__dim_idls.yml @@ -0,0 +1,53 @@ +version: 2 +models: + - name: core__dim_idls + description: The status of Program IDL's submitted for decoding events + columns: + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: IDL + description: "The complete submitted IDL that defines the program" + tests: + - dbt_expectations.expect_column_to_exist + - name: IDL_HASH + description: "The deployed hash of the program IDL" + tests: + - dbt_expectations.expect_column_to_exist + - name: EARLIEST_DECODED_BLOCK + description: "{{ doc('earliest_decoded_block') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: IS_VALID + description: "{{ doc('is_valid') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: SUBMITTED_BY + description: "{{ doc('submitted_by') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: DATE_SUBMITTED + description: "{{ doc('date_submitted') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: FIRST_BLOCK_ID + description: "{{ doc('first_block_id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: BACKFILL_STATUS + description: "{{ doc('backfill_status') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: DIM_IDLS_ID + description: "{{ doc('id') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + tests: + - dbt_expectations.expect_column_to_exist diff --git a/models/silver/idls/silver__idl.sql b/models/silver/idls/silver__idl.sql index 52e92b40..bd15e1b1 100644 --- a/models/silver/idls/silver__idl.sql +++ b/models/silver/idls/silver__idl.sql @@ -47,4 +47,4 @@ SELECT SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS invocation_id FROM - pre_final + pre_final \ No newline at end of file diff --git a/models/silver/idls/silver__idl.yml b/models/silver/idls/silver__idl.yml index 331b124d..d88878a2 100644 --- a/models/silver/idls/silver__idl.yml +++ b/models/silver/idls/silver__idl.yml @@ -17,5 +17,4 @@ models: - name: IDL_HASH description: "The deployed hash of the program IDL" tests: - - not_null - + - not_null \ No newline at end of file diff --git a/models/silver/idls/silver__idls.sql b/models/silver/idls/silver__idls.sql new file mode 100644 index 00000000..fd2df12f --- /dev/null +++ b/models/silver/idls/silver__idls.sql @@ -0,0 +1,104 @@ +{{ config( + materialized = 'incremental', + unique_key = "program_id", + merge_exclude_columns = ["inserted_timestamp"], + tags = ['scheduled_non_core'] +) }} + +WITH submitted_idls AS ( + + SELECT + A.program_id, + A.idl, + A.idl_hash, + A.is_valid, + A.discord_username, + A._inserted_timestamp, + b.first_block_id + FROM + {{ ref('silver__verified_user_idls') }} A + LEFT JOIN {{ ref('streamline__idls_history') }} + b + ON A.program_id = b.program_id qualify(ROW_NUMBER() over(PARTITION BY A.program_id + ORDER BY + A._inserted_timestamp DESC)) = 1 +), +idl_decoded_history AS ( + SELECT + MIN(block_id) AS earliest_decoded_block, + program_id + FROM + {{ ref('silver__decoded_instructions') }} + +{% if is_incremental() %} +WHERE + _inserted_timestamp >= SYSDATE() - INTERVAL '2 hours' +{% endif %} +GROUP BY + program_id +), +idls_in_progress AS ( + SELECT + table_schema, + table_name, + SPLIT_PART( + table_name, + '_', + ARRAY_SIZE(SPLIT(table_name, '_')) + ) AS in_progress_program_id + FROM + information_schema.views + WHERE + table_name LIKE 'DECODED_INSTRUCTIONS_BACKFILL_%' + AND table_name NOT LIKE '%RETRY%' +), +pre_final AS ( + SELECT + A.program_id, + A.idl, + A.idl_hash, + A.is_valid, + A.discord_username, + A._inserted_timestamp, + A.first_block_id, + {% if is_incremental() %} + iff(b.earliest_decoded_block < d.earliest_decoded_block, b.earliest_decoded_block, d.earliest_decoded_block) AS earliest_decoded_block, + {% else %} + b.earliest_decoded_block, + {% endif %} + C.in_progress_program_id + FROM + submitted_idls A + LEFT JOIN idl_decoded_history b + ON A.program_id = b.program_id + LEFT JOIN idls_in_progress C + ON A.program_id = C.in_progress_program_id + +{% if is_incremental() %} +LEFT JOIN {{ this }} +d +ON A.program_id = d.program_id + {% endif %} +) +SELECT + program_id, + idl, + idl_hash, + is_valid, + discord_username as submitted_by, + _inserted_timestamp as date_submitted, + first_block_id, + earliest_decoded_block, + CASE + WHEN earliest_decoded_block = first_block_id THEN 'complete' + WHEN in_progress_program_id IS NOT NULL THEN 'in_progress' + WHEN NOT is_valid THEN NULL + ELSE 'not_started' + END AS backfill_status, + {{ dbt_utils.generate_surrogate_key(['program_id']) }} AS idls_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id + +FROM + pre_final diff --git a/models/silver/idls/silver__idls.yml b/models/silver/idls/silver__idls.yml new file mode 100644 index 00000000..4b0d74a9 --- /dev/null +++ b/models/silver/idls/silver__idls.yml @@ -0,0 +1,29 @@ +version: 2 +models: + - name: silver__idls + columns: + - name: PROGRAM_ID + description: "{{ doc('program_id') }}" + tests: + - not_null + - name: EARLIEST_DECODED_BLOCK + description: "{{ doc('earliest_decoded_block') }}" + - name: IDL + description: "The complete IDL that defines the program" + - name: IDL_HASH + description: "The deployed hash of the program IDL" + - name: IS_VALID + description: "{{ doc('is_valid') }}" + tests: + - not_null + - name: SUBMITTED_BY + description: "{{ doc('submitted_by') }}" + tests: + - not_null + - name: FIRST_BLOCK_ID + description: "{{ doc('first_block_id') }}" + - name: BACKFILL_STATUS + description: "{{ doc('backfill_status') }}" + - name: DATE_SUBMITTED + description: "{{ doc('backfill_status') }}" +