update idl table (#537)

* update idl table

* add union for idl_hash change

* col desc

* update per pr comments

* rename

models/descriptions/backfill_status.md

@@ -0,0 +1,5 @@
+{% docs backfill_status%}
+
+Status for the decoding of historical events - complete/in progress/not started
+
+{% enddocs %}

models/descriptions/date_submitted.md

@@ -0,0 +1,5 @@
+{% docs date_submitted %}
+
+Date at which the IDL was submitted.
+
+{% enddocs %}

models/descriptions/earliest_decoded_block.md

@@ -0,0 +1,5 @@
+{% docs earliest_decoded_block %}
+
+The oldest block where events has been decoded
+
+{% enddocs %}

models/descriptions/first_block_id.md

@@ -0,0 +1,5 @@
+{% docs first_block_id %}
+
+The block where the program was first called in a transaction
+
+{% enddocs %}

models/descriptions/is_valid.md

@@ -0,0 +1,5 @@
+{% docs is_valid %}
+
+Whether events are properly decoding with the IDL
+
+{% enddocs %}

models/descriptions/submitted_by.md

@@ -0,0 +1,5 @@
+{% docs submitted_by%}
+
+The discord username for the individual that submitted the IDL, or 'flipside' if submitted by internal team
+
+{% enddocs %}

models/gold/core/core__dim_idl.yml

@@ -1,7 +1,7 @@
 version: 2
 models:
   - name: core__dim_idl
-    description: The status of Program IDL's submitted for decoding events 
+    description: Deprecating soon! This is a notice that we're replacing this table with `core__dim_idls`. Please migrate queries to the new table by 06/06/24.
     columns:
       - name: PROGRAM_ID
         description: "{{ doc('program_id') }}"
@@ -30,4 +30,4 @@ models:
       - name: INSERTED_TIMESTAMP
         description: "{{ doc('inserted_timestamp') }}"
         tests:
-          - dbt_expectations.expect_column_to_exist
+          - dbt_expectations.expect_column_to_exist

models/gold/core/core__dim_idls.sql

@@ -0,0 +1,20 @@
+{{ config(
+    materialized='view',
+    tags = ['scheduled_non_core']
+  ) 
+}}
+
+SELECT
+    program_id,
+    idl,
+    idl_hash,
+    is_valid,
+    submitted_by,
+    date_submitted,
+    first_block_id,
+    earliest_decoded_block,
+    backfill_status,
+    idls_id as dim_idls_id,
+    inserted_timestamp,
+    modified_timestamp
+FROM {{ ref('silver__idls') }}

models/gold/core/core__dim_idls.yml

@@ -0,0 +1,53 @@
+version: 2
+models:
+  - name: core__dim_idls
+    description: The status of Program IDL's submitted for decoding events 
+    columns:
+      - name: PROGRAM_ID
+        description: "{{ doc('program_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: IDL
+        description: "The complete submitted IDL that defines the program"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: IDL_HASH
+        description: "The deployed hash of the program IDL"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: EARLIEST_DECODED_BLOCK
+        description: "{{ doc('earliest_decoded_block') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: IS_VALID
+        description: "{{ doc('is_valid') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: SUBMITTED_BY
+        description: "{{ doc('submitted_by') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: DATE_SUBMITTED
+        description: "{{ doc('date_submitted') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: FIRST_BLOCK_ID
+        description: "{{ doc('first_block_id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: BACKFILL_STATUS
+        description: "{{ doc('backfill_status') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: DIM_IDLS_ID
+        description: "{{ doc('id') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: MODIFIED_TIMESTAMP
+        description: "{{ doc('modified_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist
+      - name: INSERTED_TIMESTAMP
+        description: "{{ doc('inserted_timestamp') }}"
+        tests:
+          - dbt_expectations.expect_column_to_exist

models/silver/idls/silver__idl.sql

@@ -47,4 +47,4 @@ SELECT
     SYSDATE() AS modified_timestamp,
     '{{ invocation_id }}' AS invocation_id
 FROM
-    pre_final
+    pre_final

models/silver/idls/silver__idl.yml

@@ -17,5 +17,4 @@ models:
       - name: IDL_HASH
         description: "The deployed hash of the program IDL"
         tests:
-          - not_null
-
+          - not_null

models/silver/idls/silver__idls.sql

@@ -0,0 +1,104 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = "program_id",
+    merge_exclude_columns = ["inserted_timestamp"],
+    tags = ['scheduled_non_core']
+) }}
+
+WITH submitted_idls AS (
+
+    SELECT
+        A.program_id,
+        A.idl,
+        A.idl_hash,
+        A.is_valid,
+        A.discord_username,
+        A._inserted_timestamp,
+        b.first_block_id
+    FROM
+        {{ ref('silver__verified_user_idls') }} A
+        LEFT JOIN {{ ref('streamline__idls_history') }}
+        b
+        ON A.program_id = b.program_id qualify(ROW_NUMBER() over(PARTITION BY A.program_id
+    ORDER BY
+        A._inserted_timestamp DESC)) = 1
+),
+idl_decoded_history AS (
+    SELECT
+        MIN(block_id) AS earliest_decoded_block,
+        program_id
+    FROM
+        {{ ref('silver__decoded_instructions') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= SYSDATE() - INTERVAL '2 hours'
+{% endif %}
+GROUP BY
+    program_id
+),
+idls_in_progress AS (
+    SELECT
+        table_schema,
+        table_name,
+        SPLIT_PART(
+            table_name,
+            '_',
+            ARRAY_SIZE(SPLIT(table_name, '_'))
+        ) AS in_progress_program_id
+    FROM
+        information_schema.views
+    WHERE
+        table_name LIKE 'DECODED_INSTRUCTIONS_BACKFILL_%'
+        AND table_name NOT LIKE '%RETRY%'
+),
+pre_final AS (
+    SELECT
+        A.program_id,
+        A.idl,
+        A.idl_hash,
+        A.is_valid,
+        A.discord_username,
+        A._inserted_timestamp,
+        A.first_block_id,
+    {% if is_incremental() %}
+        iff(b.earliest_decoded_block < d.earliest_decoded_block, b.earliest_decoded_block, d.earliest_decoded_block) AS earliest_decoded_block,
+    {% else %}
+        b.earliest_decoded_block,
+    {% endif %}
+        C.in_progress_program_id
+    FROM
+        submitted_idls A
+        LEFT JOIN idl_decoded_history b
+        ON A.program_id = b.program_id
+        LEFT JOIN idls_in_progress C
+        ON A.program_id = C.in_progress_program_id
+
+{% if is_incremental() %}
+LEFT JOIN {{ this }}
+d
+ON A.program_id = d.program_id
+    {% endif %}
+)
+SELECT
+    program_id,
+    idl,
+    idl_hash,
+    is_valid,
+    discord_username as submitted_by,
+    _inserted_timestamp as date_submitted,
+    first_block_id,
+    earliest_decoded_block,
+    CASE
+        WHEN earliest_decoded_block = first_block_id THEN 'complete' 
+        WHEN in_progress_program_id IS NOT NULL THEN 'in_progress'
+        WHEN NOT is_valid THEN NULL
+        ELSE 'not_started'
+    END AS backfill_status,
+    {{ dbt_utils.generate_surrogate_key(['program_id']) }} AS idls_id,
+    SYSDATE() AS inserted_timestamp,
+    SYSDATE() AS modified_timestamp,
+    '{{ invocation_id }}' AS invocation_id
+
+FROM
+    pre_final

models/silver/idls/silver__idls.yml

@@ -0,0 +1,29 @@
+version: 2
+models:
+  - name: silver__idls
+    columns:
+      - name: PROGRAM_ID
+        description: "{{ doc('program_id') }}"
+        tests:
+          - not_null
+      - name: EARLIEST_DECODED_BLOCK
+        description: "{{ doc('earliest_decoded_block') }}"
+      - name: IDL
+        description: "The complete IDL that defines the program"
+      - name: IDL_HASH
+        description: "The deployed hash of the program IDL"
+      - name: IS_VALID
+        description: "{{ doc('is_valid') }}"
+        tests:
+          - not_null
+      - name: SUBMITTED_BY
+        description: "{{ doc('submitted_by') }}"
+        tests:
+          - not_null
+      - name: FIRST_BLOCK_ID
+        description: "{{ doc('first_block_id') }}"
+      - name: BACKFILL_STATUS
+        description: "{{ doc('backfill_status') }}"
+      - name: DATE_SUBMITTED
+        description: "{{ doc('backfill_status') }}"
+