Backfill Model - Incorporate all external tables (#211)

* upd sources, convert blocks to loop model, fix collections history

* multiple external tbl query macro and blocks update

* del network version.yml output

* add back rt blocks

* lower blocks

* revert str blocks and collections and move change to history view

* bronze history views and move into folders

* history workflow - blocks

* del history step from streamline blocks

* create complete get history models and upd get history streamline views - BLOCKS

* upd remaining method get history views to use history complete tables

* upd txs history dependency note

* upd history gha and del parameterized models

* fix collections external table config

* collections template fix

* upd txs templates

* upd history views

* fix to tx results template / views

* set history schedule to hourly

* del commented out backfill from streamline RT GHAs

* move blocks filter up to cte

.github/workflows/dbt_run_history.yml

@@ -0,0 +1,34 @@
+name: dbt_run_history
+run-name: dbt_run_history
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs every hour (see https://crontab.guru)
+    - cron: '0 * * * *'
+
+env:
+  USE_VARS: "${{ vars.USE_VARS }}"
+  DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
+  DBT_VERSION: "${{ vars.DBT_VERSION }}"
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  called_workflow_template:
+    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main
+    with:
+      dbt_command: >
+        dbt run -s 2+models/silver/streamline/core/history/ --vars '{"STREAMLINE_INVOKE_STREAMS": True}'
+      environment: workflow_prod
+      warehouse: ${{ vars.WAREHOUSE }}
+    secrets: inherit 

.github/workflows/dbt_run_streamline_blocks.yml

@@ -44,7 +44,3 @@ jobs:
       - name: Run DBT Realtime
         run: |
           dbt run -s 1+streamline__get_blocks_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'
-
-      # - name: Run Backfill Models
-      #   run: |
-      #     dbt run -s models/silver/streamline/core/history/blocks --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'

.github/workflows/dbt_run_streamline_collections.yml

@@ -44,7 +44,3 @@ jobs:
       - name: Run DBT Realtime
         run: |
           dbt run -s 1+streamline__get_collections_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'
-
-      # - name: Run Backfill Models
-      #   run: |
-      #     dbt run -s models/silver/streamline/core/history/collections --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'

.github/workflows/dbt_run_streamline_transaction_results.yml

@@ -44,7 +44,3 @@ jobs:
       - name: Run DBT Realtime
         run: |
           dbt run -s 1+streamline__get_transaction_results_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True, "producer_batch_size": 60000, "worker_batch_size": 2000}'
-
-      # - name: Run Backfill Models
-      #   run: |
-      #     dbt run -s models/silver/streamline/core/history/transaction_results --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'

.github/workflows/dbt_run_streamline_transactions.yml

@@ -44,7 +44,3 @@ jobs:
       - name: Run DBT Realtime
         run: |
           dbt run -s 1+streamline__get_transactions_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'
-
-      # - name: Run Backfill Models
-      #   run: |
-      #     dbt run -s models/silver/streamline/core/history/transactions --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}'

macros/streamline/models.sql

@@ -85,3 +85,68 @@
 {% endmacro %}
 
 
+    {% macro streamline_multiple_external_table_query(
+        table_names,
+        partition_function,
+        partition_name,
+        unique_key
+    )%}
+    WITH 
+    {% for table_name in table_names %}
+        meta_{{ table_name }} AS (
+            SELECT
+                last_modified AS _inserted_timestamp,
+                file_name,
+                {{ partition_function }} AS {{ partition_name }}
+            FROM
+                TABLE(
+                    information_schema.external_table_file_registration_history(
+                        start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()),
+                        table_name => '{{ source( "bronze_streamline", table_name ) }}')
+                    ) A
+            ),
+            {{ table_name }} AS (
+                SELECT
+                    block_number,
+                    {{ unique_key }},
+                    DATA,
+                    _inserted_timestamp,
+                    MD5(
+                        CAST(
+                            COALESCE(CAST(block_number AS text), '' :: STRING) AS text
+                        )
+                    ) AS _fsc_id,
+                    s.{{ partition_name }},
+                    s.value AS VALUE
+                FROM
+                    {{ source(
+                        "bronze_streamline",
+                        table_name
+                    ) }}
+                    s
+                    JOIN meta_{{ table_name }}
+                    b
+                    ON b.file_name = metadata$filename
+                    AND b.{{ partition_name }} = s.{{ partition_name }}
+                WHERE
+                    b.{{ partition_name }} = s.{{ partition_name }}
+            ),
+    {% endfor %}
+
+    FINAL AS ({% for table_name in table_names %}
+        SELECT
+            *
+        FROM
+            {{ table_name }}
+
+        {% if not loop.last %}
+                UNION ALL
+        {% endif %}
+        {% endfor %}
+    )
+    SELECT
+        *
+    FROM
+        FINAL
+
+    {% endmacro %}

models/silver/streamline/README.md

@@ -24,7 +24,7 @@ dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTER
 
 ```zsh 
 # dev bronze__streamline_blocks.sql
-dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' -m 1+models/silver/streamline/bronze/core/bronze__streamline_blocks.sql --profile flow --target dev --profiles-dir ~/.dbt
+dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' -m 1+models/silver/streamline/bronze/core/realtime/bronze__streamline_blocks.sql --profile flow --target dev --profiles-dir ~/.dbt
 ```
 
 ```zsh

models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql

@@ -0,0 +1,16 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+{% set 
+table_names = [
+    'BLOCKS_CANDIDATE_07', 'BLOCKS_CANDIDATE_08', 'BLOCKS_CANDIDATE_09', 'BLOCKS_MAINNET_01', 'BLOCKS_MAINNET_02', 'BLOCKS_MAINNET_03', 'BLOCKS_MAINNET_04', 'BLOCKS_MAINNET_05', 'BLOCKS_MAINNET_06', 'BLOCKS_MAINNET_07', 'BLOCKS_MAINNET_08', 'BLOCKS_MAINNET_09', 'BLOCKS_MAINNET_10', 'BLOCKS_MAINNET_11', 'BLOCKS_MAINNET_12', 'BLOCKS_MAINNET_13', 'BLOCKS_MAINNET_14', 'BLOCKS_MAINNET_15', 'BLOCKS_MAINNET_16', 'BLOCKS_MAINNET_17', 'BLOCKS_MAINNET_18', 'BLOCKS_MAINNET_19', 'BLOCKS_MAINNET_20', 'BLOCKS_MAINNET_21', 'BLOCKS_MAINNET_22'
+]
+%}
+
+{{ streamline_multiple_external_table_query(
+    table_names,
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
+    partition_name = "_partition_by_block_id",
+    unique_key = "id"
+) }}

models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql

@@ -0,0 +1,17 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+{% set 
+    table_names = 
+    [
+        'COLLECTIONS_CANDIDATE_07', 'COLLECTIONS_CANDIDATE_08', 'COLLECTIONS_CANDIDATE_09', 'COLLECTIONS_MAINNET_01', 'COLLECTIONS_MAINNET_02', 'COLLECTIONS_MAINNET_03', 'COLLECTIONS_MAINNET_04', 'COLLECTIONS_MAINNET_05', 'COLLECTIONS_MAINNET_06', 'COLLECTIONS_MAINNET_07', 'COLLECTIONS_MAINNET_08', 'COLLECTIONS_MAINNET_09', 'COLLECTIONS_MAINNET_10', 'COLLECTIONS_MAINNET_11', 'COLLECTIONS_MAINNET_12', 'COLLECTIONS_MAINNET_13', 'COLLECTIONS_MAINNET_14', 'COLLECTIONS_MAINNET_15', 'COLLECTIONS_MAINNET_16', 'COLLECTIONS_MAINNET_17', 'COLLECTIONS_MAINNET_18', 'COLLECTIONS_MAINNET_19', 'COLLECTIONS_MAINNET_20', 'COLLECTIONS_MAINNET_21', 'COLLECTIONS_MAINNET_22'
+    ]
+%}
+
+{{ streamline_multiple_external_table_query(
+    table_names,
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
+    partition_name = "_partition_by_block_id",
+    unique_key = "id"
+) }}

models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql

@@ -0,0 +1,17 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+{% set 
+    table_names = 
+    [
+        'TRANSACTION_RESULTS_CANDIDATE_07', 'TRANSACTION_RESULTS_CANDIDATE_08', 'TRANSACTION_RESULTS_CANDIDATE_09', 'TRANSACTION_RESULTS_MAINNET_01', 'TRANSACTION_RESULTS_MAINNET_02', 'TRANSACTION_RESULTS_MAINNET_03', 'TRANSACTION_RESULTS_MAINNET_04', 'TRANSACTION_RESULTS_MAINNET_05', 'TRANSACTION_RESULTS_MAINNET_06', 'TRANSACTION_RESULTS_MAINNET_07', 'TRANSACTION_RESULTS_MAINNET_08', 'TRANSACTION_RESULTS_MAINNET_09', 'TRANSACTION_RESULTS_MAINNET_10', 'TRANSACTION_RESULTS_MAINNET_11', 'TRANSACTION_RESULTS_MAINNET_12', 'TRANSACTION_RESULTS_MAINNET_13', 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_20', 'TRANSACTION_RESULTS_MAINNET_21', 'TRANSACTION_RESULTS_MAINNET_22'
+    ]
+%}
+
+{{ streamline_multiple_external_table_query(
+    table_names,
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
+    partition_name = "_partition_by_block_id",
+    unique_key = "id"
+) }}

models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql

@@ -0,0 +1,17 @@
+{{ config (
+    materialized = 'view'
+) }}
+
+{% set 
+    table_names = 
+    [
+        'TRANSACTIONS_CANDIDATE_07', 'TRANSACTIONS_CANDIDATE_08', 'TRANSACTIONS_CANDIDATE_09', 'TRANSACTIONS_MAINNET_01', 'TRANSACTIONS_MAINNET_02', 'TRANSACTIONS_MAINNET_03', 'TRANSACTIONS_MAINNET_04', 'TRANSACTIONS_MAINNET_05', 'TRANSACTIONS_MAINNET_06', 'TRANSACTIONS_MAINNET_07', 'TRANSACTIONS_MAINNET_08', 'TRANSACTIONS_MAINNET_09', 'TRANSACTIONS_MAINNET_10', 'TRANSACTIONS_MAINNET_11', 'TRANSACTIONS_MAINNET_12', 'TRANSACTIONS_MAINNET_13', 'TRANSACTIONS_MAINNET_14', 'TRANSACTIONS_MAINNET_15', 'TRANSACTIONS_MAINNET_16', 'TRANSACTIONS_MAINNET_17', 'TRANSACTIONS_MAINNET_18', 'TRANSACTIONS_MAINNET_19', 'TRANSACTIONS_MAINNET_20', 'TRANSACTIONS_MAINNET_21', 'TRANSACTIONS_MAINNET_22'
+    ]
+%}
+
+{{ streamline_multiple_external_table_query(
+    table_names,
+    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)",
+    partition_name = "_partition_by_block_id",
+    unique_key = "id"
+) }}

models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql

@@ -0,0 +1,36 @@
+-- depends_on: {{ ref('bronze__streamline_blocks_history') }}
+{{ config (
+    materialized = "incremental",
+    unique_key = "block_number",
+    cluster_by = "ROUND(block_number, -3)",
+    merge_update_columns = ["block_number"],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)",
+    tags = ['streamline_history']
+) }}
+
+SELECT
+    DATA,
+    block_number,
+    _partition_by_block_id,
+    _inserted_timestamp
+FROM
+
+{{ ref('bronze__streamline_blocks_history') }}
+
+WHERE
+    TRUE
+{% if is_incremental() %}
+    AND _inserted_timestamp >= COALESCE(
+        (
+            SELECT
+                MAX(_inserted_timestamp) _inserted_timestamp
+            FROM
+                {{ this }}
+        ),
+        '1900-01-01' :: timestamp_ntz
+    )
+{% endif %}
+ 
+qualify(ROW_NUMBER() over (PARTITION BY block_number
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql

@@ -0,0 +1,36 @@
+-- depends_on: {{ ref('bronze__streamline_collections_history') }}
+{{ config (
+    materialized = "incremental",
+    unique_key = "id",
+    cluster_by = "ROUND(block_number, -3)",
+    merge_update_columns = ["id"],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)",
+    tags = ['streamline_history']
+) }}
+
+SELECT
+    id,
+    data,
+    block_number,
+    _partition_by_block_id,
+    _inserted_timestamp
+FROM
+
+{{ ref('bronze__streamline_collections_history') }} 
+WHERE
+    TRUE
+{% if is_incremental() %}
+AND _inserted_timestamp >= COALESCE(
+        (
+            SELECT
+            MAX(_inserted_timestamp) _inserted_timestamp
+            FROM
+                {{ this }}
+        ),
+        '1900-01-01'::timestamp_ntz
+    )
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql

@@ -0,0 +1,36 @@
+-- depends_on: {{ ref('bronze__streamline_transaction_results_history') }}
+{{ config (
+    materialized = "incremental",
+    unique_key = "id",
+    cluster_by = "ROUND(block_number, -3)",
+    merge_update_columns = ["id"],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)",
+    tags = ['streamline_history']
+) }}
+
+SELECT
+    id,
+    DATA,
+    block_number,
+    _partition_by_block_id,
+    _inserted_timestamp
+FROM
+
+{{ ref('bronze__streamline_transaction_results_history') }}
+WHERE
+    TRUE
+{% if is_incremental() %}
+    AND _inserted_timestamp >= COALESCE(
+        (
+            SELECT
+                MAX(_inserted_timestamp) _inserted_timestamp
+            FROM
+                {{ this }}
+        ),
+        '1900-01-01' :: timestamp_ntz
+    )
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql

@@ -0,0 +1,36 @@
+-- depends_on: {{ ref('bronze__streamline_transactions_history') }}
+{{ config (
+    materialized = "incremental",
+    unique_key = "id",
+    cluster_by = "ROUND(block_number, -3)",
+    merge_update_columns = ["id"],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)",
+    tags = ['streamline_history']
+) }}
+
+SELECT
+    id,
+    data,
+    block_number,
+    _partition_by_block_id,
+    _inserted_timestamp
+FROM
+
+{{ ref('bronze__streamline_transactions_history') }} 
+WHERE
+    TRUE
+{% if is_incremental() %}
+    AND _inserted_timestamp >= COALESCE(
+        (
+            SELECT
+            MAX(_inserted_timestamp) _inserted_timestamp
+            FROM
+                {{ this }}
+        ),
+        '1900-01-01'::timestamp_ntz
+    )
+{% endif %}
+
+qualify(ROW_NUMBER() over (PARTITION BY id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'blocks_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 4132133
+        AND 4972986
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 4132133
+        AND 4972986
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 4132133
-    AND 4972986
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'blocks_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 4972987
+        AND 6483245
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 4972987
+        AND 6483245
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 4972987
-    AND 6483245
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'blocks_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 6483246
+        AND 7601062
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 6483246
+        AND 7601062
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 6483246
-    AND 7601062
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'blocks_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 7601063
+        AND 8742958
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 7601063
+        AND 8742958
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 7601063
-    AND 8742958
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'blocks_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 8742959
+        AND 9737132
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 8742959
+        AND 9737132
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 8742959
-    AND 9737132
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'blocks_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 9737133
+        AND 9992019
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 9737133
+        AND 9992019
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 9737133
-    AND 9992019
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'blocks_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 9992020
+        AND 12020336
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 9992020
+        AND 12020336
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 9992020
-    AND 12020336
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'blocks_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 12020337
+        AND 12609236
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 12020337
+        AND 12609236
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 12020337
-    AND 12609236
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'blocks_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 12609237
+        AND 13404173
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 12609237
+        AND 13404173
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 12609237
-    AND 13404173
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'blocks_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 13404174
+        AND 13950741
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 13404174
+        AND 13950741
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 13404174
-    AND 13950741
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'blocks_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 13950742
+        AND 14892103
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 13950742
+        AND 14892103
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 13950742
-    AND 14892103
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'blocks_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 14892104
+        AND 15791890
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 14892104
+        AND 15791890
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 14892104
-    AND 15791890
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'blocks_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 15791891
+        AND 16755601
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 15791891
+        AND 16755601
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 15791891
-    AND 16755601
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'blocks_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 16755602
+        AND 17544522
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 16755602
+        AND 17544522
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 16755602
-    AND 17544522
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'blocks_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 17544523
+        AND 18587477
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 17544523
+        AND 18587477
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 17544523
-    AND 18587477
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'blocks_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 18587478
+        AND 19050752
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 18587478
+        AND 19050752
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 18587478
-    AND 19050752
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'blocks_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 19050753
+        AND 21291691
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 19050753
+        AND 21291691
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 19050753
-    AND 21291691
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'blocks_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 21291692
+        AND 23830812
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 21291692
+        AND 23830812
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 21291692
-    AND 23830812
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'blocks_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 23830813
+        AND 27341469
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 23830813
+        AND 27341469
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 23830813
-    AND 27341469
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'blocks_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 27341470
+        AND 31735954
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 27341470
+        AND 31735954
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 27341470
-    AND 31735954
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'blocks_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 31735955
+        AND 35858810
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 31735955
+        AND 35858810
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 31735955
-    AND 35858810
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'blocks_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 35858811
+        AND 40171633
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 35858811
+        AND 40171633
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 35858811
-    AND 40171633
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'blocks_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 40171634
+        AND 44950206
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 40171634
+        AND 44950206
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 40171634
-    AND 44950206
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'blocks_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 44950207
+        AND 47169686
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 44950207
+        AND 47169686
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 44950207
-    AND 47169686
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql

@@ -3,7 +3,8 @@
     post_hook = if_data_call_function(
         func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'blocks_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
-    )
+    ),
+    tags = ['streamline_history']
 ) }}
 
 WITH blocks AS (
@@ -12,23 +13,33 @@ WITH blocks AS (
         block_height
     FROM
         {{ ref("streamline__blocks") }}
+    WHERE
+        block_height BETWEEN 47169687
+        AND 55114466
     EXCEPT
     SELECT
         block_number AS block_height
     FROM
-        {{ ref("streamline__complete_get_blocks") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+    WHERE
+        block_height BETWEEN 47169687
+        AND 55114466
 )
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
+        'grpc',
+        'proto3',
+        'method',
+        'get_block_by_height',
+        'block_height',
+        block_height,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'height',
+            block_height
+        )
     ) AS request
 FROM
     blocks
-WHERE
-    block_height BETWEEN 47169687
-    AND 55114466
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'transactions_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'collections_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 4132133
+        AND 4972986
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 4132133
-    AND 4972986
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'transactions_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'collections_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 4972987
+        AND 6483245
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 4972987
-    AND 6483245
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'transactions_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'collections_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 6483246
+        AND 7601062
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 6483246
-    AND 7601062
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'transactions_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'collections_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 7601063
+        AND 8742958
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 7601063
-    AND 8742958
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'transactions_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'collections_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 8742959
+        AND 9737132
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 8742959
-    AND 9737132
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'transactions_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'collections_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 9737133
+        AND 9992019
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 9737133
-    AND 9992019
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'transactions_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'collections_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 9992020
+        AND 12020336
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 9992020
-    AND 12020336
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'transactions_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'collections_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 12020337
+        AND 12609236
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 12020337
-    AND 12609236
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'transactions_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'collections_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 12609237
+        AND 13404173
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 12609237
-    AND 13404173
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'transactions_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'collections_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 13404174
+        AND 13950741
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 13404174
-    AND 13950741
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'transactions_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'collections_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 13950742
+        AND 14892103
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 13950742
-    AND 14892103
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'transactions_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'collections_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 14892104
+        AND 15791890
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 14892104
-    AND 15791890
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'transactions_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'collections_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 15791891
+        AND 16755601
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 15791891
-    AND 16755601
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'transactions_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'collections_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 16755602
+        AND 17544522
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 16755602
-    AND 17544522
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'transactions_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'collections_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 17544523
+        AND 18587477
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 17544523
-    AND 18587477
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'transactions_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'collections_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 18587478
+        AND 19050752
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 18587478
-    AND 19050752
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transactions_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'collections_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 19050753
+        AND 21291691
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 19050753
-    AND 21291691
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transactions_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'collections_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 21291692
+        AND 23830812
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 21291692
-    AND 23830812
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transactions_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'collections_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 23830813
+        AND 27341469
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 23830813
-    AND 27341469
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transactions_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'collections_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 27341470
+        AND 31735954
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 27341470
-    AND 31735954
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transactions_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'collections_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 31735955
+        AND 35858810
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 31735955
-    AND 35858810
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transactions_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'collections_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 35858811
+        AND 40171633
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 35858811
-    AND 40171633
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'transactions_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'collections_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 40171634
+        AND 44950206
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 40171634
-    AND 44950206
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'transactions_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'collections_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 44950207
+        AND 47169686
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 44950207
-    AND 47169686
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql

@@ -1,45 +1,57 @@
 {{ config (
     materialized = "view",
     post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transactions_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
+        func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'collections_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
         target = "{{this.schema}}.{{this.identifier}}"
     )
 ) }}
 
-WITH blocks AS (
+WITH 
+-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table
+block_collections AS (
 
     SELECT
-        block_height
+        cb.block_number AS block_height,
+        collection_guarantee.value :collection_id :: STRING AS collection_id
     FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
+        {{ ref("streamline__complete_get_blocks_history") }}
+        cb,
+        LATERAL FLATTEN(
+            input => cb.data :collection_guarantees
+        ) AS collection_guarantee
+    WHERE
+        block_height BETWEEN 47169687
+        AND 55114466
 ),
-tx AS (
+-- CTE to identify collections that haven't been ingested yet
+collections_to_ingest AS (
     SELECT
-        block_number AS block_height,
-        DATA
+        bc.block_height,
+        bc.collection_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        block_collections bc
+        LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C
+        ON bc.block_height = C.block_number
+        AND bc.collection_id = C.id
+    WHERE
+        C.id IS NULL
+) 
+-- Generate the requests based on the missing collections
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_collection_by_i_d',
+        'block_height',
+        block_height :: INTEGER,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            collection_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 47169687
-    AND 55114466
+    collections_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql

@@ -1,34 +0,0 @@
-{{ config(
-    materialized = "view",
-    post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url', '{{ var('node_url', Null) }}','external_table', 'blocks', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
-        target = "{{this.schema}}.{{this.identifier}}"
-    )
-) }}
-
-WITH blocks AS (
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number as block_height
-    FROM
-        {{ ref("streamline__complete_get_blocks") }}
-)
-SELECT
-    OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_block_by_height',
-        'block_height', block_height,
-        'method_params', OBJECT_CONSTRUCT('height', block_height)
-    ) AS request
-FROM
-    blocks
-WHERE
-    block_height BETWEEN {{ var('start_block', Null) }} AND {{ var('end_block', Null) }}
-ORDER BY
-    block_height ASC
-LIMIT
-    1000000

models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql

@@ -1,77 +0,0 @@
-{{ config (
-    materialized = "view",
-    post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','{{ var('node_url', Null) }}','external_table', 'collections', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
-        target = "{{this.schema}}.{{this.identifier}}"
-    )
-) }}
-
-WITH blocks AS (
-
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_collections") }}
-),
-collections AS (
-    SELECT
-        block_number AS block_height,
-        DATA
-    FROM
-        {{ ref('streamline__complete_get_blocks') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-),
--- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks table
-block_collections AS (
-    SELECT
-        cb.block_number AS block_height,
-        collection_guarantee.value :collection_id AS collection_id
-    FROM
-        {{ ref("streamline__complete_get_blocks") }}
-        cb,
-        LATERAL FLATTEN(
-            input => cb.data :collection_guarantees
-        ) AS collection_guarantee
-),
--- CTE to identify collections that haven't been ingested yet
-collections_to_ingest AS (
-    SELECT
-        bc.block_height,
-        bc.collection_id
-    FROM
-        block_collections bc
-        LEFT JOIN {{ ref("streamline__complete_get_collections") }} C
-        ON bc.block_height = C.block_number
-        AND bc.collection_id = C.id
-    WHERE
-        C.id IS NULL
-) -- Generate the requests based on the missing collections
-SELECT
-    OBJECT_CONSTRUCT(
-        'grpc',
-        'proto3',
-        'method',
-        'get_collection_by_i_d',
-        'block_height',
-        block_height :: INTEGER,
-        'method_params',
-        OBJECT_CONSTRUCT(
-            'id',
-            collection_id
-        )
-    ) AS request
-FROM
-    collections_to_ingest
-WHERE
-    block_height BETWEEN {{ var('start_block', Null) }}
-    AND {{ var('end_block', Null) }}
-ORDER BY
-    block_height ASC
-LIMIT
-    1000000

models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql

@@ -1,44 +0,0 @@
-{{ config (
-    materialized = "view",
-    post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}', 'node_url', '{{ var('node_url', Null) }}', 'external_table', 'transaction_results', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','30000')}}, 'worker_batch_size', {{var('worker_batch_size','3000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
-        target = "{{this.schema}}.{{this.identifier}}"
-    )
-) }}
-
-WITH blocks AS (
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number as block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
-    SELECT
-        block_number as block_height,
-        data
-    FROM
-        {{ ref('streamline__complete_get_collections') }}
-    JOIN blocks ON blocks.block_height = block_number
-)
-SELECT
-    OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id',  transaction_id.value::string)
-    ) AS request
-FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN {{ var('start_block', Null) }} AND {{ var('end_block', Null) }}
-ORDER BY
-    block_height ASC
-LIMIT
-    1000000

models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql

@@ -1,44 +0,0 @@
-{{ config (
-    materialized = "view",
-    post_hook = if_data_call_function(
-        func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}', 'node_url', '{{ var('node_url', Null) }}', 'external_table', 'transactions', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','30000')}}, 'worker_batch_size', {{var('worker_batch_size','3000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))",
-        target = "{{this.schema}}.{{this.identifier}}"
-    )
-) }}
-
-WITH blocks AS (
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number as block_height
-    FROM
-        {{ ref("streamline__complete_get_transactions") }}
-),
-tx AS (
-    SELECT
-        block_number as block_height,
-        data
-    FROM
-        {{ ref('streamline__complete_get_collections') }}
-    JOIN blocks ON blocks.block_height = block_number
-)
-SELECT
-    OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id',  transaction_id.value::string)
-    ) AS request
-FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN {{ var('start_block', Null) }} AND {{ var('end_block', Null) }}
-ORDER BY
-    block_height ASC
-LIMIT
-    1000000

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 4132133
+        AND 4972986
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 4132133
-    AND 4972986
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 4972987
+        AND 6483245
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 4972987
-    AND 6483245
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 6483246
+        AND 7601062
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 6483246
-    AND 7601062
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 7601063
+        AND 8742958
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 7601063
-    AND 8742958
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 8742959
+        AND 9737132
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 8742959
-    AND 9737132
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 9737133
+        AND 9992019
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 9737133
-    AND 9992019
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 9992020
+        AND 12020336
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 9992020
-    AND 12020336
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 12020337
+        AND 12609236
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 12020337
-    AND 12609236
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 12609237
+        AND 13404173
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 12609237
-    AND 13404173
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 13404174
+        AND 13950741
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 13404174
-    AND 13950741
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 13950742
+        AND 14892103
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 13950742
-    AND 14892103
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 14892104
+        AND 15791890
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 14892104
-    AND 15791890
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 15791891
+        AND 16755601
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 15791891
-    AND 16755601
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 16755602
+        AND 17544522
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 16755602
-    AND 17544522
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 17544523
+        AND 18587477
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 17544523
-    AND 18587477
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 18587478
+        AND 19050752
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 18587478
-    AND 19050752
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 19050753
+        AND 21291691
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 19050753
-    AND 21291691
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 21291692
+        AND 23830812
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 21291692
-    AND 23830812
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 23830813
+        AND 27341469
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 23830813
-    AND 27341469
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 27341470
+        AND 31735954
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 27341470
-    AND 31735954
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 31735955
+        AND 35858810
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 31735955
-    AND 35858810
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 35858811
+        AND 40171633
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 35858811
-    AND 40171633
+    transactions_to_ingest
 ORDER BY
     block_height ASC

models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql

@@ -6,40 +6,51 @@
     )
 ) }}
 
-WITH blocks AS (
+WITH collection_transactions AS (
 
-    SELECT
-        block_height
-    FROM
-        {{ ref("streamline__blocks") }}
-    EXCEPT
-    SELECT
-        block_number AS block_height
-    FROM
-        {{ ref("streamline__complete_get_transaction_results") }}
-),
-tx AS (
     SELECT
         block_number AS block_height,
-        DATA
+        TRANSACTION.value :: STRING AS transaction_id
     FROM
-        {{ ref('streamline__complete_get_collections') }}
-        JOIN blocks
-        ON blocks.block_height = block_number
-)
+        {{ ref('streamline__complete_get_collections_history') }}
+        cch,
+        LATERAL FLATTEN(
+            input => cch.data :transaction_ids
+        ) AS TRANSACTION
+    WHERE
+        block_height BETWEEN 40171634
+        AND 44950206
+),
+-- CTE to identify transactions that haven't been ingested yet
+transactions_to_ingest AS (
+    SELECT
+        ct.block_height,
+        ct.transaction_id
+    FROM
+        collection_transactions ct
+        LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }}
+        t
+        ON ct.transaction_id = t.id
+    WHERE
+        t.id IS NULL
+) -- Generate the requests based on the missing transactions
 SELECT
     OBJECT_CONSTRUCT(
-        'grpc', 'proto3',
-        'method', 'get_transaction_result',
-        'block_height', block_height::INTEGER,
-        'transaction_id', transaction_id.value::string,
-        'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING)
+        'grpc',
+        'proto3',
+        'method',
+        'get_transaction_result',
+        'block_height',
+        block_height :: INTEGER,
+        'transaction_id',
+        transaction_id,
+        'method_params',
+        OBJECT_CONSTRUCT(
+            'id',
+            transaction_id
+        )
     ) AS request
 FROM
-    tx,
-    LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id
-WHERE
-    block_height BETWEEN 40171634
-    AND 44950206
+    transactions_to_ingest
 ORDER BY
     block_height ASC