From eb40b6d91d8d30a4a5e0f85acd3dd31718131573 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 26 Oct 2023 11:31:19 -0600 Subject: [PATCH] Backfill Model - Incorporate all external tables (#211) * upd sources, convert blocks to loop model, fix collections history * multiple external tbl query macro and blocks update * del network version.yml output * add back rt blocks * lower blocks * revert str blocks and collections and move change to history view * bronze history views and move into folders * history workflow - blocks * del history step from streamline blocks * create complete get history models and upd get history streamline views - BLOCKS * upd remaining method get history views to use history complete tables * upd txs history dependency note * upd history gha and del parameterized models * fix collections external table config * collections template fix * upd txs templates * upd history views * fix to tx results template / views * set history schedule to hourly * del commented out backfill from streamline RT GHAs * move blocks filter up to cte --- .github/workflows/dbt_run_history.yml | 34 ++++++ .../workflows/dbt_run_streamline_blocks.yml | 4 - .../dbt_run_streamline_collections.yml | 4 - ...dbt_run_streamline_transaction_results.yml | 4 - .../dbt_run_streamline_transactions.yml | 4 - macros/streamline/models.sql | 65 +++++++++++ models/silver/streamline/README.md | 2 +- .../bronze__streamline_blocks_history.sql | 16 +++ ...bronze__streamline_collections_history.sql | 17 +++ ...streamline_transaction_results_history.sql | 17 +++ ...ronze__streamline_transactions_history.sql | 17 +++ .../bronze__streamline_blocks.sql | 0 .../bronze__streamline_collections.sql | 0 .../bronze__streamline_fr_blocks.sql | 0 .../bronze__streamline_fr_collections.sql | 0 ...nze__streamline_fr_transaction_results.sql | 0 .../bronze__streamline_fr_transactions.sql | 0 ...bronze__streamline_transaction_results.sql | 0 .../bronze__streamline_transactions.sql | 0 ...treamline__complete_get_blocks_history.sql | 36 +++++++ ...line__complete_get_collections_history.sql | 36 +++++++ ...mplete_get_transaction_results_history.sql | 36 +++++++ ...ine__complete_get_transactions_history.sql | 36 +++++++ ...mline__get_blocks_history_candidate_07.sql | 29 +++-- ...mline__get_blocks_history_candidate_08.sql | 29 +++-- ...mline__get_blocks_history_candidate_09.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_01.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_02.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_03.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_04.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_05.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_06.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_07.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_08.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_09.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_10.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_11.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_12.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_13.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_14.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_15.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_16.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_17.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_18.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_19.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_20.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_21.sql | 29 +++-- ...eamline__get_blocks_history_mainnet_22.sql | 29 +++-- ...__get_collections_history_candidate_07.sql | 64 ++++++----- ...__get_collections_history_candidate_08.sql | 64 ++++++----- ...__get_collections_history_candidate_09.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_01.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_02.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_03.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_04.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_05.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_06.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_07.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_08.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_09.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_10.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_11.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_12.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_13.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_14.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_15.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_16.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_17.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_18.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_19.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_20.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_21.sql | 64 ++++++----- ...ne__get_collections_history_mainnet_22.sql | 64 ++++++----- .../streamline__get_blocks_history.sql | 34 ------ .../streamline__get_collections_history.sql | 77 ------------- ...mline__get_transaction_results_history.sql | 44 -------- .../streamline__get_transactions_history.sql | 44 -------- ...ansaction_results_history_candidate_07.sql | 65 ++++++----- ...ansaction_results_history_candidate_08.sql | 65 ++++++----- ...ansaction_results_history_candidate_09.sql | 65 ++++++----- ...transaction_results_history_mainnet_01.sql | 65 ++++++----- ...transaction_results_history_mainnet_02.sql | 65 ++++++----- ...transaction_results_history_mainnet_03.sql | 65 ++++++----- ...transaction_results_history_mainnet_04.sql | 65 ++++++----- ...transaction_results_history_mainnet_05.sql | 65 ++++++----- ...transaction_results_history_mainnet_06.sql | 65 ++++++----- ...transaction_results_history_mainnet_07.sql | 65 ++++++----- ...transaction_results_history_mainnet_08.sql | 65 ++++++----- ...transaction_results_history_mainnet_09.sql | 65 ++++++----- ...transaction_results_history_mainnet_10.sql | 65 ++++++----- ...transaction_results_history_mainnet_11.sql | 65 ++++++----- ...transaction_results_history_mainnet_12.sql | 65 ++++++----- ...transaction_results_history_mainnet_13.sql | 65 ++++++----- ...transaction_results_history_mainnet_14.sql | 65 ++++++----- ...transaction_results_history_mainnet_15.sql | 65 ++++++----- ...transaction_results_history_mainnet_16.sql | 65 ++++++----- ...transaction_results_history_mainnet_17.sql | 65 ++++++----- ...transaction_results_history_mainnet_18.sql | 65 ++++++----- ...transaction_results_history_mainnet_19.sql | 65 ++++++----- ...transaction_results_history_mainnet_20.sql | 65 ++++++----- ...transaction_results_history_mainnet_21.sql | 65 ++++++----- ...transaction_results_history_mainnet_22.sql | 65 ++++++----- ..._get_transactions_history_candidate_07.sql | 62 ++++++----- ..._get_transactions_history_candidate_08.sql | 62 ++++++----- ..._get_transactions_history_candidate_09.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_01.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_02.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_03.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_04.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_05.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_06.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_07.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_08.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_09.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_10.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_11.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_12.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_13.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_14.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_15.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_16.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_17.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_18.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_19.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_20.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_21.sql | 62 ++++++----- ...e__get_transactions_history_mainnet_22.sql | 62 ++++++----- models/sources.yml | 101 ++++++++++++++++++ ...ne__get_blocks_history_NETWORK_VERSION.sql | 29 +++-- ...et_collections_history_NETWORK_VERSION.sql | 64 ++++++----- ...action_results_history_NETWORK_VERSION.sql | 65 ++++++----- ...t_transactions_history_NETWORK_VERSION.sql | 62 ++++++----- .../list_backfill_external_tables.py | 26 +++++ python/create_views/table_names.md | 15 +++ 134 files changed, 3937 insertions(+), 2452 deletions(-) create mode 100644 .github/workflows/dbt_run_history.yml create mode 100644 models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql create mode 100644 models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql create mode 100644 models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql create mode 100644 models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_blocks.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_collections.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_fr_blocks.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_fr_collections.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_fr_transaction_results.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_fr_transactions.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_transaction_results.sql (100%) rename models/silver/streamline/bronze/core/{ => realtime}/bronze__streamline_transactions.sql (100%) create mode 100644 models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql create mode 100644 models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql create mode 100644 models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql create mode 100644 models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql delete mode 100644 models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql delete mode 100644 models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql delete mode 100644 models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql delete mode 100644 models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql create mode 100644 python/create_views/list_backfill_external_tables.py create mode 100644 python/create_views/table_names.md diff --git a/.github/workflows/dbt_run_history.yml b/.github/workflows/dbt_run_history.yml new file mode 100644 index 0000000..1b1f216 --- /dev/null +++ b/.github/workflows/dbt_run_history.yml @@ -0,0 +1,34 @@ +name: dbt_run_history +run-name: dbt_run_history + +on: + workflow_dispatch: + schedule: + # Runs every hour (see https://crontab.guru) + - cron: '0 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: > + dbt run -s 2+models/silver/streamline/core/history/ --vars '{"STREAMLINE_INVOKE_STREAMS": True}' + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit diff --git a/.github/workflows/dbt_run_streamline_blocks.yml b/.github/workflows/dbt_run_streamline_blocks.yml index 3badb33..c7923b3 100644 --- a/.github/workflows/dbt_run_streamline_blocks.yml +++ b/.github/workflows/dbt_run_streamline_blocks.yml @@ -44,7 +44,3 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_blocks_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' - - # - name: Run Backfill Models - # run: | - # dbt run -s models/silver/streamline/core/history/blocks --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/.github/workflows/dbt_run_streamline_collections.yml b/.github/workflows/dbt_run_streamline_collections.yml index f88dba0..895b50b 100644 --- a/.github/workflows/dbt_run_streamline_collections.yml +++ b/.github/workflows/dbt_run_streamline_collections.yml @@ -44,7 +44,3 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_collections_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' - - # - name: Run Backfill Models - # run: | - # dbt run -s models/silver/streamline/core/history/collections --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/.github/workflows/dbt_run_streamline_transaction_results.yml b/.github/workflows/dbt_run_streamline_transaction_results.yml index 9f5124a..3674b59 100644 --- a/.github/workflows/dbt_run_streamline_transaction_results.yml +++ b/.github/workflows/dbt_run_streamline_transaction_results.yml @@ -44,7 +44,3 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_transaction_results_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True, "producer_batch_size": 60000, "worker_batch_size": 2000}' - - # - name: Run Backfill Models - # run: | - # dbt run -s models/silver/streamline/core/history/transaction_results --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/.github/workflows/dbt_run_streamline_transactions.yml b/.github/workflows/dbt_run_streamline_transactions.yml index e12a372..cb0f1de 100644 --- a/.github/workflows/dbt_run_streamline_transactions.yml +++ b/.github/workflows/dbt_run_streamline_transactions.yml @@ -44,7 +44,3 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_transactions_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' - - # - name: Run Backfill Models - # run: | - # dbt run -s models/silver/streamline/core/history/transactions --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql index 1099b26..c8cca92 100644 --- a/macros/streamline/models.sql +++ b/macros/streamline/models.sql @@ -85,3 +85,68 @@ {% endmacro %} + {% macro streamline_multiple_external_table_query( + table_names, + partition_function, + partition_name, + unique_key + )%} + WITH + {% for table_name in table_names %} + meta_{{ table_name }} AS ( + SELECT + last_modified AS _inserted_timestamp, + file_name, + {{ partition_function }} AS {{ partition_name }} + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", table_name ) }}') + ) A + ), + {{ table_name }} AS ( + SELECT + block_number, + {{ unique_key }}, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s.{{ partition_name }}, + s.value AS VALUE + FROM + {{ source( + "bronze_streamline", + table_name + ) }} + s + JOIN meta_{{ table_name }} + b + ON b.file_name = metadata$filename + AND b.{{ partition_name }} = s.{{ partition_name }} + WHERE + b.{{ partition_name }} = s.{{ partition_name }} + ), + {% endfor %} + + FINAL AS ({% for table_name in table_names %} + SELECT + * + FROM + {{ table_name }} + + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} + ) + SELECT + * + FROM + FINAL + + {% endmacro %} diff --git a/models/silver/streamline/README.md b/models/silver/streamline/README.md index 269f29c..07c9e95 100644 --- a/models/silver/streamline/README.md +++ b/models/silver/streamline/README.md @@ -24,7 +24,7 @@ dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTER ```zsh # dev bronze__streamline_blocks.sql -dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' -m 1+models/silver/streamline/bronze/core/bronze__streamline_blocks.sql --profile flow --target dev --profiles-dir ~/.dbt +dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' -m 1+models/silver/streamline/bronze/core/realtime/bronze__streamline_blocks.sql --profile flow --target dev --profiles-dir ~/.dbt ``` ```zsh diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql new file mode 100644 index 0000000..48e08d5 --- /dev/null +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql @@ -0,0 +1,16 @@ +{{ config ( + materialized = 'view' +) }} + +{% set +table_names = [ + 'BLOCKS_CANDIDATE_07', 'BLOCKS_CANDIDATE_08', 'BLOCKS_CANDIDATE_09', 'BLOCKS_MAINNET_01', 'BLOCKS_MAINNET_02', 'BLOCKS_MAINNET_03', 'BLOCKS_MAINNET_04', 'BLOCKS_MAINNET_05', 'BLOCKS_MAINNET_06', 'BLOCKS_MAINNET_07', 'BLOCKS_MAINNET_08', 'BLOCKS_MAINNET_09', 'BLOCKS_MAINNET_10', 'BLOCKS_MAINNET_11', 'BLOCKS_MAINNET_12', 'BLOCKS_MAINNET_13', 'BLOCKS_MAINNET_14', 'BLOCKS_MAINNET_15', 'BLOCKS_MAINNET_16', 'BLOCKS_MAINNET_17', 'BLOCKS_MAINNET_18', 'BLOCKS_MAINNET_19', 'BLOCKS_MAINNET_20', 'BLOCKS_MAINNET_21', 'BLOCKS_MAINNET_22' +] +%} + +{{ streamline_multiple_external_table_query( + table_names, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql new file mode 100644 index 0000000..1917c13 --- /dev/null +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql @@ -0,0 +1,17 @@ +{{ config ( + materialized = 'view' +) }} + +{% set + table_names = + [ + 'COLLECTIONS_CANDIDATE_07', 'COLLECTIONS_CANDIDATE_08', 'COLLECTIONS_CANDIDATE_09', 'COLLECTIONS_MAINNET_01', 'COLLECTIONS_MAINNET_02', 'COLLECTIONS_MAINNET_03', 'COLLECTIONS_MAINNET_04', 'COLLECTIONS_MAINNET_05', 'COLLECTIONS_MAINNET_06', 'COLLECTIONS_MAINNET_07', 'COLLECTIONS_MAINNET_08', 'COLLECTIONS_MAINNET_09', 'COLLECTIONS_MAINNET_10', 'COLLECTIONS_MAINNET_11', 'COLLECTIONS_MAINNET_12', 'COLLECTIONS_MAINNET_13', 'COLLECTIONS_MAINNET_14', 'COLLECTIONS_MAINNET_15', 'COLLECTIONS_MAINNET_16', 'COLLECTIONS_MAINNET_17', 'COLLECTIONS_MAINNET_18', 'COLLECTIONS_MAINNET_19', 'COLLECTIONS_MAINNET_20', 'COLLECTIONS_MAINNET_21', 'COLLECTIONS_MAINNET_22' + ] +%} + +{{ streamline_multiple_external_table_query( + table_names, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql new file mode 100644 index 0000000..46620ab --- /dev/null +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql @@ -0,0 +1,17 @@ +{{ config ( + materialized = 'view' +) }} + +{% set + table_names = + [ + 'TRANSACTION_RESULTS_CANDIDATE_07', 'TRANSACTION_RESULTS_CANDIDATE_08', 'TRANSACTION_RESULTS_CANDIDATE_09', 'TRANSACTION_RESULTS_MAINNET_01', 'TRANSACTION_RESULTS_MAINNET_02', 'TRANSACTION_RESULTS_MAINNET_03', 'TRANSACTION_RESULTS_MAINNET_04', 'TRANSACTION_RESULTS_MAINNET_05', 'TRANSACTION_RESULTS_MAINNET_06', 'TRANSACTION_RESULTS_MAINNET_07', 'TRANSACTION_RESULTS_MAINNET_08', 'TRANSACTION_RESULTS_MAINNET_09', 'TRANSACTION_RESULTS_MAINNET_10', 'TRANSACTION_RESULTS_MAINNET_11', 'TRANSACTION_RESULTS_MAINNET_12', 'TRANSACTION_RESULTS_MAINNET_13', 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_20', 'TRANSACTION_RESULTS_MAINNET_21', 'TRANSACTION_RESULTS_MAINNET_22' + ] +%} + +{{ streamline_multiple_external_table_query( + table_names, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql new file mode 100644 index 0000000..ff0a376 --- /dev/null +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql @@ -0,0 +1,17 @@ +{{ config ( + materialized = 'view' +) }} + +{% set + table_names = + [ + 'TRANSACTIONS_CANDIDATE_07', 'TRANSACTIONS_CANDIDATE_08', 'TRANSACTIONS_CANDIDATE_09', 'TRANSACTIONS_MAINNET_01', 'TRANSACTIONS_MAINNET_02', 'TRANSACTIONS_MAINNET_03', 'TRANSACTIONS_MAINNET_04', 'TRANSACTIONS_MAINNET_05', 'TRANSACTIONS_MAINNET_06', 'TRANSACTIONS_MAINNET_07', 'TRANSACTIONS_MAINNET_08', 'TRANSACTIONS_MAINNET_09', 'TRANSACTIONS_MAINNET_10', 'TRANSACTIONS_MAINNET_11', 'TRANSACTIONS_MAINNET_12', 'TRANSACTIONS_MAINNET_13', 'TRANSACTIONS_MAINNET_14', 'TRANSACTIONS_MAINNET_15', 'TRANSACTIONS_MAINNET_16', 'TRANSACTIONS_MAINNET_17', 'TRANSACTIONS_MAINNET_18', 'TRANSACTIONS_MAINNET_19', 'TRANSACTIONS_MAINNET_20', 'TRANSACTIONS_MAINNET_21', 'TRANSACTIONS_MAINNET_22' + ] +%} + +{{ streamline_multiple_external_table_query( + table_names, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/silver/streamline/bronze/core/bronze__streamline_blocks.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_blocks.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_blocks.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_blocks.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_collections.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_collections.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_collections.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_collections.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_fr_blocks.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_blocks.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_fr_blocks.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_blocks.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_fr_collections.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_collections.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_fr_collections.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_collections.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_fr_transaction_results.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transaction_results.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_fr_transaction_results.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transaction_results.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_fr_transactions.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transactions.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_fr_transactions.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transactions.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_transaction_results.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_transaction_results.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_transaction_results.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_transaction_results.sql diff --git a/models/silver/streamline/bronze/core/bronze__streamline_transactions.sql b/models/silver/streamline/bronze/core/realtime/bronze__streamline_transactions.sql similarity index 100% rename from models/silver/streamline/bronze/core/bronze__streamline_transactions.sql rename to models/silver/streamline/bronze/core/realtime/bronze__streamline_transactions.sql diff --git a/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql new file mode 100644 index 0000000..d0c04e5 --- /dev/null +++ b/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql @@ -0,0 +1,36 @@ +-- depends_on: {{ ref('bronze__streamline_blocks_history') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["block_number"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_history'] +) }} + +SELECT + DATA, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{{ ref('bronze__streamline_blocks_history') }} + +WHERE + TRUE +{% if is_incremental() %} + AND _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql new file mode 100644 index 0000000..79ec85d --- /dev/null +++ b/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql @@ -0,0 +1,36 @@ +-- depends_on: {{ ref('bronze__streamline_collections_history') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_history'] +) }} + +SELECT + id, + data, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{{ ref('bronze__streamline_collections_history') }} +WHERE + TRUE +{% if is_incremental() %} +AND _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01'::timestamp_ntz + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql new file mode 100644 index 0000000..dc42240 --- /dev/null +++ b/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql @@ -0,0 +1,36 @@ +-- depends_on: {{ ref('bronze__streamline_transaction_results_history') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_history'] +) }} + +SELECT + id, + DATA, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{{ ref('bronze__streamline_transaction_results_history') }} +WHERE + TRUE +{% if is_incremental() %} + AND _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql new file mode 100644 index 0000000..b1b2e09 --- /dev/null +++ b/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql @@ -0,0 +1,36 @@ +-- depends_on: {{ ref('bronze__streamline_transactions_history') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_history'] +) }} + +SELECT + id, + data, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{{ ref('bronze__streamline_transactions_history') }} +WHERE + TRUE +{% if is_incremental() %} + AND _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01'::timestamp_ntz + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql index a734eed..6f39120 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'blocks_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 4132133 + AND 4972986 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 4132133 + AND 4972986 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 4132133 - AND 4972986 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql index a5506ca..a2a9a7a 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'blocks_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 4972987 + AND 6483245 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 4972987 + AND 6483245 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 4972987 - AND 6483245 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql index f4352e7..0ed1338 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'blocks_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 6483246 + AND 7601062 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 6483246 + AND 7601062 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 6483246 - AND 7601062 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql index 6348465..e19ef16 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'blocks_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 7601063 + AND 8742958 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 7601063 + AND 8742958 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 7601063 - AND 8742958 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql index 12bb36f..63d3dab 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'blocks_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 8742959 + AND 9737132 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 8742959 + AND 9737132 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 8742959 - AND 9737132 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql index 6ad8639..e7095d5 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'blocks_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 9737133 + AND 9992019 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 9737133 + AND 9992019 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 9737133 - AND 9992019 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql index 48efe19..279c086 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'blocks_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 9992020 + AND 12020336 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 9992020 + AND 12020336 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 9992020 - AND 12020336 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql index 0e9fb48..fdc62ec 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'blocks_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 12020337 + AND 12609236 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 12020337 + AND 12609236 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 12020337 - AND 12609236 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql index d8082a3..a4da3c5 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'blocks_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 12609237 + AND 13404173 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 12609237 + AND 13404173 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 12609237 - AND 13404173 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql index 475d0d7..dac477e 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'blocks_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 13404174 + AND 13950741 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 13404174 + AND 13950741 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 13404174 - AND 13950741 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql index 91da0f9..9beca4c 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'blocks_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 13950742 + AND 14892103 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 13950742 + AND 14892103 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 13950742 - AND 14892103 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql index eff828d..9bcd643 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'blocks_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 14892104 + AND 15791890 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 14892104 + AND 15791890 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 14892104 - AND 15791890 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql index 9019d48..4070b56 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'blocks_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 15791891 + AND 16755601 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 15791891 + AND 16755601 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 15791891 - AND 16755601 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql index abdf614..42bafa9 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'blocks_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 16755602 + AND 17544522 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 16755602 + AND 17544522 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 16755602 - AND 17544522 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql index c2cf23b..30f87e7 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'blocks_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 17544523 + AND 18587477 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 17544523 + AND 18587477 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 17544523 - AND 18587477 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql index 9be0489..5bc710b 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'blocks_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 18587478 + AND 19050752 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 18587478 + AND 19050752 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 18587478 - AND 19050752 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql index a476671..7f8acff 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'blocks_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 19050753 + AND 21291691 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 19050753 + AND 21291691 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 19050753 - AND 21291691 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql index a8fc830..2d870f4 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'blocks_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 21291692 + AND 23830812 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 21291692 + AND 23830812 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 21291692 - AND 23830812 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql index 65c97bd..662a9d1 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'blocks_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 23830813 + AND 27341469 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 23830813 + AND 27341469 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 23830813 - AND 27341469 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql index d3c59fb..186b046 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'blocks_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 27341470 + AND 31735954 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 27341470 + AND 31735954 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 27341470 - AND 31735954 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql index 6e31aae..60f4976 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'blocks_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 31735955 + AND 35858810 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 31735955 + AND 35858810 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 31735955 - AND 35858810 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql index 62556ce..f57a4e3 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'blocks_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 35858811 + AND 40171633 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 35858811 + AND 40171633 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 35858811 - AND 40171633 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql index 3b15369..3ff7266 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'blocks_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 40171634 + AND 44950206 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 40171634 + AND 44950206 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 40171634 - AND 44950206 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql index e6b4c93..ac06a21 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'blocks_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 44950207 + AND 47169686 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 44950207 + AND 47169686 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 44950207 - AND 47169686 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql index d458e13..8f61335 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'blocks_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 47169687 + AND 55114466 EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN 47169687 + AND 55114466 ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN 47169687 - AND 55114466 ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql index 589bf28..44b008a 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'transactions_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'collections_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 4132133 + AND 4972986 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 4132133 - AND 4972986 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql index 733e277..de8b7e1 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'transactions_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'collections_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 4972987 + AND 6483245 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 4972987 - AND 6483245 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql index de98325..3cb6b84 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'transactions_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'collections_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 6483246 + AND 7601062 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 6483246 - AND 7601062 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql index 1f52544..a49df83 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'transactions_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'collections_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 7601063 + AND 8742958 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 7601063 - AND 8742958 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql index 9968d02..3d2d38e 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'transactions_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'collections_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 8742959 + AND 9737132 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 8742959 - AND 9737132 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql index b4eafdb..3a87e7d 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'transactions_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'collections_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 9737133 + AND 9992019 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 9737133 - AND 9992019 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql index 418385a..924a2e2 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'transactions_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'collections_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 9992020 + AND 12020336 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 9992020 - AND 12020336 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql index 820a85a..253cf10 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'transactions_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'collections_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 12020337 + AND 12609236 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 12020337 - AND 12609236 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql index 6959bec..9d79e42 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'transactions_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'collections_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 12609237 + AND 13404173 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 12609237 - AND 13404173 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql index 06e6007..6526d95 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'transactions_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'collections_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 13404174 + AND 13950741 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 13404174 - AND 13950741 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql index 6afe007..ae54d6d 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'transactions_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'collections_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 13950742 + AND 14892103 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 13950742 - AND 14892103 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql index 61b60c0..9d8aea9 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'transactions_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'collections_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 14892104 + AND 15791890 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 14892104 - AND 15791890 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql index 2f0bf5b..d7c70ae 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'transactions_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'collections_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 15791891 + AND 16755601 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 15791891 - AND 16755601 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql index 64df07b..b938fbd 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'transactions_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'collections_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 16755602 + AND 17544522 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 16755602 - AND 17544522 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql index 37d78ed..07167e3 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'transactions_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'collections_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 17544523 + AND 18587477 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 17544523 - AND 18587477 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql index d080f78..3e20d7c 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'transactions_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'collections_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 18587478 + AND 19050752 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 18587478 - AND 19050752 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql index 4cdcf66..3254f2b 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transactions_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'collections_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 19050753 + AND 21291691 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 19050753 - AND 21291691 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql index 4859138..9415778 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transactions_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'collections_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 21291692 + AND 23830812 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 21291692 - AND 23830812 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql index 301df79..90f9905 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transactions_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'collections_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 23830813 + AND 27341469 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 23830813 - AND 27341469 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql index a676df8..2a9c708 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transactions_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'collections_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 27341470 + AND 31735954 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 27341470 - AND 31735954 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql index 2a122df..aa1d1dc 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transactions_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'collections_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 31735955 + AND 35858810 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 31735955 - AND 35858810 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql index 1d22c5c..b64a002 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transactions_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'collections_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 35858811 + AND 40171633 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 35858811 - AND 40171633 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql index 2880e8a..8aae6a1 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'transactions_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'collections_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 40171634 + AND 44950206 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 40171634 - AND 44950206 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql index ce14a9b..2b10bf6 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'transactions_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'collections_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 44950207 + AND 47169686 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 44950207 - AND 47169686 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql index 282fc1a..ca3e458 100644 --- a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transactions_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'collections_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN 47169687 + AND 55114466 ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 47169687 - AND 55114466 + collections_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql deleted file mode 100644 index 126d19b..0000000 --- a/models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql +++ /dev/null @@ -1,34 +0,0 @@ -{{ config( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url', '{{ var('node_url', Null) }}','external_table', 'blocks', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH blocks AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_blocks") }} -) -SELECT - OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) - ) AS request -FROM - blocks -WHERE - block_height BETWEEN {{ var('start_block', Null) }} AND {{ var('end_block', Null) }} -ORDER BY - block_height ASC -LIMIT - 1000000 diff --git a/models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql deleted file mode 100644 index 492a8e0..0000000 --- a/models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql +++ /dev/null @@ -1,77 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','{{ var('node_url', Null) }}','external_table', 'collections', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH blocks AS ( - - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_collections") }} -), -collections AS ( - SELECT - block_number AS block_height, - DATA - FROM - {{ ref('streamline__complete_get_blocks') }} - JOIN blocks - ON blocks.block_height = block_number -), --- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks table -block_collections AS ( - SELECT - cb.block_number AS block_height, - collection_guarantee.value :collection_id AS collection_id - FROM - {{ ref("streamline__complete_get_blocks") }} - cb, - LATERAL FLATTEN( - input => cb.data :collection_guarantees - ) AS collection_guarantee -), --- CTE to identify collections that haven't been ingested yet -collections_to_ingest AS ( - SELECT - bc.block_height, - bc.collection_id - FROM - block_collections bc - LEFT JOIN {{ ref("streamline__complete_get_collections") }} C - ON bc.block_height = C.block_number - AND bc.collection_id = C.id - WHERE - C.id IS NULL -) -- Generate the requests based on the missing collections -SELECT - OBJECT_CONSTRUCT( - 'grpc', - 'proto3', - 'method', - 'get_collection_by_i_d', - 'block_height', - block_height :: INTEGER, - 'method_params', - OBJECT_CONSTRUCT( - 'id', - collection_id - ) - ) AS request -FROM - collections_to_ingest -WHERE - block_height BETWEEN {{ var('start_block', Null) }} - AND {{ var('end_block', Null) }} -ORDER BY - block_height ASC -LIMIT - 1000000 diff --git a/models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql deleted file mode 100644 index 5589ba0..0000000 --- a/models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql +++ /dev/null @@ -1,44 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}', 'node_url', '{{ var('node_url', Null) }}', 'external_table', 'transaction_results', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','30000')}}, 'worker_batch_size', {{var('worker_batch_size','3000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH blocks AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( - SELECT - block_number as block_height, - data - FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) -SELECT - OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) - ) AS request -FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN {{ var('start_block', Null) }} AND {{ var('end_block', Null) }} -ORDER BY - block_height ASC -LIMIT - 1000000 diff --git a/models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql deleted file mode 100644 index 6b89267..0000000 --- a/models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql +++ /dev/null @@ -1,44 +0,0 @@ -{{ config ( - materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}', 'node_url', '{{ var('node_url', Null) }}', 'external_table', 'transactions', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','30000')}}, 'worker_batch_size', {{var('worker_batch_size','3000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" - ) -) }} - -WITH blocks AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} -), -tx AS ( - SELECT - block_number as block_height, - data - FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) -SELECT - OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) - ) AS request -FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN {{ var('start_block', Null) }} AND {{ var('end_block', Null) }} -ORDER BY - block_height ASC -LIMIT - 1000000 diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql index 849fc71..d54cde6 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 4132133 + AND 4972986 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 4132133 - AND 4972986 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql index e550d0d..58f2bfd 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 4972987 + AND 6483245 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 4972987 - AND 6483245 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql index f0a7771..4d9700b 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 6483246 + AND 7601062 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 6483246 - AND 7601062 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql index e7bc03b..17d8c1c 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 7601063 + AND 8742958 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 7601063 - AND 8742958 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql index c5b2c99..ac7635a 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 8742959 + AND 9737132 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 8742959 - AND 9737132 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql index ab8788d..26c7ca4 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 9737133 + AND 9992019 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 9737133 - AND 9992019 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql index b626965..acce952 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 9992020 + AND 12020336 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 9992020 - AND 12020336 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql index c86ff89..23e9b8f 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 12020337 + AND 12609236 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 12020337 - AND 12609236 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql index 6843805..2cb89d1 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 12609237 + AND 13404173 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 12609237 - AND 13404173 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql index 160f2d6..7bcdaf5 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 13404174 + AND 13950741 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 13404174 - AND 13950741 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql index 082a7fe..422b936 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 13950742 + AND 14892103 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 13950742 - AND 14892103 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql index 7294ca2..cc18801 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 14892104 + AND 15791890 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 14892104 - AND 15791890 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql index 164faff..f849767 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 15791891 + AND 16755601 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 15791891 - AND 16755601 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql index bb3e4a4..0a72d3e 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 16755602 + AND 17544522 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 16755602 - AND 17544522 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql index 418421b..2adb2b1 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 17544523 + AND 18587477 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 17544523 - AND 18587477 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql index 1375cdd..b08c203 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 18587478 + AND 19050752 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 18587478 - AND 19050752 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql index 6d2740f..3e3861a 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 19050753 + AND 21291691 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 19050753 - AND 21291691 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql index 5d1247a..52120e3 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 21291692 + AND 23830812 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 21291692 - AND 23830812 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql index d033d8b..0f43df6 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 23830813 + AND 27341469 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 23830813 - AND 27341469 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql index 2a6c165..718c4d9 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 27341470 + AND 31735954 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 27341470 - AND 31735954 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql index ea0536b..14032bc 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 31735955 + AND 35858810 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 31735955 - AND 35858810 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql index 0fe03d2..9de1992 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 35858811 + AND 40171633 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 35858811 - AND 40171633 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql index ce5a336..e3654d3 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 40171634 + AND 44950206 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 40171634 - AND 44950206 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql index 2e2353d..33570d1 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 44950207 + AND 47169686 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 44950207 - AND 47169686 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql index 80bb4b1..b3bf9ab 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 47169687 + AND 55114466 +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 47169687 - AND 55114466 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql index fc030b3..0730e03 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 4132133 + AND 4972986 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 4132133 - AND 4972986 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql index f951264..fb03886 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 4972987 + AND 6483245 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 4972987 - AND 6483245 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql index d5f33a1..b48c214 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 6483246 + AND 7601062 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 6483246 - AND 7601062 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql index 46224ac..f6c8d24 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 7601063 + AND 8742958 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 7601063 - AND 8742958 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql index 7feab2a..197031b 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 8742959 + AND 9737132 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 8742959 - AND 9737132 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql index 6cc2466..b72492a 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 9737133 + AND 9992019 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 9737133 - AND 9992019 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql index a38c81c..99eb275 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 9992020 + AND 12020336 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 9992020 - AND 12020336 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql index 2ddec96..bccbbe9 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 12020337 + AND 12609236 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 12020337 - AND 12609236 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql index 343a496..d670ba9 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 12609237 + AND 13404173 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 12609237 - AND 13404173 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql index 6305494..5813e75 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 13404174 + AND 13950741 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 13404174 - AND 13950741 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql index 503ca5d..5d05c31 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 13950742 + AND 14892103 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 13950742 - AND 14892103 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql index a554027..d51bf43 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 14892104 + AND 15791890 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 14892104 - AND 15791890 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql index cedc9db..fb542cd 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 15791891 + AND 16755601 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 15791891 - AND 16755601 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql index 5ce0028..e059fd0 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 16755602 + AND 17544522 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 16755602 - AND 17544522 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql index f22d389..4a28772 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 17544523 + AND 18587477 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 17544523 - AND 18587477 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql index 9790973..20dcc55 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 18587478 + AND 19050752 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 18587478 - AND 19050752 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql index 048a448..6ba563f 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 19050753 + AND 21291691 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 19050753 - AND 21291691 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql index b77ce2a..f126c5c 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 21291692 + AND 23830812 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 21291692 - AND 23830812 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql index f42f7eb..ed17589 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 23830813 + AND 27341469 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 23830813 - AND 27341469 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql index 1d1e560..8e740f2 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 27341470 + AND 31735954 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 27341470 - AND 31735954 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql index 108c915..e368e63 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 31735955 + AND 35858810 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 31735955 - AND 35858810 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql index 228e9a8..ca42f2d 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 35858811 + AND 40171633 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 35858811 - AND 40171633 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql index 5bcd35a..96d4776 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 40171634 + AND 44950206 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 40171634 - AND 44950206 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql index dd4f463..488fa5d 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 44950207 + AND 47169686 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 44950207 - AND 47169686 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql index 2a7e343..0875ef6 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 47169687 + AND 55114466 ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN 47169687 - AND 55114466 + transactions_to_ingest ORDER BY block_height ASC diff --git a/models/sources.yml b/models/sources.yml index 6e3a3d9..f15dd89 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -24,6 +24,107 @@ sources: - name: collections - name: transactions - name: transaction_results + - name: BLOCKS_CANDIDATE_07 + - name: BLOCKS_CANDIDATE_08 + - name: BLOCKS_CANDIDATE_09 + - name: BLOCKS_MAINNET_01 + - name: BLOCKS_MAINNET_02 + - name: BLOCKS_MAINNET_03 + - name: BLOCKS_MAINNET_04 + - name: BLOCKS_MAINNET_05 + - name: BLOCKS_MAINNET_06 + - name: BLOCKS_MAINNET_07 + - name: BLOCKS_MAINNET_08 + - name: BLOCKS_MAINNET_09 + - name: BLOCKS_MAINNET_10 + - name: BLOCKS_MAINNET_11 + - name: BLOCKS_MAINNET_12 + - name: BLOCKS_MAINNET_13 + - name: BLOCKS_MAINNET_14 + - name: BLOCKS_MAINNET_15 + - name: BLOCKS_MAINNET_16 + - name: BLOCKS_MAINNET_17 + - name: BLOCKS_MAINNET_18 + - name: BLOCKS_MAINNET_19 + - name: BLOCKS_MAINNET_20 + - name: BLOCKS_MAINNET_21 + - name: BLOCKS_MAINNET_22 + - name: COLLECTIONS_CANDIDATE_07 + - name: COLLECTIONS_CANDIDATE_08 + - name: COLLECTIONS_CANDIDATE_09 + - name: COLLECTIONS_MAINNET_01 + - name: COLLECTIONS_MAINNET_02 + - name: COLLECTIONS_MAINNET_03 + - name: COLLECTIONS_MAINNET_04 + - name: COLLECTIONS_MAINNET_05 + - name: COLLECTIONS_MAINNET_06 + - name: COLLECTIONS_MAINNET_07 + - name: COLLECTIONS_MAINNET_08 + - name: COLLECTIONS_MAINNET_09 + - name: COLLECTIONS_MAINNET_10 + - name: COLLECTIONS_MAINNET_11 + - name: COLLECTIONS_MAINNET_12 + - name: COLLECTIONS_MAINNET_13 + - name: COLLECTIONS_MAINNET_14 + - name: COLLECTIONS_MAINNET_15 + - name: COLLECTIONS_MAINNET_16 + - name: COLLECTIONS_MAINNET_17 + - name: COLLECTIONS_MAINNET_18 + - name: COLLECTIONS_MAINNET_19 + - name: COLLECTIONS_MAINNET_20 + - name: COLLECTIONS_MAINNET_21 + - name: COLLECTIONS_MAINNET_22 + - name: TRANSACTIONS_CANDIDATE_07 + - name: TRANSACTIONS_CANDIDATE_08 + - name: TRANSACTIONS_CANDIDATE_09 + - name: TRANSACTIONS_MAINNET_01 + - name: TRANSACTIONS_MAINNET_02 + - name: TRANSACTIONS_MAINNET_03 + - name: TRANSACTIONS_MAINNET_04 + - name: TRANSACTIONS_MAINNET_05 + - name: TRANSACTIONS_MAINNET_06 + - name: TRANSACTIONS_MAINNET_07 + - name: TRANSACTIONS_MAINNET_08 + - name: TRANSACTIONS_MAINNET_09 + - name: TRANSACTIONS_MAINNET_10 + - name: TRANSACTIONS_MAINNET_11 + - name: TRANSACTIONS_MAINNET_12 + - name: TRANSACTIONS_MAINNET_13 + - name: TRANSACTIONS_MAINNET_14 + - name: TRANSACTIONS_MAINNET_15 + - name: TRANSACTIONS_MAINNET_16 + - name: TRANSACTIONS_MAINNET_17 + - name: TRANSACTIONS_MAINNET_18 + - name: TRANSACTIONS_MAINNET_19 + - name: TRANSACTIONS_MAINNET_20 + - name: TRANSACTIONS_MAINNET_21 + - name: TRANSACTIONS_MAINNET_22 + - name: TRANSACTION_RESULTS_CANDIDATE_07 + - name: TRANSACTION_RESULTS_CANDIDATE_08 + - name: TRANSACTION_RESULTS_CANDIDATE_09 + - name: TRANSACTION_RESULTS_MAINNET_01 + - name: TRANSACTION_RESULTS_MAINNET_02 + - name: TRANSACTION_RESULTS_MAINNET_03 + - name: TRANSACTION_RESULTS_MAINNET_04 + - name: TRANSACTION_RESULTS_MAINNET_05 + - name: TRANSACTION_RESULTS_MAINNET_06 + - name: TRANSACTION_RESULTS_MAINNET_07 + - name: TRANSACTION_RESULTS_MAINNET_08 + - name: TRANSACTION_RESULTS_MAINNET_09 + - name: TRANSACTION_RESULTS_MAINNET_10 + - name: TRANSACTION_RESULTS_MAINNET_11 + - name: TRANSACTION_RESULTS_MAINNET_12 + - name: TRANSACTION_RESULTS_MAINNET_13 + - name: TRANSACTION_RESULTS_MAINNET_14 + - name: TRANSACTION_RESULTS_MAINNET_15 + - name: TRANSACTION_RESULTS_MAINNET_16 + - name: TRANSACTION_RESULTS_MAINNET_17 + - name: TRANSACTION_RESULTS_MAINNET_18 + - name: TRANSACTION_RESULTS_MAINNET_19 + - name: TRANSACTION_RESULTS_MAINNET_20 + - name: TRANSACTION_RESULTS_MAINNET_21 + - name: TRANSACTION_RESULTS_MAINNET_22 + - name: crosschain_silver diff --git a/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql index c99ce50..1511adf 100644 --- a/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql +++ b/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'blocks_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_history'] ) }} WITH blocks AS ( @@ -12,23 +13,33 @@ WITH blocks AS ( block_height FROM {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN root_height + AND end_height EXCEPT SELECT block_number AS block_height FROM - {{ ref("streamline__complete_get_blocks") }} + {{ ref("streamline__complete_get_blocks_history") }} + WHERE + block_height BETWEEN root_height + AND end_height ) SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_block_by_height', - 'block_height', block_height, - 'method_params', OBJECT_CONSTRUCT('height', block_height) + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) ) AS request FROM blocks -WHERE - block_height BETWEEN root_height - AND end_height ORDER BY block_height ASC diff --git a/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql index d006eb9..aef45a6 100644 --- a/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql +++ b/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql @@ -1,45 +1,57 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'transactions_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'collections_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} -WITH blocks AS ( +WITH +-- CTE to get all block_heights and their associated collection_ids from the complete_get_blocks_history table +block_collections AS ( SELECT - block_height + cb.block_number AS block_height, + collection_guarantee.value :collection_id :: STRING AS collection_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref("streamline__complete_get_blocks_history") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height BETWEEN root_height + AND end_height ), -tx AS ( +-- CTE to identify collections that haven't been ingested yet +collections_to_ingest AS ( SELECT - block_number AS block_height, - DATA + bc.block_height, + bc.collection_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_collections_history") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL +) +-- Generate the requests based on the missing collections SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN root_height - AND end_height + collections_to_ingest ORDER BY block_height ASC diff --git a/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql index 54e9599..a6852f0 100644 --- a/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql +++ b/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql @@ -6,40 +6,51 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( - SELECT - block_height - FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number AS block_height - FROM - {{ ref("streamline__complete_get_transaction_results") }} -), -tx AS ( SELECT block_number AS block_height, - DATA + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks - ON blocks.block_height = block_number -) + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN root_height + AND end_height +), +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transaction_results_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) -- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction_result', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id -WHERE - block_height BETWEEN root_height - AND end_height + transactions_to_ingest ORDER BY block_height ASC diff --git a/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql index 2b49509..c00a0b6 100644 --- a/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql +++ b/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql @@ -6,38 +6,52 @@ ) ) }} -WITH blocks AS ( +WITH collection_transactions AS ( + SELECT - block_height + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id FROM - {{ ref("streamline__blocks") }} - EXCEPT - SELECT - block_number as block_height - FROM - {{ ref("streamline__complete_get_transactions") }} + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN root_height + AND end_height ), -tx AS ( +-- CTE to identify transactions that haven't been ingested yet +transactions_to_ingest AS ( SELECT - block_number as block_height, - data + ct.block_height, + ct.transaction_id FROM - {{ ref('streamline__complete_get_collections') }} - JOIN blocks ON blocks.block_height = block_number -) + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +) +-- Generate the requests based on the missing transactions SELECT OBJECT_CONSTRUCT( - 'grpc', 'proto3', - 'method', 'get_transaction', - 'block_height', block_height::INTEGER, - 'transaction_id', transaction_id.value::string, - 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id + ) ) AS request FROM - tx, - LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id -WHERE - block_height BETWEEN root_height - AND end_height + transactions_to_ingest ORDER BY block_height ASC diff --git a/python/create_views/list_backfill_external_tables.py b/python/create_views/list_backfill_external_tables.py new file mode 100644 index 0000000..c395265 --- /dev/null +++ b/python/create_views/list_backfill_external_tables.py @@ -0,0 +1,26 @@ +import csv +import os + +csv_file_path = '../../data/seeds__network_version.csv' + +# Read the network data from the CSV +with open(csv_file_path, 'r') as f: + reader = csv.DictReader(f) + network_version = [row['network_version'].replace('-', '_').upper() for row in reader] + +methods = ["BLOCKS", "COLLECTIONS", "TRANSACTIONS", "TRANSACTION_RESULTS"] + +# list external table names +table_names = [] +for method in methods: + for network in network_version: + table_names.append(f"{method}_{network}") + +# Write output to yml file, copy to sources.yml +with open('network_versions.yml', 'w') as f: + for table_name in table_names: + f.write(f"- name: {table_name}\n") + +# output table names for copying to sql file +with open("table_names.md", "w") as f: + f.write(f"{table_names}\n") diff --git a/python/create_views/table_names.md b/python/create_views/table_names.md new file mode 100644 index 0000000..5949586 --- /dev/null +++ b/python/create_views/table_names.md @@ -0,0 +1,15 @@ +[ + 'BLOCKS_CANDIDATE_07', 'BLOCKS_CANDIDATE_08', 'BLOCKS_CANDIDATE_09', 'BLOCKS_MAINNET_01', 'BLOCKS_MAINNET_02', 'BLOCKS_MAINNET_03', 'BLOCKS_MAINNET_04', 'BLOCKS_MAINNET_05', 'BLOCKS_MAINNET_06', 'BLOCKS_MAINNET_07', 'BLOCKS_MAINNET_08', 'BLOCKS_MAINNET_09', 'BLOCKS_MAINNET_10', 'BLOCKS_MAINNET_11', 'BLOCKS_MAINNET_12', 'BLOCKS_MAINNET_13', 'BLOCKS_MAINNET_14', 'BLOCKS_MAINNET_15', 'BLOCKS_MAINNET_16', 'BLOCKS_MAINNET_17', 'BLOCKS_MAINNET_18', 'BLOCKS_MAINNET_19', 'BLOCKS_MAINNET_20', 'BLOCKS_MAINNET_21', 'BLOCKS_MAINNET_22' +] + +[ + 'COLLECTIONS_CANDIDATE_07', 'COLLECTIONS_CANDIDATE_08', 'COLLECTIONS_CANDIDATE_09', 'COLLECTIONS_MAINNET_01', 'COLLECTIONS_MAINNET_02', 'COLLECTIONS_MAINNET_03', 'COLLECTIONS_MAINNET_04', 'COLLECTIONS_MAINNET_05', 'COLLECTIONS_MAINNET_06', 'COLLECTIONS_MAINNET_07', 'COLLECTIONS_MAINNET_08', 'COLLECTIONS_MAINNET_09', 'COLLECTIONS_MAINNET_10', 'COLLECTIONS_MAINNET_11', 'COLLECTIONS_MAINNET_12', 'COLLECTIONS_MAINNET_13', 'COLLECTIONS_MAINNET_14', 'COLLECTIONS_MAINNET_15', 'COLLECTIONS_MAINNET_16', 'COLLECTIONS_MAINNET_17', 'COLLECTIONS_MAINNET_18', 'COLLECTIONS_MAINNET_19', 'COLLECTIONS_MAINNET_20', 'COLLECTIONS_MAINNET_21', 'COLLECTIONS_MAINNET_22' +] + +[ + 'TRANSACTIONS_CANDIDATE_07', 'TRANSACTIONS_CANDIDATE_08', 'TRANSACTIONS_CANDIDATE_09', 'TRANSACTIONS_MAINNET_01', 'TRANSACTIONS_MAINNET_02', 'TRANSACTIONS_MAINNET_03', 'TRANSACTIONS_MAINNET_04', 'TRANSACTIONS_MAINNET_05', 'TRANSACTIONS_MAINNET_06', 'TRANSACTIONS_MAINNET_07', 'TRANSACTIONS_MAINNET_08', 'TRANSACTIONS_MAINNET_09', 'TRANSACTIONS_MAINNET_10', 'TRANSACTIONS_MAINNET_11', 'TRANSACTIONS_MAINNET_12', 'TRANSACTIONS_MAINNET_13', 'TRANSACTIONS_MAINNET_14', 'TRANSACTIONS_MAINNET_15', 'TRANSACTIONS_MAINNET_16', 'TRANSACTIONS_MAINNET_17', 'TRANSACTIONS_MAINNET_18', 'TRANSACTIONS_MAINNET_19', 'TRANSACTIONS_MAINNET_20', 'TRANSACTIONS_MAINNET_21', 'TRANSACTIONS_MAINNET_22' +] + +[ + 'TRANSACTION_RESULTS_CANDIDATE_07', 'TRANSACTION_RESULTS_CANDIDATE_08', 'TRANSACTION_RESULTS_CANDIDATE_09', 'TRANSACTION_RESULTS_MAINNET_01', 'TRANSACTION_RESULTS_MAINNET_02', 'TRANSACTION_RESULTS_MAINNET_03', 'TRANSACTION_RESULTS_MAINNET_04', 'TRANSACTION_RESULTS_MAINNET_05', 'TRANSACTION_RESULTS_MAINNET_06', 'TRANSACTION_RESULTS_MAINNET_07', 'TRANSACTION_RESULTS_MAINNET_08', 'TRANSACTION_RESULTS_MAINNET_09', 'TRANSACTION_RESULTS_MAINNET_10', 'TRANSACTION_RESULTS_MAINNET_11', 'TRANSACTION_RESULTS_MAINNET_12', 'TRANSACTION_RESULTS_MAINNET_13', 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_20', 'TRANSACTION_RESULTS_MAINNET_21', 'TRANSACTION_RESULTS_MAINNET_22' +]