From 856ac1629ff871e3523c2e85658fb12394d6a2b3 Mon Sep 17 00:00:00 2001 From: Jack Forgash <58153492+forgxyz@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:12:13 -0600 Subject: [PATCH] AN-3935/backfill views (#208) * archive parameterized model * create views for each network ver * network version views * newline * switch tx results realtime back to us-east-1 * del blocks history mainnet23 * add backfill step to streamline GHA * newline * rm dup WHERE * upd name and del dup where from template --- .../workflows/dbt_run_streamline_blocks.yml | 8 +--- .../dbt_run_streamline_collections.yml | 4 ++ .../workflows/dbt_run_streamline_hourly.yml | 37 --------------- ...dbt_run_streamline_transaction_results.yml | 4 ++ .../dbt_run_streamline_transactions.yml | 4 ++ ...line__get_blocks_history_candidate_07.sql} | 9 ++-- ...mline__get_blocks_history_candidate_08.sql | 34 ++++++++++++++ ...mline__get_blocks_history_candidate_09.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_01.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_02.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_03.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_04.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_05.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_06.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_07.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_08.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_09.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_10.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_11.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_12.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_13.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_14.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_15.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_16.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_17.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_18.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_19.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_20.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_21.sql | 34 ++++++++++++++ ...eamline__get_blocks_history_mainnet_22.sql | 34 ++++++++++++++ ...__get_collections_history_candidate_07.sql | 45 +++++++++++++++++++ ...__get_collections_history_candidate_08.sql | 45 +++++++++++++++++++ ...__get_collections_history_candidate_09.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_01.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_02.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_03.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_04.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_05.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_06.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_07.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_08.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_09.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_10.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_11.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_12.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_13.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_14.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_15.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_16.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_17.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_18.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_19.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_20.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_21.sql | 45 +++++++++++++++++++ ...ne__get_collections_history_mainnet_22.sql | 45 +++++++++++++++++++ .../streamline__get_blocks_history.sql | 3 +- .../streamline__get_collections_history.sql | 3 +- ...mline__get_transaction_results_history.sql | 3 +- .../streamline__get_transactions_history.sql | 3 +- ...ansaction_results_history_candidate_07.sql | 45 +++++++++++++++++++ ...ansaction_results_history_candidate_08.sql | 45 +++++++++++++++++++ ...ansaction_results_history_candidate_09.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_01.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_02.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_03.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_04.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_05.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_06.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_07.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_08.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_09.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_10.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_11.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_12.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_13.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_14.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_15.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_16.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_17.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_18.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_19.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_20.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_21.sql | 45 +++++++++++++++++++ ...transaction_results_history_mainnet_22.sql | 45 +++++++++++++++++++ ..._get_transactions_history_candidate_07.sql | 43 ++++++++++++++++++ ..._get_transactions_history_candidate_08.sql | 43 ++++++++++++++++++ ..._get_transactions_history_candidate_09.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_01.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_02.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_03.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_04.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_05.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_06.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_07.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_08.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_09.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_10.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_11.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_12.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_13.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_14.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_15.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_16.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_17.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_18.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_19.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_20.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_21.sql | 43 ++++++++++++++++++ ...e__get_transactions_history_mainnet_22.sql | 43 ++++++++++++++++++ ...line__get_transaction_results_realtime.sql | 2 +- python/create_views/README.md | 3 ++ python/create_views/create_history_views.py | 38 ++++++++++++++++ ...ne__get_blocks_history_NETWORK_VERSION.sql | 34 ++++++++++++++ ...et_collections_history_NETWORK_VERSION.sql | 45 +++++++++++++++++++ ...action_results_history_NETWORK_VERSION.sql | 45 +++++++++++++++++++ ...t_transactions_history_NETWORK_VERSION.sql | 43 ++++++++++++++++++ 116 files changed, 4373 insertions(+), 56 deletions(-) delete mode 100644 .github/workflows/dbt_run_streamline_hourly.yml rename models/silver/streamline/core/history/blocks/{streamline__get_blocks_history_mainnet22.sql => streamline__get_blocks_history_candidate_07.sql} (62%) create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql create mode 100644 models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql create mode 100644 models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql rename models/silver/streamline/core/history/{ => parameterized}/streamline__get_blocks_history.sql (96%) rename models/silver/streamline/core/history/{ => parameterized}/streamline__get_collections_history.sql (98%) rename models/silver/streamline/core/history/{ => parameterized}/streamline__get_transaction_results_history.sql (97%) rename models/silver/streamline/core/history/{ => parameterized}/streamline__get_transactions_history.sql (97%) create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql create mode 100644 models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql create mode 100644 models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql create mode 100644 python/create_views/README.md create mode 100644 python/create_views/create_history_views.py create mode 100644 python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql create mode 100644 python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql create mode 100644 python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql create mode 100644 python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql diff --git a/.github/workflows/dbt_run_streamline_blocks.yml b/.github/workflows/dbt_run_streamline_blocks.yml index a9eddaf..3f2af8e 100644 --- a/.github/workflows/dbt_run_streamline_blocks.yml +++ b/.github/workflows/dbt_run_streamline_blocks.yml @@ -44,11 +44,7 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_blocks_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' - - - name: Run DBT History Jobs - Mainnet22 - run: | - dbt run -s 1+streamline__get_blocks_history --vars '{"STREAMLINE_INVOKE_STREAMS": True, "node_url": "access-001.mainnet22.nodes.onflow.org:9000", "start_block": 47169687, "end_block": 55114466}' - - name: Run DBT History Jobs - Mainnet19 + - name: Run Backfill Models run: | - dbt run -s 1+streamline__get_blocks_history --vars '{"STREAMLINE_INVOKE_STREAMS": True, "node_url": "access-001.mainnet19.nodes.onflow.org:9000", "start_block": 35858811, "end_block": 40171633}' + dbt run -s models/silver/streamline/core/history/blocks --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/.github/workflows/dbt_run_streamline_collections.yml b/.github/workflows/dbt_run_streamline_collections.yml index 71e414a..f3c68f1 100644 --- a/.github/workflows/dbt_run_streamline_collections.yml +++ b/.github/workflows/dbt_run_streamline_collections.yml @@ -44,3 +44,7 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_collections_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' + + - name: Run Backfill Models + run: | + dbt run -s models/silver/streamline/core/history/collections --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/.github/workflows/dbt_run_streamline_hourly.yml b/.github/workflows/dbt_run_streamline_hourly.yml deleted file mode 100644 index e30710e..0000000 --- a/.github/workflows/dbt_run_streamline_hourly.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: dbt_run_streamline_hourly -run-name: dbt_run_streamline_hourly - -on: - workflow_dispatch: - # disabling 10/10/23 moving workflow to dbt_run_scheduled - # schedule: - # Runs "every hour" (see https://crontab.guru) - # - cron: '0 */1 * * *' - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; - dbt seed; - dbt run -s tag:streamline_load tag:streamline_complete - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit diff --git a/.github/workflows/dbt_run_streamline_transaction_results.yml b/.github/workflows/dbt_run_streamline_transaction_results.yml index a86c8fa..9b1748d 100644 --- a/.github/workflows/dbt_run_streamline_transaction_results.yml +++ b/.github/workflows/dbt_run_streamline_transaction_results.yml @@ -44,3 +44,7 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_transaction_results_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True, "producer_batch_size": 60000, "worker_batch_size": 2000}' + + - name: Run Backfill Models + run: | + dbt run -s models/silver/streamline/core/history/transaction_results --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/.github/workflows/dbt_run_streamline_transactions.yml b/.github/workflows/dbt_run_streamline_transactions.yml index fdd37be..628c658 100644 --- a/.github/workflows/dbt_run_streamline_transactions.yml +++ b/.github/workflows/dbt_run_streamline_transactions.yml @@ -44,3 +44,7 @@ jobs: - name: Run DBT Realtime run: | dbt run -s 1+streamline__get_transactions_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' + + - name: Run Backfill Models + run: | + dbt run -s models/silver/streamline/core/history/transactions --vars '{"STREAMLINE_INVOKE_STREAMS": True, "STREAMLINE_RUN_HISTORY": True}' diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet22.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql similarity index 62% rename from models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet22.sql rename to models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql index 25b6bcf..a734eed 100644 --- a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet22.sql +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_07.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'blocks', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'blocks_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} @@ -14,7 +14,7 @@ WITH blocks AS ( {{ ref("streamline__blocks") }} EXCEPT SELECT - block_number as block_height + block_number AS block_height FROM {{ ref("streamline__complete_get_blocks") }} ) @@ -28,6 +28,7 @@ SELECT FROM blocks WHERE - block_height BETWEEN 47169687 AND 55114466 -- Mainnet22 block range + block_height BETWEEN 4132133 + AND 4972986 ORDER BY - block_height ASC \ No newline at end of file + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql new file mode 100644 index 0000000..a5506ca --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_08.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'blocks_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 4972987 + AND 6483245 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql new file mode 100644 index 0000000..f4352e7 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_candidate_09.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'blocks_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 6483246 + AND 7601062 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql new file mode 100644 index 0000000..6348465 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_01.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'blocks_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 7601063 + AND 8742958 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql new file mode 100644 index 0000000..12bb36f --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_02.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'blocks_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 8742959 + AND 9737132 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql new file mode 100644 index 0000000..6ad8639 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_03.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'blocks_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 9737133 + AND 9992019 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql new file mode 100644 index 0000000..48efe19 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_04.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'blocks_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 9992020 + AND 12020336 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql new file mode 100644 index 0000000..0e9fb48 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_05.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'blocks_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 12020337 + AND 12609236 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql new file mode 100644 index 0000000..d8082a3 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_06.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'blocks_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 12609237 + AND 13404173 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql new file mode 100644 index 0000000..475d0d7 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_07.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'blocks_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 13404174 + AND 13950741 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql new file mode 100644 index 0000000..91da0f9 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_08.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'blocks_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 13950742 + AND 14892103 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql new file mode 100644 index 0000000..eff828d --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_09.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'blocks_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 14892104 + AND 15791890 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql new file mode 100644 index 0000000..9019d48 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_10.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'blocks_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 15791891 + AND 16755601 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql new file mode 100644 index 0000000..abdf614 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_11.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'blocks_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 16755602 + AND 17544522 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql new file mode 100644 index 0000000..c2cf23b --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_12.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'blocks_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 17544523 + AND 18587477 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql new file mode 100644 index 0000000..9be0489 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_13.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'blocks_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 18587478 + AND 19050752 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql new file mode 100644 index 0000000..a476671 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_14.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'blocks_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 19050753 + AND 21291691 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql new file mode 100644 index 0000000..a8fc830 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_15.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'blocks_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 21291692 + AND 23830812 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql new file mode 100644 index 0000000..65c97bd --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_16.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'blocks_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 23830813 + AND 27341469 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql new file mode 100644 index 0000000..d3c59fb --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_17.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'blocks_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 27341470 + AND 31735954 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql new file mode 100644 index 0000000..6e31aae --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_18.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'blocks_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 31735955 + AND 35858810 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql new file mode 100644 index 0000000..62556ce --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_19.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'blocks_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 35858811 + AND 40171633 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql new file mode 100644 index 0000000..3b15369 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_20.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'blocks_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 40171634 + AND 44950206 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql new file mode 100644 index 0000000..e6b4c93 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_21.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'blocks_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 44950207 + AND 47169686 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql new file mode 100644 index 0000000..d458e13 --- /dev/null +++ b/models/silver/streamline/core/history/blocks/streamline__get_blocks_history_mainnet_22.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'blocks_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN 47169687 + AND 55114466 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql new file mode 100644 index 0000000..589bf28 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_07.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'transactions_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 4132133 + AND 4972986 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql new file mode 100644 index 0000000..733e277 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_08.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'transactions_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 4972987 + AND 6483245 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql new file mode 100644 index 0000000..de98325 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_candidate_09.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'transactions_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 6483246 + AND 7601062 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql new file mode 100644 index 0000000..1f52544 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_01.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'transactions_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 7601063 + AND 8742958 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql new file mode 100644 index 0000000..9968d02 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_02.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'transactions_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 8742959 + AND 9737132 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql new file mode 100644 index 0000000..b4eafdb --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_03.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'transactions_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 9737133 + AND 9992019 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql new file mode 100644 index 0000000..418385a --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_04.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'transactions_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 9992020 + AND 12020336 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql new file mode 100644 index 0000000..820a85a --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_05.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'transactions_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 12020337 + AND 12609236 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql new file mode 100644 index 0000000..6959bec --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_06.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'transactions_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 12609237 + AND 13404173 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql new file mode 100644 index 0000000..06e6007 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_07.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'transactions_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 13404174 + AND 13950741 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql new file mode 100644 index 0000000..6afe007 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_08.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'transactions_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 13950742 + AND 14892103 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql new file mode 100644 index 0000000..61b60c0 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_09.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'transactions_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 14892104 + AND 15791890 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql new file mode 100644 index 0000000..2f0bf5b --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_10.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'transactions_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 15791891 + AND 16755601 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql new file mode 100644 index 0000000..64df07b --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_11.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'transactions_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 16755602 + AND 17544522 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql new file mode 100644 index 0000000..37d78ed --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_12.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'transactions_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 17544523 + AND 18587477 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql new file mode 100644 index 0000000..d080f78 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_13.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'transactions_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 18587478 + AND 19050752 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql new file mode 100644 index 0000000..4cdcf66 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_14.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transactions_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 19050753 + AND 21291691 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql new file mode 100644 index 0000000..4859138 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_15.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transactions_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 21291692 + AND 23830812 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql new file mode 100644 index 0000000..301df79 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_16.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transactions_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 23830813 + AND 27341469 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql new file mode 100644 index 0000000..a676df8 --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_17.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transactions_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 27341470 + AND 31735954 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql new file mode 100644 index 0000000..2a122df --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_18.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transactions_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 31735955 + AND 35858810 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql new file mode 100644 index 0000000..1d22c5c --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_19.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transactions_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 35858811 + AND 40171633 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql new file mode 100644 index 0000000..2880e8a --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_20.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'transactions_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 40171634 + AND 44950206 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql new file mode 100644 index 0000000..ce14a9b --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_21.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'transactions_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 44950207 + AND 47169686 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql new file mode 100644 index 0000000..282fc1a --- /dev/null +++ b/models/silver/streamline/core/history/collections/streamline__get_collections_history_mainnet_22.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transactions_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 47169687 + AND 55114466 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/streamline__get_blocks_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql similarity index 96% rename from models/silver/streamline/core/history/streamline__get_blocks_history.sql rename to models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql index 63ff62a..126d19b 100644 --- a/models/silver/streamline/core/history/streamline__get_blocks_history.sql +++ b/models/silver/streamline/core/history/parameterized/streamline__get_blocks_history.sql @@ -3,8 +3,7 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url', '{{ var('node_url', Null) }}','external_table', 'blocks', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ), - tags = ['streamline_history'] + ) ) }} WITH blocks AS ( diff --git a/models/silver/streamline/core/history/streamline__get_collections_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql similarity index 98% rename from models/silver/streamline/core/history/streamline__get_collections_history.sql rename to models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql index f25f2d0..492a8e0 100644 --- a/models/silver/streamline/core/history/streamline__get_collections_history.sql +++ b/models/silver/streamline/core/history/parameterized/streamline__get_collections_history.sql @@ -3,8 +3,7 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','{{ var('node_url', Null) }}','external_table', 'collections', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ), - tags = ['streamline_history'] + ) ) }} WITH blocks AS ( diff --git a/models/silver/streamline/core/history/streamline__get_transaction_results_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql similarity index 97% rename from models/silver/streamline/core/history/streamline__get_transaction_results_history.sql rename to models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql index 6135d76..5589ba0 100644 --- a/models/silver/streamline/core/history/streamline__get_transaction_results_history.sql +++ b/models/silver/streamline/core/history/parameterized/streamline__get_transaction_results_history.sql @@ -3,8 +3,7 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}', 'node_url', '{{ var('node_url', Null) }}', 'external_table', 'transaction_results', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','30000')}}, 'worker_batch_size', {{var('worker_batch_size','3000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ), - tags = ['streamline_history'] + ) ) }} WITH blocks AS ( diff --git a/models/silver/streamline/core/history/streamline__get_transactions_history.sql b/models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql similarity index 97% rename from models/silver/streamline/core/history/streamline__get_transactions_history.sql rename to models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql index d23c914..6b89267 100644 --- a/models/silver/streamline/core/history/streamline__get_transactions_history.sql +++ b/models/silver/streamline/core/history/parameterized/streamline__get_transactions_history.sql @@ -3,8 +3,7 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}', 'node_url', '{{ var('node_url', Null) }}', 'external_table', 'transactions', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','30000')}}, 'worker_batch_size', {{var('worker_batch_size','3000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ), - tags = ['streamline_history'] + ) ) }} WITH blocks AS ( diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql new file mode 100644 index 0000000..849fc71 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_07.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'transaction_results_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 4132133 + AND 4972986 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql new file mode 100644 index 0000000..e550d0d --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_08.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'transaction_results_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 4972987 + AND 6483245 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql new file mode 100644 index 0000000..f0a7771 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_candidate_09.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'transaction_results_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 6483246 + AND 7601062 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql new file mode 100644 index 0000000..e7bc03b --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_01.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 7601063 + AND 8742958 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql new file mode 100644 index 0000000..c5b2c99 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_02.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 8742959 + AND 9737132 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql new file mode 100644 index 0000000..ab8788d --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_03.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 9737133 + AND 9992019 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql new file mode 100644 index 0000000..b626965 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_04.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 9992020 + AND 12020336 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql new file mode 100644 index 0000000..c86ff89 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_05.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 12020337 + AND 12609236 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql new file mode 100644 index 0000000..6843805 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_06.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 12609237 + AND 13404173 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql new file mode 100644 index 0000000..160f2d6 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_07.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 13404174 + AND 13950741 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql new file mode 100644 index 0000000..082a7fe --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_08.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 13950742 + AND 14892103 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql new file mode 100644 index 0000000..7294ca2 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_09.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 14892104 + AND 15791890 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql new file mode 100644 index 0000000..164faff --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_10.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 15791891 + AND 16755601 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql new file mode 100644 index 0000000..bb3e4a4 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_11.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 16755602 + AND 17544522 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql new file mode 100644 index 0000000..418421b --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_12.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 17544523 + AND 18587477 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql new file mode 100644 index 0000000..1375cdd --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_13.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 18587478 + AND 19050752 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql new file mode 100644 index 0000000..6d2740f --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 19050753 + AND 21291691 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql new file mode 100644 index 0000000..5d1247a --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 21291692 + AND 23830812 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql new file mode 100644 index 0000000..d033d8b --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 23830813 + AND 27341469 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql new file mode 100644 index 0000000..2a6c165 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 27341470 + AND 31735954 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql new file mode 100644 index 0000000..ea0536b --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 31735955 + AND 35858810 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql new file mode 100644 index 0000000..0fe03d2 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 35858811 + AND 40171633 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql new file mode 100644 index 0000000..ce5a336 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_20.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 40171634 + AND 44950206 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql new file mode 100644 index 0000000..2e2353d --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_21.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 44950207 + AND 47169686 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql new file mode 100644 index 0000000..80bb4b1 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 47169687 + AND 55114466 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql new file mode 100644 index 0000000..fc030b3 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_07.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate7.nodes.onflow.org:9000','external_table', 'transactions_candidate_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 4132133 + AND 4972986 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql new file mode 100644 index 0000000..f951264 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_08.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate8.nodes.onflow.org:9000','external_table', 'transactions_candidate_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 4972987 + AND 6483245 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql new file mode 100644 index 0000000..d5f33a1 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_candidate_09.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.candidate9.nodes.onflow.org:9000','external_table', 'transactions_candidate_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 6483246 + AND 7601062 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql new file mode 100644 index 0000000..46224ac --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_01.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet1.nodes.onflow.org:9000','external_table', 'transactions_mainnet_01', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 7601063 + AND 8742958 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql new file mode 100644 index 0000000..7feab2a --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_02.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet2.nodes.onflow.org:9000','external_table', 'transactions_mainnet_02', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 8742959 + AND 9737132 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql new file mode 100644 index 0000000..6cc2466 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_03.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet3.nodes.onflow.org:9000','external_table', 'transactions_mainnet_03', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 9737133 + AND 9992019 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql new file mode 100644 index 0000000..a38c81c --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_04.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet4.nodes.onflow.org:9000','external_table', 'transactions_mainnet_04', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 9992020 + AND 12020336 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql new file mode 100644 index 0000000..2ddec96 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_05.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet5.nodes.onflow.org:9000','external_table', 'transactions_mainnet_05', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 12020337 + AND 12609236 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql new file mode 100644 index 0000000..343a496 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_06.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet6.nodes.onflow.org:9000','external_table', 'transactions_mainnet_06', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 12609237 + AND 13404173 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql new file mode 100644 index 0000000..6305494 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_07.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet7.nodes.onflow.org:9000','external_table', 'transactions_mainnet_07', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 13404174 + AND 13950741 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql new file mode 100644 index 0000000..503ca5d --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_08.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet8.nodes.onflow.org:9000','external_table', 'transactions_mainnet_08', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 13950742 + AND 14892103 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql new file mode 100644 index 0000000..a554027 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_09.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet9.nodes.onflow.org:9000','external_table', 'transactions_mainnet_09', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 14892104 + AND 15791890 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql new file mode 100644 index 0000000..cedc9db --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_10.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet10.nodes.onflow.org:9000','external_table', 'transactions_mainnet_10', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 15791891 + AND 16755601 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql new file mode 100644 index 0000000..5ce0028 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_11.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet11.nodes.onflow.org:9000','external_table', 'transactions_mainnet_11', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 16755602 + AND 17544522 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql new file mode 100644 index 0000000..f22d389 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_12.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet12.nodes.onflow.org:9000','external_table', 'transactions_mainnet_12', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 17544523 + AND 18587477 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql new file mode 100644 index 0000000..9790973 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_13.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet13.nodes.onflow.org:9000','external_table', 'transactions_mainnet_13', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 18587478 + AND 19050752 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql new file mode 100644 index 0000000..048a448 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_14.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transactions_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 19050753 + AND 21291691 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql new file mode 100644 index 0000000..b77ce2a --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_15.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transactions_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 21291692 + AND 23830812 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql new file mode 100644 index 0000000..f42f7eb --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_16.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transactions_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 23830813 + AND 27341469 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql new file mode 100644 index 0000000..1d1e560 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_17.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transactions_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 27341470 + AND 31735954 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql new file mode 100644 index 0000000..108c915 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transactions_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 31735955 + AND 35858810 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql new file mode 100644 index 0000000..228e9a8 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transactions_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 35858811 + AND 40171633 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql new file mode 100644 index 0000000..5bcd35a --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_20.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet20.nodes.onflow.org:9000','external_table', 'transactions_mainnet_20', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 40171634 + AND 44950206 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql new file mode 100644 index 0000000..dd4f463 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_21.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet21.nodes.onflow.org:9000','external_table', 'transactions_mainnet_21', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 44950207 + AND 47169686 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql new file mode 100644 index 0000000..2a7e343 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_22.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transactions_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN 47169687 + AND 55114466 +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/realtime/streamline__get_transaction_results_realtime.sql b/models/silver/streamline/core/realtime/streamline__get_transaction_results_realtime.sql index fc8f49b..5790cd0 100644 --- a/models/silver/streamline/core/realtime/streamline__get_transaction_results_realtime.sql +++ b/models/silver/streamline/core/realtime/streamline__get_transaction_results_realtime.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access.mainnet.nodes.onflow.org:9000','external_table', 'transaction_results', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access.mainnet.nodes.onflow.org:9000','external_table', 'transaction_results', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ), tags = ['streamline_realtime'] diff --git a/python/create_views/README.md b/python/create_views/README.md new file mode 100644 index 0000000..c2b1b19 --- /dev/null +++ b/python/create_views/README.md @@ -0,0 +1,3 @@ +# Create Views for Backfill + +Read the seed file of network versions and output history model(s) for each network version. diff --git a/python/create_views/create_history_views.py b/python/create_views/create_history_views.py new file mode 100644 index 0000000..7a5b202 --- /dev/null +++ b/python/create_views/create_history_views.py @@ -0,0 +1,38 @@ +import csv +import os + +# Define the paths to the CSV and SQL template files +csv_file_path = '../../data/seeds__network_version.csv' +sql_template_dir = 'input' +output_dir = 'output' + +# Get the SQL template file names +sql_template_files = [f for f in os.listdir(sql_template_dir) if f.endswith('.sql')] + +# Read the network data from the CSV +with open(csv_file_path, 'r') as f: + reader = csv.DictReader(f) + network_data = {row['network_version']: {'root_height': int(row['root_height']), 'node_url': row['node_url'], 'end_height': int(row['end_height'])} for row in reader} + +# Loop through the network data and replace the variables in the SQL templates +for network, data in network_data.items(): + # Loop through the SQL template file names + for sql_template_file in sql_template_files: + # Define the path to the SQL template file + sql_template_path = os.path.join(sql_template_dir, sql_template_file) + # Read the SQL template file + with open(sql_template_path, 'r') as f: + sql = f.read() + # Replace the variables in the SQL template + sql = sql.replace('NETWORK_NODE_URL', data['node_url']) + sql = sql.replace('NETWORK_VERSION', network.replace('-', '_')) + sql = sql.replace('root_height', str(data['root_height'])) + sql = sql.replace('end_height', str(data['end_height'])) + # Replace the NETWORK_VERSION keyword with the network name + sql_file_name = sql_template_file.replace('NETWORK_VERSION', network.replace('-', '_')) + # Define the path to the output SQL file + output_sql_path = os.path.join(output_dir, sql_file_name) + # Write the modified SQL to the output file + with open(output_sql_path, 'w') as f: + f.write(sql) + print(f"File {output_sql_path} created.") diff --git a/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql new file mode 100644 index 0000000..c99ce50 --- /dev/null +++ b/python/create_views/input/streamline__get_blocks_history_NETWORK_VERSION.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'blocks_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_blocks") }} +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_block_by_height', + 'block_height', block_height, + 'method_params', OBJECT_CONSTRUCT('height', block_height) + ) AS request +FROM + blocks +WHERE + block_height BETWEEN root_height + AND end_height +ORDER BY + block_height ASC diff --git a/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql new file mode 100644 index 0000000..d006eb9 --- /dev/null +++ b/python/create_views/input/streamline__get_collections_history_NETWORK_VERSION.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'transactions_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN root_height + AND end_height +ORDER BY + block_height ASC diff --git a/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql new file mode 100644 index 0000000..54e9599 --- /dev/null +++ b/python/create_views/input/streamline__get_transaction_results_history_NETWORK_VERSION.sql @@ -0,0 +1,45 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'transaction_results_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results") }} +), +tx AS ( + SELECT + block_number AS block_height, + DATA + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks + ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction_result', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value :: STRING) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(DATA) :transaction_ids) AS transaction_id +WHERE + block_height BETWEEN root_height + AND end_height +ORDER BY + block_height ASC diff --git a/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql b/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql new file mode 100644 index 0000000..2b49509 --- /dev/null +++ b/python/create_views/input/streamline__get_transactions_history_NETWORK_VERSION.sql @@ -0,0 +1,43 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','NETWORK_NODE_URL','external_table', 'transactions_NETWORK_VERSION', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ) +) }} + +WITH blocks AS ( + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + EXCEPT + SELECT + block_number as block_height + FROM + {{ ref("streamline__complete_get_transactions") }} +), +tx AS ( + SELECT + block_number as block_height, + data + FROM + {{ ref('streamline__complete_get_collections') }} + JOIN blocks ON blocks.block_height = block_number +) +SELECT + OBJECT_CONSTRUCT( + 'grpc', 'proto3', + 'method', 'get_transaction', + 'block_height', block_height::INTEGER, + 'transaction_id', transaction_id.value::string, + 'method_params', OBJECT_CONSTRUCT('id', transaction_id.value::string) + ) AS request +FROM + tx, + LATERAL FLATTEN(input => TRY_PARSE_JSON(data):transaction_ids) AS transaction_id +WHERE + block_height BETWEEN root_height + AND end_height +ORDER BY + block_height ASC