diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql index 1917c13..d2fc068 100644 --- a/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql @@ -2,12 +2,21 @@ materialized = 'view' ) }} +{# {% set table_names = [ 'COLLECTIONS_CANDIDATE_07', 'COLLECTIONS_CANDIDATE_08', 'COLLECTIONS_CANDIDATE_09', 'COLLECTIONS_MAINNET_01', 'COLLECTIONS_MAINNET_02', 'COLLECTIONS_MAINNET_03', 'COLLECTIONS_MAINNET_04', 'COLLECTIONS_MAINNET_05', 'COLLECTIONS_MAINNET_06', 'COLLECTIONS_MAINNET_07', 'COLLECTIONS_MAINNET_08', 'COLLECTIONS_MAINNET_09', 'COLLECTIONS_MAINNET_10', 'COLLECTIONS_MAINNET_11', 'COLLECTIONS_MAINNET_12', 'COLLECTIONS_MAINNET_13', 'COLLECTIONS_MAINNET_14', 'COLLECTIONS_MAINNET_15', 'COLLECTIONS_MAINNET_16', 'COLLECTIONS_MAINNET_17', 'COLLECTIONS_MAINNET_18', 'COLLECTIONS_MAINNET_19', 'COLLECTIONS_MAINNET_20', 'COLLECTIONS_MAINNET_21', 'COLLECTIONS_MAINNET_22' ] %} +#} + +{% set + table_names = + [ + 'COLLECTIONS_MAINNET_18' + ] +%} {{ streamline_multiple_external_table_query( table_names, diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql index 91c8cf8..c31aa45 100644 --- a/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql @@ -23,7 +23,7 @@ {% set table_names = [ - 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_22' + 'TRANSACTION_RESULTS_MAINNET_18' ] %} diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql new file mode 100644 index 0000000..718cd15 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql @@ -0,0 +1,66 @@ +{{ config ( + materialized = "view", + post_hook = fsc_utils.if_data_call_function_v2( + func = 'udf_bulk_grpc_us_east_2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "node_url":"access-001.mainnet18.nodes.onflow.org:9000", + "external_table": "transaction_results_batch_mainnet_18", + "sql_limit": "100", + "producer_batch_size": "100", + "worker_batch_size": "100", + "sql_source": "{{this.identifier}}" + } + ) +) }} + +WITH blocks AS ( +-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 31735955 + AND 35858810 + EXCEPT + SELECT + distinct block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results_history") }} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_results_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql index 2bbf6b6..3f55456 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql @@ -1,8 +1,16 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_18', 'sql_limit', '500000', 'producer_batch_size', '1250', 'worker_batch_size', '1', 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" + post_hook = fsc_utils.if_data_call_function_v2( + func = 'udf_bulk_grpc_us_east_2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "node_url":"access-001.mainnet18.nodes.onflow.org:9000", + "external_table": "transaction_results_mainnet_18", + "sql_limit": "25000", + "producer_batch_size": "1000", + "worker_batch_size": "200", + "sql_source": "{{this.identifier}}" + } ) ) }} diff --git a/package-lock.yml b/package-lock.yml index 34171f2..2de10b9 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -1,16 +1,16 @@ packages: - - package: calogica/dbt_expectations - version: 0.8.0 - - package: dbt-labs/dbt_external_tables - version: 0.8.0 - - package: dbt-labs/dbt_utils - version: 1.0.0 - - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: 3a45311f3e94c8a695443f77aab433a05b894f90 - - package: get-select/dbt_snowflake_query_tags - version: 2.3.2 - - package: calogica/dbt_date - version: 0.7.2 - - git: https://github.com/FlipsideCrypto/livequery-models.git - revision: bca494102fbd2d621d32746e9a7fe780678044f8 -sha1_hash: 82abbdb1baa6b6b88e7d0e40f3ee8aaf2850e052 +- package: calogica/dbt_expectations + version: 0.8.0 +- package: dbt-labs/dbt_external_tables + version: 0.8.0 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: ec7703e90ad70dfd997abe57717a07400aae2f17 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.3 +- package: calogica/dbt_date + version: 0.7.2 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: bb079c5efcbe9e6e5f17498d0c787df276e58f67 diff --git a/packages.yml b/packages.yml index 97f06d3..5c945f5 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.9.3" + revision: "v1.17.2" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"]