diff --git a/dbt_project.yml b/dbt_project.yml index e8fc3d5..16f5f92 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -57,11 +57,11 @@ query-comment: vars: "dbt_date:time_zone": GMT - STREAMLINE_INVOKE_STREAMS: TRUE - STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: true - "UPDATE_SNOWFLAKE_TAGS": True - OBSERV_FULL_TEST: False - START_GHA_TASKS: False + STREAMLINE_INVOKE_STREAMS: FALSE + STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: FALSE + UPDATE_SNOWFLAKE_TAGS: TRUE + OBSERV_FULL_TEST: FALSE + START_GHA_TASKS: FALSE #### STREAMLINE 2.0 BEGIN #### @@ -75,11 +75,14 @@ vars: API_INTEGRATION: aws_axelar_api_dev EXTERNAL_FUNCTION_URI: 5v36zckfli.execute-api.us-east-1.amazonaws.com/stg/ ROLES: + - AWS_LAMBDA_AXELAR_API - INTERNAL_DEV prod: API_INTEGRATION: aws_axelar_api - EXTERNAL_FUNCTION_URI: '' + EXTERNAL_FUNCTION_URI: sfl36j9j2c.execute-api.us-east-1.amazonaws.com/prod/ ROLES: + - AWS_LAMBDA_AXELAR_API + - DBT_CLOUD_AXELAR - INTERNAL_DEV #### STREAMLINE 2.0 END #### \ No newline at end of file diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 27f2d12..e77e91b 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -9,9 +9,7 @@ {% endset %} {% do run_query(sql) %} {% set sql %} - {{ create_udf_bulk_get_blocks() }} - {{ create_udf_bulk_get_txs() }} - {{ create_udf_bulk_get_validators() }} + {{ create_udf_rest_api() }} {% endset %} {% do run_query(sql) %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql index 6d6d4f9..e5f830b 100644 --- a/macros/streamline/api_integrations.sql +++ b/macros/streamline/api_integrations.sql @@ -5,8 +5,8 @@ {% if target.name == "prod" %} {% set sql %} - CREATE api integration IF NOT EXISTS aws_axelar_api api_provider = aws_api_gateway api_aws_role_arn = '' api_allowed_prefixes = ( - '' + CREATE api integration IF NOT EXISTS aws_axelar_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::924682671219:role/axelar-api-prod-rolesnowflakeudfsAF733095-T68fPXKYBRAj' api_allowed_prefixes = ( + 'https://sfl36j9j2c.execute-api.us-east-1.amazonaws.com/prod/' ) enabled = TRUE; {% endset %} {% do run_query(sql) %} diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index fd0cf1f..42e9df6 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -36,3 +36,13 @@ {%- endif %}; {% endmacro %} +{% macro create_udf_rest_api() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_rest_api( + json OBJECT + ) returns ARRAY api_integration = {% if target.name == "prod" %} + aws_axelar_api AS 'https://sfl36j9j2c.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api' + {% else %} + aws_axelar_api_dev AS 'https://5v36zckfli.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api' + {%- endif %}; +{% endmacro %} diff --git a/models/bronze/streamline/bronze__streamline_FR_blocks.sql b/models/bronze/streamline/bronze__streamline_FR_blocks.sql index cb56d4d..6e425a7 100644 --- a/models/bronze/streamline/bronze__streamline_FR_blocks.sql +++ b/models/bronze/streamline/bronze__streamline_FR_blocks.sql @@ -12,7 +12,7 @@ SELECT _PARTITION_BY_BLOCK_ID, VALUE FROM - {{ ref('bronze__streamline_FR_transactions_v2') }} + {{ ref('bronze__streamline_FR_blocks_v2') }} UNION ALL SELECT block_number, @@ -24,4 +24,4 @@ SELECT _PARTITION_BY_BLOCK_ID, VALUE FROM - {{ ref('bronze__streamline_FR_transactions_v1') }} + {{ ref('bronze__streamline_FR_blocks_v1') }} diff --git a/models/bronze/streamline/bronze__streamline_FR_transactions.sql b/models/bronze/streamline/bronze__streamline_FR_transactions.sql index bdee4e2..cb56d4d 100644 --- a/models/bronze/streamline/bronze__streamline_FR_transactions.sql +++ b/models/bronze/streamline/bronze__streamline_FR_transactions.sql @@ -1,9 +1,27 @@ {{ config ( materialized = 'view' ) }} -{{ streamline_external_table_FR_query( - "txs_v2", - partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )", - partition_name = "_partition_by_block_id", - unique_key = "partition_key" -) }} + +SELECT + partition_key, + DATA, + _INSERTED_TIMESTAMP, + id, + metadata, + file_name, + _PARTITION_BY_BLOCK_ID, + VALUE +FROM + {{ ref('bronze__streamline_FR_transactions_v2') }} +UNION ALL +SELECT + block_number, + DATA, + _INSERTED_TIMESTAMP, + id, + metadata, + file_name, + _PARTITION_BY_BLOCK_ID, + VALUE +FROM + {{ ref('bronze__streamline_FR_transactions_v1') }} diff --git a/models/streamline/complete/streamline__complete_tx_counts.sql b/models/streamline/complete/streamline__complete_tx_counts.sql index 3d847b9..28e64de 100644 --- a/models/streamline/complete/streamline__complete_tx_counts.sql +++ b/models/streamline/complete/streamline__complete_tx_counts.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_transactions') }} +-- depends_on: {{ ref('bronze__streamline_tx_counts') }} {{ config ( materialized = "incremental", unique_key = "block_number", @@ -7,7 +7,13 @@ ) }} SELECT - DATA :height :: INT AS block_number, + REPLACE( + COALESCE( + metadata :request :data :params [0], + metadata :request :params [0] + ), + 'tx.height=' + ) :: INT AS block_number, DATA :: INT AS tx_count, {{ dbt_utils.generate_surrogate_key( ['block_number'] @@ -17,9 +23,9 @@ SELECT _inserted_timestamp, '{{ invocation_id }}' AS _invocation_id FROM - {{ ref('bronze__streamline_tx_counts') }} {% if is_incremental() %} +{{ ref('bronze__streamline_tx_counts') }} WHERE _inserted_timestamp >= ( SELECT diff --git a/models/streamline/realtime/streamline__transactions_realtime.sql b/models/streamline/realtime/streamline__transactions_realtime.sql index b797bb6..a4afb24 100644 --- a/models/streamline/realtime/streamline__transactions_realtime.sql +++ b/models/streamline/realtime/streamline__transactions_realtime.sql @@ -22,9 +22,8 @@ ON A.block_number = b.block_number WHERE b.block_number IS NULL {% endif %} -LIMIT - 10 -), numbers AS ( +), +numbers AS ( -- Recursive CTE to generate numbers. We'll use the maximum txcount value to limit our recursion. SELECT 1 AS n @@ -53,6 +52,8 @@ LIMIT tt.tx_count / 100 ) + 1 END + WHERE + tt.tx_count > 0 ) SELECT ROUND( diff --git a/models/streamline/streamline__blocks.sql b/models/streamline/streamline__blocks.sql index 264c8f1..3923098 100644 --- a/models/streamline/streamline__blocks.sql +++ b/models/streamline/streamline__blocks.sql @@ -3,13 +3,6 @@ tags = ['streamline_view'] ) }} -{% if execute %} - {% set height = run_query("SELECT live.udf_api( 'POST', '{service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json' ), OBJECT_CONSTRUCT( 'id', 0, 'jsonrpc', '2.0', 'method', 'status', 'params', [] ), 'vault/stg/axelar/node/mainnet' ):data:result:sync_info:latest_block_height::INT as block") %} - {% set block_height = height.columns [0].values() [0] %} -{% else %} - {% set block_height = 0 %} -{% endif %} - SELECT _id AS block_number FROM @@ -18,4 +11,9 @@ FROM 'number_sequence' ) }} WHERE - _id <= {{ block_height }} + _id <= ( + SELECT + MAX(block_number) + FROM + {{ ref('streamline__chainhead') }} + ) diff --git a/models/streamline/streamline__chainhead.sql b/models/streamline/streamline__chainhead.sql new file mode 100644 index 0000000..2e63790 --- /dev/null +++ b/models/streamline/streamline__chainhead.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = "view", + tags = ['streamline_view'] +) }} + +SELECT + {{ target.database }}.live.udf_api( + 'POST', + '{service}/{Authentication}', + OBJECT_CONSTRUCT( + 'Content-Type', + 'application/json', + 'fsc-quantum-state', + 'livequery' + ), + OBJECT_CONSTRUCT( + 'id', + 0, + 'jsonrpc', + '2.0', + 'method', + 'status', + 'params', + [] + ), + 'vault/prod/axelar/node/mainnet' + ) :data :result :sync_info :latest_block_height :: INT AS block_number diff --git a/package-lock.yml b/package-lock.yml index e50fc5f..f2e6af4 100644 --- a/package-lock.yml +++ b/package-lock.yml @@ -6,11 +6,11 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: e670a0edf1c1bb2316212b368724dc9942b6220f + revision: e94e3d6964f10d8fcca239a64ec47bc8a230261e - package: get-select/dbt_snowflake_query_tags version: 2.3.3 - package: calogica/dbt_date version: 0.7.2 - git: https://github.com/FlipsideCrypto/livequery-models.git - revision: de2ade547b8e87be0902605d42152ed97f53e05d -sha1_hash: 23a3f5af6aea92b1391c89c7e8a7024fa822e65f + revision: 8b105444ef516f94efd6b9a420e62fe9f69277f9 +sha1_hash: c82fc81890babf0760cd6426e803d0bff21d9264 diff --git a/packages.yml b/packages.yml index 05126e0..36887f0 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: [">=1.0.0", "<1.1.0"] - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.21.6-dev + revision: v1.22.0 - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] \ No newline at end of file