final updates

dbt_project.yml

@@ -57,11 +57,11 @@ query-comment:
 
 vars:
   "dbt_date:time_zone": GMT
-  STREAMLINE_INVOKE_STREAMS: TRUE
-  STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: true
-  "UPDATE_SNOWFLAKE_TAGS": True
-  OBSERV_FULL_TEST: False
-  START_GHA_TASKS: False
+  STREAMLINE_INVOKE_STREAMS: FALSE
+  STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: FALSE
+  UPDATE_SNOWFLAKE_TAGS: TRUE
+  OBSERV_FULL_TEST: FALSE
+  START_GHA_TASKS: FALSE
 
 #### STREAMLINE 2.0 BEGIN ####
 
@@ -75,11 +75,14 @@ vars:
       API_INTEGRATION: aws_axelar_api_dev
       EXTERNAL_FUNCTION_URI: 5v36zckfli.execute-api.us-east-1.amazonaws.com/stg/
       ROLES:
+        - AWS_LAMBDA_AXELAR_API
         - INTERNAL_DEV
     prod:
       API_INTEGRATION: aws_axelar_api
-      EXTERNAL_FUNCTION_URI: ''
+      EXTERNAL_FUNCTION_URI: sfl36j9j2c.execute-api.us-east-1.amazonaws.com/prod/
       ROLES:
+        - AWS_LAMBDA_AXELAR_API
+        - DBT_CLOUD_AXELAR
         - INTERNAL_DEV
 
 #### STREAMLINE 2.0 END ####

macros/create_udfs.sql

@@ -9,9 +9,7 @@
     {% endset %}
     {% do run_query(sql) %}
     {% set sql %}
-    {{ create_udf_bulk_get_blocks() }}
-    {{ create_udf_bulk_get_txs() }}
-    {{ create_udf_bulk_get_validators() }}
+    {{ create_udf_rest_api() }}
 
     {% endset %}
     {% do run_query(sql) %}

macros/streamline/api_integrations.sql

@@ -5,8 +5,8 @@
 
     {% if target.name == "prod" %}
         {% set sql %}
-        CREATE api integration IF NOT EXISTS aws_axelar_api api_provider = aws_api_gateway api_aws_role_arn = '' api_allowed_prefixes = (
-            ''
+        CREATE api integration IF NOT EXISTS aws_axelar_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::924682671219:role/axelar-api-prod-rolesnowflakeudfsAF733095-T68fPXKYBRAj' api_allowed_prefixes = (
+            'https://sfl36j9j2c.execute-api.us-east-1.amazonaws.com/prod/'
         ) enabled = TRUE;
 {% endset %}
         {% do run_query(sql) %}

macros/streamline/streamline_udfs.sql

@@ -36,3 +36,13 @@
     {%- endif %};
 {% endmacro %}
 
+{% macro create_udf_rest_api() %}
+    CREATE
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_rest_api(
+        json OBJECT
+    ) returns ARRAY api_integration = {% if target.name == "prod" %}
+        aws_axelar_api AS 'https://sfl36j9j2c.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api'
+    {% else %}
+        aws_axelar_api_dev AS 'https://5v36zckfli.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api'
+    {%- endif %};
+{% endmacro %}

models/bronze/streamline/bronze__streamline_FR_blocks.sql

@@ -12,7 +12,7 @@ SELECT
     _PARTITION_BY_BLOCK_ID,
     VALUE
 FROM
-    {{ ref('bronze__streamline_FR_transactions_v2') }}
+    {{ ref('bronze__streamline_FR_blocks_v2') }}
 UNION ALL
 SELECT
     block_number,
@@ -24,4 +24,4 @@ SELECT
     _PARTITION_BY_BLOCK_ID,
     VALUE
 FROM
-    {{ ref('bronze__streamline_FR_transactions_v1') }}
+    {{ ref('bronze__streamline_FR_blocks_v1') }}

models/bronze/streamline/bronze__streamline_FR_transactions.sql

@@ -1,9 +1,27 @@
 {{ config (
     materialized = 'view'
 ) }}
-{{ streamline_external_table_FR_query(
-    "txs_v2",
-    partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER )",
-    partition_name = "_partition_by_block_id",
-    unique_key = "partition_key"
-) }}
+
+SELECT
+    partition_key,
+    DATA,
+    _INSERTED_TIMESTAMP,
+    id,
+    metadata,
+    file_name,
+    _PARTITION_BY_BLOCK_ID,
+    VALUE
+FROM
+    {{ ref('bronze__streamline_FR_transactions_v2') }}
+UNION ALL
+SELECT
+    block_number,
+    DATA,
+    _INSERTED_TIMESTAMP,
+    id,
+    metadata,
+    file_name,
+    _PARTITION_BY_BLOCK_ID,
+    VALUE
+FROM
+    {{ ref('bronze__streamline_FR_transactions_v1') }}

models/streamline/complete/streamline__complete_tx_counts.sql

@@ -1,4 +1,4 @@
--- depends_on: {{ ref('bronze__streamline_transactions') }}
+-- depends_on: {{ ref('bronze__streamline_tx_counts') }}
 {{ config (
     materialized = "incremental",
     unique_key = "block_number",
@@ -7,7 +7,13 @@
 ) }}
 
 SELECT
-    DATA :height :: INT AS block_number,
+    REPLACE(
+        COALESCE(
+            metadata :request :data :params [0],
+            metadata :request :params [0]
+        ),
+        'tx.height='
+    ) :: INT AS block_number,
     DATA :: INT AS tx_count,
     {{ dbt_utils.generate_surrogate_key(
         ['block_number']
@@ -17,9 +23,9 @@ SELECT
     _inserted_timestamp,
     '{{ invocation_id }}' AS _invocation_id
 FROM
-    {{ ref('bronze__streamline_tx_counts') }}
 
 {% if is_incremental() %}
+{{ ref('bronze__streamline_tx_counts') }}
 WHERE
     _inserted_timestamp >= (
         SELECT

models/streamline/realtime/streamline__transactions_realtime.sql

@@ -22,9 +22,8 @@ ON A.block_number = b.block_number
 WHERE
     b.block_number IS NULL
 {% endif %}
-LIMIT
-    10
-), numbers AS (
+),
+numbers AS (
     -- Recursive CTE to generate numbers. We'll use the maximum txcount value to limit our recursion.
     SELECT
         1 AS n
@@ -53,6 +52,8 @@ LIMIT
                         tt.tx_count / 100
                     ) + 1
                 END
+            WHERE
+                tt.tx_count > 0
         )
     SELECT
         ROUND(

models/streamline/streamline__blocks.sql

@@ -3,13 +3,6 @@
     tags = ['streamline_view']
 ) }}
 
-{% if execute %}
-    {% set height = run_query("SELECT live.udf_api( 'POST', '{service}/{Authentication}', OBJECT_CONSTRUCT( 'Content-Type', 'application/json' ), OBJECT_CONSTRUCT( 'id', 0, 'jsonrpc', '2.0', 'method', 'status', 'params', [] ), 'vault/stg/axelar/node/mainnet' ):data:result:sync_info:latest_block_height::INT as block") %}
-    {% set block_height = height.columns [0].values() [0] %}
-{% else %}
-    {% set block_height = 0 %}
-{% endif %}
-
 SELECT
     _id AS block_number
 FROM
@@ -18,4 +11,9 @@ FROM
         'number_sequence'
     ) }}
 WHERE
-    _id <= {{ block_height }}
+    _id <= (
+        SELECT
+            MAX(block_number)
+        FROM
+            {{ ref('streamline__chainhead') }}
+    )

models/streamline/streamline__chainhead.sql

@@ -0,0 +1,27 @@
+{{ config (
+    materialized = "view",
+    tags = ['streamline_view']
+) }}
+
+SELECT
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{service}/{Authentication}',
+        OBJECT_CONSTRUCT(
+            'Content-Type',
+            'application/json',
+            'fsc-quantum-state',
+            'livequery'
+        ),
+        OBJECT_CONSTRUCT(
+            'id',
+            0,
+            'jsonrpc',
+            '2.0',
+            'method',
+            'status',
+            'params',
+            []
+        ),
+        'vault/prod/axelar/node/mainnet'
+    ) :data :result :sync_info :latest_block_height :: INT AS block_number

package-lock.yml

@@ -6,11 +6,11 @@ packages:
 - package: dbt-labs/dbt_utils
   version: 1.0.0
 - git: https://github.com/FlipsideCrypto/fsc-utils.git
-  revision: e670a0edf1c1bb2316212b368724dc9942b6220f
+  revision: e94e3d6964f10d8fcca239a64ec47bc8a230261e
 - package: get-select/dbt_snowflake_query_tags
   version: 2.3.3
 - package: calogica/dbt_date
   version: 0.7.2
 - git: https://github.com/FlipsideCrypto/livequery-models.git
-  revision: de2ade547b8e87be0902605d42152ed97f53e05d
-sha1_hash: 23a3f5af6aea92b1391c89c7e8a7024fa822e65f
+  revision: 8b105444ef516f94efd6b9a420e62fe9f69277f9
+sha1_hash: c82fc81890babf0760cd6426e803d0bff21d9264

packages.yml

@@ -6,6 +6,6 @@ packages:
   - package: dbt-labs/dbt_utils
     version: [">=1.0.0", "<1.1.0"]
   - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.21.6-dev
+    revision: v1.22.0
   - package: get-select/dbt_snowflake_query_tags
     version: [">=2.0.0", "<3.0.0"]