revert models; update sql limit; prod endpoint; tags

dbt_project.yml

@@ -88,7 +88,7 @@ vars:
         - INTERNAL_DEV
     prod:
       API_INTEGRATION: aws_aptos_api_prod_v2
-      EXTERNAL_FUNCTION_URI: 9v6g64rv1e.execute-api.us-east-1.amazonaws.com/stg/
+      EXTERNAL_FUNCTION_URI: mxus3semvi.execute-api.us-east-1.amazonaws.com/prod/
       ROLES:
         - AWS_LAMBDA_APTOS_API
         - DBT_CLOUD_APTOS

macros/streamline/models.sql

@@ -76,19 +76,21 @@ WHERE
         FROM
             TABLE(
                 information_schema.external_table_file_registration_history(
-                    start_time => DATEADD('day', -7, CURRENT_TIMESTAMP()),
+                    start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()),
                     table_name => '{{ source( "bronze_streamline", model) }}')
                 ) A
             )
         SELECT
+            {{ unique_key }},
+            DATA,
+            _inserted_timestamp,
             MD5(
                 CAST(
                     COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text
                 )
             ) AS id,
-            s.*,
-            b.file_name,
-            _inserted_timestamp
+            s.{{ partition_name }},
+            s.value AS VALUE
         FROM
             {{ source(
                 "bronze_streamline",
@@ -113,9 +115,7 @@ WHERE
                     '-32007',
                     '-32008',
                     '-32009',
-                    '-32010',
-                    '-32602',
-                    '-32603'
+                    '-32010'
                 )
             )
 {% endmacro %}
@@ -139,14 +139,16 @@ WHERE
             ) A
     )
 SELECT
+    {{ unique_key }},
+    DATA,
+    _inserted_timestamp,
     MD5(
         CAST(
             COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text
         )
     ) AS id,
-    s.*,
-    b.file_name,
-    _inserted_timestamp
+    s.{{ partition_name }},
+    s.value AS VALUE
 FROM
     {{ source(
         "bronze_streamline",
@@ -171,9 +173,7 @@ WHERE
             '-32007',
             '-32008',
             '-32009',
-            '-32010',
-            '-32602',
-            '-32603'
+            '-32010'
         )
     )
 {% endmacro %}

macros/streamline/streamline_udfs.sql

@@ -29,7 +29,7 @@
     OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_rest_api_v2(
         json OBJECT
     ) returns ARRAY api_integration = {% if target.name == "prod" %}
-        aws_aptos_api_prod_v2 AS ''
+        aws_aptos_api_prod_v2 AS 'https://mxus3semvi.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api'
     {% else %}
         aws_aptos_api_stg_v2 AS 'https://9v6g64rv1e.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api'
     {%- endif %};

models/bronze/core/bronze__streamline_FR_blocks_tx.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 SELECT

models/bronze/core/bronze__streamline_FR_transaction_batch.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 SELECT

models/streamline/core/realtime/streamline__blocks_tx_realtime.sql

@@ -4,7 +4,7 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"blocks_tx_v2",
-        "sql_limit" :"10",
+        "sql_limit" :"1200000",
         "producer_batch_size" :"300000",
         "worker_batch_size" :"50000",
         "sql_source" :"{{this.identifier}}",