From da2da83e70b27873a112dae61b35837a2cb6eb5c Mon Sep 17 00:00:00 2001
From: tarikceric <tarik@flipsidecrypto.com>
Date: Wed, 20 Nov 2024 12:13:16 -0800
Subject: [PATCH] revert models; update sql limit; prod endpoint; tags

---
 dbt_project.yml                               |  2 +-
 macros/streamline/models.sql                  | 26 +++++++++----------
 macros/streamline/streamline_udfs.sql         |  2 +-
 .../core/bronze__streamline_FR_blocks_tx.sql  |  3 ++-
 ...ronze__streamline_FR_transaction_batch.sql |  3 ++-
 .../streamline__blocks_tx_realtime.sql        |  2 +-
 6 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/dbt_project.yml b/dbt_project.yml
index 9c38396..400bb41 100644
--- a/dbt_project.yml
+++ b/dbt_project.yml
@@ -88,7 +88,7 @@ vars:
         - INTERNAL_DEV
     prod:
       API_INTEGRATION: aws_aptos_api_prod_v2
-      EXTERNAL_FUNCTION_URI: 9v6g64rv1e.execute-api.us-east-1.amazonaws.com/stg/
+      EXTERNAL_FUNCTION_URI: mxus3semvi.execute-api.us-east-1.amazonaws.com/prod/
       ROLES:
         - AWS_LAMBDA_APTOS_API
         - DBT_CLOUD_APTOS
diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql
index 6a672a0..265f945 100644
--- a/macros/streamline/models.sql
+++ b/macros/streamline/models.sql
@@ -76,19 +76,21 @@ WHERE
         FROM
             TABLE(
                 information_schema.external_table_file_registration_history(
-                    start_time => DATEADD('day', -7, CURRENT_TIMESTAMP()),
+                    start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()),
                     table_name => '{{ source( "bronze_streamline", model) }}')
                 ) A
             )
         SELECT
+            {{ unique_key }},
+            DATA,
+            _inserted_timestamp,
             MD5(
                 CAST(
                     COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text
                 )
             ) AS id,
-            s.*,
-            b.file_name,
-            _inserted_timestamp
+            s.{{ partition_name }},
+            s.value AS VALUE
         FROM
             {{ source(
                 "bronze_streamline",
@@ -113,9 +115,7 @@ WHERE
                     '-32007',
                     '-32008',
                     '-32009',
-                    '-32010',
-                    '-32602',
-                    '-32603'
+                    '-32010'
                 )
             )
 {% endmacro %}
@@ -139,14 +139,16 @@ WHERE
             ) A
     )
 SELECT
+    {{ unique_key }},
+    DATA,
+    _inserted_timestamp,
     MD5(
         CAST(
             COALESCE(CAST({{ unique_key }} AS text), '' :: STRING) AS text
         )
     ) AS id,
-    s.*,
-    b.file_name,
-    _inserted_timestamp
+    s.{{ partition_name }},
+    s.value AS VALUE
 FROM
     {{ source(
         "bronze_streamline",
@@ -171,9 +173,7 @@ WHERE
             '-32007',
             '-32008',
             '-32009',
-            '-32010',
-            '-32602',
-            '-32603'
+            '-32010'
         )
     )
 {% endmacro %}
diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql
index cfb475a..a4ade3e 100644
--- a/macros/streamline/streamline_udfs.sql
+++ b/macros/streamline/streamline_udfs.sql
@@ -29,7 +29,7 @@
     OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_rest_api_v2(
         json OBJECT
     ) returns ARRAY api_integration = {% if target.name == "prod" %}
-        aws_aptos_api_prod_v2 AS ''
+        aws_aptos_api_prod_v2 AS 'https://mxus3semvi.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api'
     {% else %}
         aws_aptos_api_stg_v2 AS 'https://9v6g64rv1e.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api'
     {%- endif %};
diff --git a/models/bronze/core/bronze__streamline_FR_blocks_tx.sql b/models/bronze/core/bronze__streamline_FR_blocks_tx.sql
index eb518b1..b404bd8 100644
--- a/models/bronze/core/bronze__streamline_FR_blocks_tx.sql
+++ b/models/bronze/core/bronze__streamline_FR_blocks_tx.sql
@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 SELECT
diff --git a/models/bronze/core/bronze__streamline_FR_transaction_batch.sql b/models/bronze/core/bronze__streamline_FR_transaction_batch.sql
index bb088c4..37920b4 100644
--- a/models/bronze/core/bronze__streamline_FR_transaction_batch.sql
+++ b/models/bronze/core/bronze__streamline_FR_transaction_batch.sql
@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['core']
 ) }}
 
 SELECT
diff --git a/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql b/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql
index 22a6780..a4a5fd8 100644
--- a/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql
+++ b/models/streamline/core/realtime/streamline__blocks_tx_realtime.sql
@@ -4,7 +4,7 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"blocks_tx_v2",
-        "sql_limit" :"10",
+        "sql_limit" :"1200000",
         "producer_batch_size" :"300000",
         "worker_batch_size" :"50000",
         "sql_source" :"{{this.identifier}}",