From 1fb33c6620cfc87655c50f8593bd47e92b60ac99 Mon Sep 17 00:00:00 2001
From: Jack Forgash <58153492+forgxyz@users.noreply.github.com>
Date: Wed, 29 Jan 2025 16:31:02 -0700
Subject: [PATCH] 2.0 stg config. blocks realtime test

---
 dbt_project.yml                               | 18 +++++++++
 macros/create_udfs.sql                        | 37 +++++++++++--------
 .../realtime/streamline__blocks_realtime.sql  | 36 ++++++++++++++++++
 package-lock.yml                              | 26 ++++++-------
 4 files changed, 88 insertions(+), 29 deletions(-)
 create mode 100644 models/streamline/core/realtime/streamline__blocks_realtime.sql

diff --git a/dbt_project.yml b/dbt_project.yml
index ee40b1c..38ed094 100644
--- a/dbt_project.yml
+++ b/dbt_project.yml
@@ -47,6 +47,10 @@ models:
   +on_schema_change: "append_new_columns"
   near_models:
     +pre-hook: '{{ fsc_utils.set_query_tag() }}'
+  livequery_models:
+    deploy:
+      core:
+        materialized: ephemeral
 
 tests:
   near:
@@ -73,6 +77,19 @@ vars:
     'gold/core',
     'streamline'
   ]
+  API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}' 
+  EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}'
+  API_AWS_ROLE_ARN: '{{ var("config")[target.name]["API_AWS_ROLE_ARN"] if var("config")[target.name] else var("config")["dev"]["API_AWS_ROLE_ARN"] }}'
+  ROLES: '{{ var("config")[target.name]["ROLES"] }}'
+  config:
+  # The keys correspond to dbt profiles and are case sensitive
+    dev:
+        API_INTEGRATION: AWS_NEAR_API_STG_V2
+        EXTERNAL_FUNCTION_URI: cx7cyhtcjf.execute-api.us-east-1.amazonaws.com/stg/
+        API_AWS_ROLE_ARN: arn:aws:lambda:us-east-1:704693948482:function:near-api-stg-APIHandler-JIPzugohypYl
+        ROLES:
+          - AWS_LAMBDA_NEAR_API
+          - INTERNAL_DEV
 
 dispatch:
   - macro_namespace: dbt
@@ -84,3 +101,4 @@ dispatch:
 query-comment:
   comment: "{{ dbt_snowflake_query_tags.get_query_comment(node) }}"
   append: true # Snowflake removes prefixed comments.
+
diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql
index 0dd40bb..ba76b6d 100644
--- a/macros/create_udfs.sql
+++ b/macros/create_udfs.sql
@@ -1,20 +1,25 @@
 {% macro create_udfs() %}
     {% if var("UPDATE_UDFS_AND_SPS") %}
-        {% if target.database != "NEAR_COMMUNITY_DEV" %}
-            {% set sql %}
-                CREATE schema if NOT EXISTS silver;
-                CREATE schema if NOT EXISTS streamline;
-                {{ create_udf_introspect() }}
-                {{ create_udf_s3_list_directories() }}
-                {{ create_udf_s3_list_objects() }}
-                {{ create_udf_s3_copy_objects() }}
-                {{ create_udf_s3_copy_objects_overwrite() }}
-                {{ create_UDTF_CALL_CONTRACT_FUNCTION() }}
-                {{ create_UDTF_CALL_CONTRACT_FUNCTION_BY_HEIGHT() }}
-                {{ create_UDF_GET_CHAINHEAD() }}
-            {% endset %}
-            {% do run_query(sql) %}
-        {{- fsc_utils.create_udfs() -}}
-        {% endif %}
+
+        {% set sql %}
+            CREATE schema if NOT EXISTS silver;
+            CREATE schema if NOT EXISTS streamline;
+            {{ create_udf_introspect() }}
+            {{ create_udf_s3_list_directories() }}
+            {{ create_udf_s3_list_objects() }}
+            {{ create_udf_s3_copy_objects() }}
+            {{ create_udf_s3_copy_objects_overwrite() }}
+            {{ create_UDTF_CALL_CONTRACT_FUNCTION() }}
+            {{ create_UDTF_CALL_CONTRACT_FUNCTION_BY_HEIGHT() }}
+            {{ create_UDF_GET_CHAINHEAD() }}
+        {% endset %}
+
+        {% do run_query(sql) %}
+
+        {% set utils_sql %}
+            {{ fsc_utils.create_udfs() }}
+        {% endset %}
+        {% do run_query(utils_sql) %}
+
     {% endif %}
 {% endmacro %}
diff --git a/models/streamline/core/realtime/streamline__blocks_realtime.sql b/models/streamline/core/realtime/streamline__blocks_realtime.sql
new file mode 100644
index 0000000..c24aefd
--- /dev/null
+++ b/models/streamline/core/realtime/streamline__blocks_realtime.sql
@@ -0,0 +1,36 @@
+{{ config (
+    materialized = "view",
+    post_hook = fsc_utils.if_data_call_function_v2(
+        func = '{{this.schema}}.udf_bulk_rest_api_v2',
+        target = "{{this.schema}}.{{this.identifier}}",
+        params = {
+            "external_table": "blocks",
+            "sql_limit": "100",
+            "producer_batch_size": "100",
+            "worker_batch_size": "100",
+            "sql_source": "{{this.identifier}}"
+        }
+    )
+) }}
+
+-- single block for testing
+SELECT
+    138515000 AS block_number,
+    DATE_PART('EPOCH', SYSDATE()) :: INTEGER AS partition_key,
+    {{ target.database }}.live.udf_api(
+        'POST',
+        '{Service}',
+        OBJECT_CONSTRUCT(
+            'Content-Type', 'application/json'
+        ),
+        OBJECT_CONSTRUCT(
+            'jsonrpc', '2.0',
+            'method', 'block',
+            'id', 'dontcare',
+            'params', OBJECT_CONSTRUCT(
+                'block_id',  block_number
+            )
+        ),
+        'Vault/prod/near/quicknode/mainneet'
+    ) AS request
+
diff --git a/package-lock.yml b/package-lock.yml
index dfa2464..d2370f6 100644
--- a/package-lock.yml
+++ b/package-lock.yml
@@ -1,14 +1,14 @@
 packages:
-  - package: calogica/dbt_expectations
-    version: 0.8.2
-  - package: dbt-labs/dbt_utils
-    version: 1.0.0
-  - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: d3cf679e079f0cf06142de9386f215e55fe26b3b
-  - package: get-select/dbt_snowflake_query_tags
-    version: 2.5.0
-  - package: calogica/dbt_date
-    version: 0.7.2
-  - git: https://github.com/FlipsideCrypto/livequery-models.git
-    revision: b024188be4e9c6bc00ed77797ebdc92d351d620e
-sha1_hash: afea4db1f3b20c6d83e5854437824d1b589981f2
+- package: calogica/dbt_expectations
+  version: 0.8.2
+- package: dbt-labs/dbt_utils
+  version: 1.0.0
+- git: https://github.com/FlipsideCrypto/fsc-utils.git
+  revision: 6096046f00c5c27fa7b8d60ae6ec1dfe31d7fa10
+- package: get-select/dbt_snowflake_query_tags
+  version: 2.5.0
+- package: calogica/dbt_date
+  version: 0.7.2
+- git: https://github.com/FlipsideCrypto/livequery-base.git
+  revision: 006980d40fbbc48c2ad0f0b74efa37ec6fa1c74b
+sha1_hash: 197fe41257817dfd90f292d52002c6efb6c09305