Aptos streamline

macros/create_udfs.sql

@@ -1,2 +1,19 @@
 {% macro create_udfs() %}
-{% endmacro %}
+    {% if var("UPDATE_UDFS_AND_SPS") %}
+        {% set sql %}
+        CREATE schema if NOT EXISTS silver;
+
+        {% endset %}
+        {% do run_query(sql) %}
+        {% if target.database != "APTOS_COMMUNITY_DEV" %}
+            {% set sql %}
+            {{ create_udf_bulk_json_rpc() }}
+            {{ create_udf_bulk_rest_api() }}
+
+
+            {% endset %}
+            {% do run_query(sql) %}
+        {% endif %}
+        {{- fsc_utils.create_udfs() -}}
+    {% endif %}
+{% endmacro %}

macros/streamline/api_integrations.sql

@@ -0,0 +1,21 @@
+{% macro create_aws_aptos_api() %}
+    {{ log(
+        "Creating integration for target:" ~ target
+    ) }}
+
+    {% if target.name == "prod" %}
+        {% set sql %}
+        CREATE api integration IF NOT EXISTS aws_aptos_api api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/aptos-api-prod-rolesnowflakeudfsAF733095-wEyotLQyFEIl' api_allowed_prefixes = (
+            'https://dedvhh9fi1.execute-api.us-east-1.amazonaws.com/prod/'
+        ) enabled = TRUE;
+{% endset %}
+        {% do run_query(sql) %}
+        {% elif target.name == "dev" %}
+        {% set sql %}
+        CREATE api integration IF NOT EXISTS aws_aptos_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/aptos-api-dev-rolesnowflakeudfsAF733095-sLREQ0qf4XVH' api_allowed_prefixes = (
+            'https://jx15f84555.execute-api.us-east-1.amazonaws.com/dev/'
+        ) enabled = TRUE;
+{% endset %}
+        {% do run_query(sql) %}
+    {% endif %}
+{% endmacro %}

macros/streamline/grant_streamline_permissions.sql

@@ -0,0 +1,77 @@
+{% macro grant_streamline_permissions(project, warehouse_name, integration_name, dev_integration_name, lambda_role, dbt_cloud_role, internal_dev_role) %}
+
+{% set db_name = project %}
+{% set dev_db_name = project + 'dev' %}
+{# {% set warehouse_name = 'dbt_cloud' %}
+{% set integration_name = 'aws_' + project + '_api' %}
+{% set dev_integration_name = 'aws_' + project + '_api_dev' %}
+{% set lambda_role = 'aws_lambda_' + project + '_api' %}
+{% set dbt_cloud_role = 'dbt_cloud_' + project %}
+{% set internal_dev_role = 'internal_dev' %} #}
+
+grant role {{ lambda_role }} to user {{ lambda_role }};
+grant role {{ internal_dev_role }} to role {{ lambda_role }};
+grant usage on warehouse {{ warehouse_name }} to role {{ lambda_role }};
+grant role {{ dbt_cloud_role }} to role ACCOUNTADMIN;
+grant role {{ internal_dev_role }} to role {{ dbt_cloud_role }};
+grant usage on integration {{ integration_name }} to role {{ dbt_cloud_role }};
+grant usage on integration {{ integration_name }} to role {{ lambda_role }};
+grant usage on integration {{ integration_name }} to role {{ internal_dev_role }};
+grant create integration on account to role {{ dbt_cloud_role }};
+grant usage on integration {{ dev_integration_name }} to role {{ dbt_cloud_role }};
+grant usage on integration {{ dev_integration_name }} to role {{ lambda_role }};
+grant usage on integration {{ dev_integration_name }} to role {{ internal_dev_role }};
+grant usage on warehouse {{ warehouse_name }} to role {{ dbt_cloud_role }};
+grant create database on account to role {{ dbt_cloud_role }};
+grant manage grants on account to role {{ dbt_cloud_role }};
+grant execute task on account to role {{ dbt_cloud_role }};
+
+--External Table Permissions
+grant role {{ dbt_cloud_role }} to user {{ dbt_cloud_role }};
+grant usage on stage streamline.bronze.external_tables to role {{ dbt_cloud_role }};
+grant usage on stage streamline.bronze.external_tables to role {{ lambda_role }};
+
+--Internal_Dev Prod Permissions
+grant usage on all schemas in database {{ db_name }} to role {{ internal_dev_role }};
+grant usage on future schemas in database {{ db_name }} to role {{ internal_dev_role }};
+grant usage on all functions in database {{ db_name }} to role {{ internal_dev_role }};
+grant usage on future functions in database {{ db_name }} to role {{ internal_dev_role }};
+grant select on all tables in database {{ db_name }} to role {{ internal_dev_role }};
+grant select on future tables in database {{ db_name }} to role {{ internal_dev_role }};
+grant select on all views in database {{ db_name }} to role {{ internal_dev_role }};
+grant select on future views in database {{ db_name }} to role {{ internal_dev_role }};
+
+-- The following will grant permissions to the specified roles for both `project` and `project_dev` databases:
+{% for suffix in ['', '_dev'] %}
+    grant usage on database {{ db_name + suffix }} to role {{ lambda_role }};
+    grant usage on database {{ db_name + suffix }} to role {{ dbt_cloud_role }};
+    grant usage on database {{ db_name + suffix }} to role {{ internal_dev_role }};
+
+    grant select on all views in schema {{ db_name + suffix }}.streamline to role {{ lambda_role }};
+    grant select on all tables in schema {{ db_name + suffix }}.streamline to role {{ lambda_role }};
+    grant usage on all functions in schema {{ db_name + suffix }}.streamline to role {{ lambda_role }};
+    grant select on future views in schema {{ db_name + suffix }}.streamline to role {{ lambda_role }};
+    grant select on future tables in schema {{ db_name + suffix }}.streamline to role {{ lambda_role }};
+    grant usage on future functions in schema {{ db_name + suffix }}.streamline to role {{ lambda_role }};
+
+    grant usage on schema streamline.{{ db_name + suffix }} to role {{ lambda_role }};
+    grant select on all tables in schema streamline.{{ db_name + suffix }} to role {{ lambda_role }};
+    grant select on all views in schema streamline.{{ db_name + suffix }} to role {{ lambda_role }};
+    grant select on future tables in schema streamline.{{ db_name + suffix }} to role {{ lambda_role }};
+    grant select on future views in schema streamline.{{ db_name + suffix }} to role {{ lambda_role }};
+
+    grant usage on schema streamline.{{ db_name + suffix }} to role {{ dbt_cloud_role }};
+    grant select on all tables in schema streamline.{{ db_name + suffix }} to role {{ dbt_cloud_role }};
+    grant select on all views in schema streamline.{{ db_name + suffix }} to role {{ dbt_cloud_role }};
+    grant select on future tables in schema streamline.{{ db_name + suffix }} to role {{ dbt_cloud_role }};
+    grant select on future views in schema streamline.{{ db_name + suffix }} to role {{ dbt_cloud_role }};
+
+    grant usage on schema streamline.{{ db_name + suffix }} to role {{ internal_dev_role }};
+    grant select on all tables in schema streamline.{{ db_name + suffix }} to role {{ internal_dev_role }};
+    grant select on all views in schema streamline.{{ db_name + suffix }} to role {{ internal_dev_role }};
+    grant select on future tables in schema streamline.{{ db_name + suffix }} to role {{ internal_dev_role }};
+    grant select on future views in schema streamline.{{ db_name + suffix }} to role {{ internal_dev_role }};
+
+{% endfor %}
+
+{% endmacro %}

macros/streamline/streamline_udfs.sql

@@ -0,0 +1,25 @@
+
+{% macro create_udf_bulk_json_rpc() %}
+    CREATE
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_json_rpc(
+        json variant
+    ) returns text api_integration = {% if target.name == "prod" %}
+        aws_aptos_api AS 'https://dedvhh9fi1.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_json_rpc'
+    {% else %}
+        aws_aptos_api_dev AS 'https://jx15f84555.execute-api.us-east-1.amazonaws.com/dev/udf_bulk_json_rpc'
+    {%- endif %};
+{% endmacro %}
+
+{% macro create_udf_bulk_rest_api() %}
+    {% if target.name == "prod" %}
+        CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_rest_api(
+            json OBJECT
+        ) returns ARRAY api_integration = aws_aptos_api AS
+            'https://dedvhh9fi1.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api'
+    {% else %}
+        CREATE OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_rest_api(
+            json OBJECT
+        ) returns ARRAY api_integration = aws_aptos_api_dev AS
+            'https://jx15f84555.execute-api.us-east-1.amazonaws.com/dev/udf_bulk_rest_api'
+    {%- endif %};
+{% endmacro %}

macros/utils.sql

@@ -0,0 +1,78 @@
+{% macro if_data_call_function(
+        func,
+        target
+    ) %}
+    {% if var(
+            "STREAMLINE_INVOKE_STREAMS"
+        ) %}
+        {% if execute %}
+            {{ log(
+                "Running macro `if_data_call_function`: Calling udf " ~ func ~ " on " ~ target,
+                True
+            ) }}
+        {% endif %}
+    SELECT
+        {{ func }}
+    WHERE
+        EXISTS(
+            SELECT
+                1
+            FROM
+                {{ target }}
+            LIMIT
+                1
+        )
+    {% else %}
+        {% if execute %}
+            {{ log(
+                "Running macro `if_data_call_function`: NOOP",
+                False
+            ) }}
+        {% endif %}
+    SELECT
+        NULL
+    {% endif %}
+{% endmacro %}
+
+{% macro if_data_call_wait() %}
+    {% if var(
+            "STREAMLINE_INVOKE_STREAMS"
+        ) %}
+        {% set query %}
+    SELECT
+        1
+    WHERE
+        EXISTS(
+            SELECT
+                1
+            FROM
+                {{ model.schema ~ "." ~ model.alias }}
+            LIMIT
+                1
+        ) {% endset %}
+        {% if execute %}
+            {% set results = run_query(
+                query
+            ) %}
+            {% if results %}
+                {{ log(
+                    "Waiting...",
+                    info = True
+                ) }}
+
+                {% set wait_query %}
+            SELECT
+                system$wait(
+                    {{ var(
+                        "WAIT",
+                        600
+                    ) }}
+                ) {% endset %}
+                {% do run_query(wait_query) %}
+            {% else %}
+            SELECT
+                NULL;
+            {% endif %}
+        {% endif %}
+    {% endif %}
+{% endmacro %}

models/silver/silver__temp.sql

@@ -0,0 +1,7 @@
+{{ config (
+    materialized = "view",
+    tags = ['streamline_view']
+) }}
+
+SELECT
+    1 AS temp

models/streamline/core/realtime/streamline__blocks_tx_realtime.sql

@@ -0,0 +1,63 @@
+{{ config (
+    materialized = "view",
+    post_hook = if_data_call_function(
+        func = "{{this.schema}}.udf_bulk_rest_api(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'blocks_tx', 'sql_limit', {{var('sql_limit','300000')}}, 'producer_batch_size', {{var('producer_batch_size','300000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}))",
+        target = "{{this.schema}}.{{this.identifier}}"
+    ),
+    tags = ['streamline_core_realtime']
+) }}
+
+WITH gen AS (
+
+    SELECT
+        ROW_NUMBER() over (
+            ORDER BY
+                SEQ4()
+        ) AS block_height
+    FROM
+        TABLE(GENERATOR(rowcount => 106157220))
+),
+blocks AS (
+    SELECT
+        block_height
+    FROM
+        gen
+    ORDER BY
+        1 DESC
+),
+calls AS (
+    SELECT
+        'https://twilight-silent-gas.aptos-mainnet.quiknode.pro/f64d711fb5881ce64cf18a31f796885050178031/v1/blocks/by_height/' || block_height || '?with_transactions=true' calls,
+        block_height
+    FROM
+        (
+            SELECT
+                block_height
+            FROM
+                blocks
+            {# EXCEPT
+            SELECT
+                block_height
+            FROM
+                aptos.bronze.lq_blocks_txs A
+            ORDER BY
+                1 DESC
+            LIMIT
+                75 #}
+        )
+)
+SELECT
+    ARRAY_CONSTRUCT(
+        block_height,
+        ARRAY_CONSTRUCT(
+            'GET',
+            calls,
+            PARSE_JSON('{}'),
+            PARSE_JSON('{}'),
+            ''
+        )
+    ) AS request
+FROM
+    calls
+ORDER BY
+    block_height DESC