flip/terra-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/terra-models.git synced 2026-02-06 13:51:46 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

added terra_udf_bulk_json_rpc analyses

Browse Source
This commit is contained in:
shah 2023-05-05 08:21:08 -07:00
parent bf336d7b39
commit 14e471fbac
4 changed files with 105 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
analyses/terra_udf_bulk_json_rpc.sql Normal file
View File

@ -0,0 +1,23 @@
{{if_data_call_function(
func = "streamline.udf_bulk_json_rpc_sbx_shah(object_construct('sql_source', 'view_name', 'external_table', 'qn_getBlockWithReceipts', 'sql_limit', 4000, 'producer_batch_size', 10000s, 'worker_batch_size', 1000, 'batch_call_limit', 10))",
target = "streamline.view_name"
)
}}
SELECT
streamline.udf_bulk_json_rpc(
object_construct(
'sql_source',
'view_name',
'external_table',
'qn_getBlockWithReceipts',
'sql_limit',
4000,
'producer_batch_size',
10000,
'worker_batch_size',
1000,
'batch_call_limit',
10
)
)

6
dbt_project.yml
View File

@ -51,5 +51,7 @@ tests:
vars:
"dbt_date:time_zone": GMT
"UPDATE_SNOWFLAKE_TAGS": TRUE
STREAMLINE_INVOKE_STREAMS: True
STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
UPDATE_UDFS_AND_SPS: False
UPDATE_SNOWFLAKE_TAGS: True

78
macros/utils.sql Normal file
View File

@ -0,0 +1,78 @@
{% macro if_data_call_function(
func,
target
) %}
{% if var(
"STREAMLINE_INVOKE_STREAMS"
) %}
{% if execute %}
{{ log(
"Running macro `if_data_call_function`: Calling udf " ~ func ~ " on " ~ target,
True
) }}
{% endif %}
SELECT
{{ func }}
WHERE
EXISTS(
SELECT
1
FROM
{{ target }}
LIMIT
1
)
{% else %}
{% if execute %}
{{ log(
"Running macro `if_data_call_function`: NOOP",
False
) }}
{% endif %}
SELECT
NULL
{% endif %}
{% endmacro %}
{% macro if_data_call_wait() %}
{% if var(
"STREAMLINE_INVOKE_STREAMS"
) %}
{% set query %}
SELECT
1
WHERE
EXISTS(
SELECT
1
FROM
{{ model.schema ~ "." ~ model.alias }}
LIMIT
1
) {% endset %}
{% if execute %}
{% set results = run_query(
query
) %}
{% if results %}
{{ log(
"Waiting...",
info = True
) }}
{% set wait_query %}
SELECT
system$wait(
{{ var(
"WAIT",
600
) }}
) {% endset %}
{% do run_query(wait_query) %}
{% else %}
SELECT
NULL;
{% endif %}
{% endif %}
{% endif %}
{% endmacro %}

74
models/silver/streamline/core/realtime/streamline__pc_getBlock_realtime.sql
View File

@ -1,74 +0,0 @@
{{ config (
materialized = "view",
post_hook = if_data_call_function(
func = "{{this.schema}}.udf_bulk_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'pc_getBlock', 'sql_limit', {{var('sql_limit','40000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','10')}}))",
target = "{{this.schema}}.{{this.identifier}}"
)
) }}
-- WITH last_3_days AS (
-- SELECT
-- block_number
-- FROM
-- {{ ref("_max_block_by_date") }}
-- qualify ROW_NUMBER() over (
-- ORDER BY
-- block_number DESC
-- ) = 3
-- ),
-- blocks AS (
-- SELECT
-- block_number :: STRING AS block_number
-- FROM
-- {{ ref("streamline__blocks") }}
-- WHERE
-- (
-- block_number >= (
-- SELECT
-- block_number
-- FROM
-- last_3_days
-- )
-- )
-- -- EXCEPT
-- -- SELECT
-- -- block_number :: STRING
-- -- FROM
-- -- {{ ref("streamline__complete_qn_getBlockWithReceipts") }}
-- -- WHERE
-- -- (
-- -- block_number >= (
-- -- SELECT
-- -- block_number
-- -- FROM
-- -- last_3_days
-- -- )
-- -- )
-- )
-- SELECT
-- PARSE_JSON(
-- CONCAT(
-- '{"jsonrpc": "2.0",',
-- '"method": "block", "params":["',
-- REPLACE(
-- concat_ws(
-- '',
-- '0x',
-- to_char(
-- block_number :: INTEGER,
-- 'XXXXXXXX'
-- )
-- ),
-- ' ',
-- ''
-- ),
-- '"],"id":"',
-- block_number :: STRING,
-- '"}'
-- )
-- ) AS request
-- FROM
-- blocks
-- ORDER BY
-- block_number ASC