From eeed833eefb6a23da79289d36edfc20d3bc060a0 Mon Sep 17 00:00:00 2001
From: desmond-hui <97470747+desmond-hui@users.noreply.github.com>
Date: Wed, 24 Aug 2022 14:31:40 -0700
Subject: [PATCH] An 1871/switch cmc ohlc job to tasks (#20)

* udf endpoint condition on database name

* schedule cmc ohlc job with sf task

* also refresh asset table in case of new/deprecated coins
---
 ...df_bulk_fill_cmc_historical_price_gaps.sql |  2 +-
 ...oin_gecko_asset_market_data_historical.sql |  2 +-
 ...udf_bulk_get_coin_gecko_asset_metadata.sql |  2 +-
 .../task_get_coin_gecko_ohlc.sql              |  1 +
 .../udf_bulk_get_coin_gecko_ohlc.sql          |  2 +-
 .../udf_bulk_get_coin_gecko_prices.sql        |  2 +-
 ...ulk_get_coin_market_cap_asset_metadata.sql |  2 +-
 .../task_get_coin_market_cap_hourly_ohlc.sql  | 28 +++++++++++++++++++
 .../udf_get_coin_market_cap_hourly_ohlc.sql   |  2 +-
 .../udf_bulk_get_coin_market_cap_prices.sql   |  2 +-
 10 files changed, 37 insertions(+), 8 deletions(-)
 create mode 100644 macros/streamline/get_coin_market_cap_hourly_ohlc/task_get_coin_market_cap_hourly_ohlc.sql

diff --git a/macros/streamline/fill_cmc_historical_price_gaps/udf_bulk_fill_cmc_historical_price_gaps.sql b/macros/streamline/fill_cmc_historical_price_gaps/udf_bulk_fill_cmc_historical_price_gaps.sql
index 5edb035..65a92b2 100644
--- a/macros/streamline/fill_cmc_historical_price_gaps/udf_bulk_fill_cmc_historical_price_gaps.sql
+++ b/macros/streamline/fill_cmc_historical_price_gaps/udf_bulk_fill_cmc_historical_price_gaps.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_fill_cmc_historical_price_gaps() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_fill_cmc_historical_price_gaps() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_fill_cmc_historical_price_gaps() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_fill_coin_market_cap_historical_price_gaps'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_fill_coin_market_cap_historical_price_gaps'
diff --git a/macros/streamline/get_coin_gecko_asset_market_data_historical/udf_bulk_get_coin_gecko_asset_market_data_historical.sql b/macros/streamline/get_coin_gecko_asset_market_data_historical/udf_bulk_get_coin_gecko_asset_market_data_historical.sql
index 502b219..b07c6a0 100644
--- a/macros/streamline/get_coin_gecko_asset_market_data_historical/udf_bulk_get_coin_gecko_asset_market_data_historical.sql
+++ b/macros/streamline/get_coin_gecko_asset_market_data_historical/udf_bulk_get_coin_gecko_asset_market_data_historical.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_asset_market_data_historical() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_asset_market_data_historical() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_asset_market_data_historical() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_asset_market_data_historical'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_asset_market_data_historical'
diff --git a/macros/streamline/get_coin_gecko_asset_metadata/udf_bulk_get_coin_gecko_asset_metadata.sql b/macros/streamline/get_coin_gecko_asset_metadata/udf_bulk_get_coin_gecko_asset_metadata.sql
index 514a65f..cf24ab0 100644
--- a/macros/streamline/get_coin_gecko_asset_metadata/udf_bulk_get_coin_gecko_asset_metadata.sql
+++ b/macros/streamline/get_coin_gecko_asset_metadata/udf_bulk_get_coin_gecko_asset_metadata.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_asset_metadata() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_asset_metadata'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_asset_metadata'
diff --git a/macros/streamline/get_coin_gecko_ohlc/task_get_coin_gecko_ohlc.sql b/macros/streamline/get_coin_gecko_ohlc/task_get_coin_gecko_ohlc.sql
index efc44c0..9ede057 100644
--- a/macros/streamline/get_coin_gecko_ohlc/task_get_coin_gecko_ohlc.sql
+++ b/macros/streamline/get_coin_gecko_ohlc/task_get_coin_gecko_ohlc.sql
@@ -6,6 +6,7 @@ execute immediate 'create or replace task streamline.bulk_get_coin_gecko_ohlc
     schedule = \'USING CRON 15 0 * * * UTC\'
 as
 BEGIN
+    call streamline.refresh_external_table_by_recent_date(\'asset_metadata_api\');
     call streamline.refresh_external_table_by_recent_date(\'asset_ohlc_coin_gecko_api\');
 
     select streamline.udf_bulk_get_coin_gecko_ohlc()
diff --git a/macros/streamline/get_coin_gecko_ohlc/udf_bulk_get_coin_gecko_ohlc.sql b/macros/streamline/get_coin_gecko_ohlc/udf_bulk_get_coin_gecko_ohlc.sql
index f0bc1b6..873dcdd 100644
--- a/macros/streamline/get_coin_gecko_ohlc/udf_bulk_get_coin_gecko_ohlc.sql
+++ b/macros/streamline/get_coin_gecko_ohlc/udf_bulk_get_coin_gecko_ohlc.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_ohlc() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_ohlc'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_ohlc'
diff --git a/macros/streamline/get_coin_gecko_prices/udf_bulk_get_coin_gecko_prices.sql b/macros/streamline/get_coin_gecko_prices/udf_bulk_get_coin_gecko_prices.sql
index edfaa72..7e49291 100644
--- a/macros/streamline/get_coin_gecko_prices/udf_bulk_get_coin_gecko_prices.sql
+++ b/macros/streamline/get_coin_gecko_prices/udf_bulk_get_coin_gecko_prices.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_prices() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_prices'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_prices'
diff --git a/macros/streamline/get_coin_market_cap_asset_metadata/udf_bulk_get_coin_market_cap_asset_metadata.sql b/macros/streamline/get_coin_market_cap_asset_metadata/udf_bulk_get_coin_market_cap_asset_metadata.sql
index 29b04d4..0ba066b 100644
--- a/macros/streamline/get_coin_market_cap_asset_metadata/udf_bulk_get_coin_market_cap_asset_metadata.sql
+++ b/macros/streamline/get_coin_market_cap_asset_metadata/udf_bulk_get_coin_market_cap_asset_metadata.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_market_cap_asset_metadata() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_asset_metadata'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_asset_metadata'
diff --git a/macros/streamline/get_coin_market_cap_hourly_ohlc/task_get_coin_market_cap_hourly_ohlc.sql b/macros/streamline/get_coin_market_cap_hourly_ohlc/task_get_coin_market_cap_hourly_ohlc.sql
new file mode 100644
index 0000000..d0badc3
--- /dev/null
+++ b/macros/streamline/get_coin_market_cap_hourly_ohlc/task_get_coin_market_cap_hourly_ohlc.sql
@@ -0,0 +1,28 @@
+{% macro task_bulk_get_coin_market_cap_hourly_ohlc() %}
+{% set sql %}
+execute immediate 'create or replace task streamline.bulk_get_coin_market_cap_hourly_ohlc
+    warehouse = dbt_cloud_crosschain
+    allow_overlapping_execution = false
+    schedule = \'USING CRON 15,30,45 * * * * UTC\'
+as
+BEGIN
+    call streamline.refresh_external_table_by_recent_date(\'asset_metadata_api\');
+    call streamline.refresh_external_table_by_recent_date(\'asset_ohlc_coin_market_cap_api\');
+
+    select streamline.udf_bulk_get_coin_market_cap_hourly_ohlc()
+    where exists (
+        select 1
+        from streamline.all_unknown_coin_market_cap_asset_ohlc_hourly
+        limit 1
+    );
+END;'
+{% endset %}
+{% do run_query(sql) %}
+
+{% if target.database == 'CROSSCHAIN' %}
+    {% set sql %}
+        alter task streamline.bulk_get_coin_market_cap_hourly_ohlc resume;
+    {% endset %}
+    {% do run_query(sql) %}
+{% endif %}
+{% endmacro %}
\ No newline at end of file
diff --git a/macros/streamline/get_coin_market_cap_hourly_ohlc/udf_get_coin_market_cap_hourly_ohlc.sql b/macros/streamline/get_coin_market_cap_hourly_ohlc/udf_get_coin_market_cap_hourly_ohlc.sql
index dadc1d3..a481b5d 100644
--- a/macros/streamline/get_coin_market_cap_hourly_ohlc/udf_get_coin_market_cap_hourly_ohlc.sql
+++ b/macros/streamline/get_coin_market_cap_hourly_ohlc/udf_get_coin_market_cap_hourly_ohlc.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_market_cap_hourly_ohlc() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_market_cap_hourly_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_market_cap_hourly_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_hourly_ohlc'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_hourly_ohlc'
diff --git a/macros/streamline/get_coin_market_cap_prices/udf_bulk_get_coin_market_cap_prices.sql b/macros/streamline/get_coin_market_cap_prices/udf_bulk_get_coin_market_cap_prices.sql
index 512be98..2bfb887 100644
--- a/macros/streamline/get_coin_market_cap_prices/udf_bulk_get_coin_market_cap_prices.sql
+++ b/macros/streamline/get_coin_market_cap_prices/udf_bulk_get_coin_market_cap_prices.sql
@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_market_cap_prices() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_prices'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_prices'