An 1871/switch cmc ohlc job to tasks (#20)

* udf endpoint condition on database name

* schedule cmc ohlc job with sf task

* also refresh asset table in case of new/deprecated coins

macros/streamline/fill_cmc_historical_price_gaps/udf_bulk_fill_cmc_historical_price_gaps.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_fill_cmc_historical_price_gaps() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_fill_cmc_historical_price_gaps() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_fill_cmc_historical_price_gaps() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_fill_coin_market_cap_historical_price_gaps'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_fill_coin_market_cap_historical_price_gaps'

macros/streamline/get_coin_gecko_asset_market_data_historical/udf_bulk_get_coin_gecko_asset_market_data_historical.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_asset_market_data_historical() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_asset_market_data_historical() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_asset_market_data_historical() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_asset_market_data_historical'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_asset_market_data_historical'

macros/streamline/get_coin_gecko_asset_metadata/udf_bulk_get_coin_gecko_asset_metadata.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_asset_metadata() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_asset_metadata'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_asset_metadata'

macros/streamline/get_coin_gecko_ohlc/task_get_coin_gecko_ohlc.sql

@@ -6,6 +6,7 @@ execute immediate 'create or replace task streamline.bulk_get_coin_gecko_ohlc
     schedule = \'USING CRON 15 0 * * * UTC\'
 as
 BEGIN
+    call streamline.refresh_external_table_by_recent_date(\'asset_metadata_api\');
     call streamline.refresh_external_table_by_recent_date(\'asset_ohlc_coin_gecko_api\');
 
     select streamline.udf_bulk_get_coin_gecko_ohlc()

macros/streamline/get_coin_gecko_ohlc/udf_bulk_get_coin_gecko_ohlc.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_ohlc() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_gecko_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_ohlc'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_ohlc'

macros/streamline/get_coin_gecko_prices/udf_bulk_get_coin_gecko_prices.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_gecko_prices() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_gecko_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_gecko_prices'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_gecko_prices'

macros/streamline/get_coin_market_cap_asset_metadata/udf_bulk_get_coin_market_cap_asset_metadata.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_market_cap_asset_metadata() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_asset_metadata() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_asset_metadata'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_asset_metadata'

macros/streamline/get_coin_market_cap_hourly_ohlc/task_get_coin_market_cap_hourly_ohlc.sql

@@ -0,0 +1,28 @@
+{% macro task_bulk_get_coin_market_cap_hourly_ohlc() %}
+{% set sql %}
+execute immediate 'create or replace task streamline.bulk_get_coin_market_cap_hourly_ohlc
+    warehouse = dbt_cloud_crosschain
+    allow_overlapping_execution = false
+    schedule = \'USING CRON 15,30,45 * * * * UTC\'
+as
+BEGIN
+    call streamline.refresh_external_table_by_recent_date(\'asset_metadata_api\');
+    call streamline.refresh_external_table_by_recent_date(\'asset_ohlc_coin_market_cap_api\');
+
+    select streamline.udf_bulk_get_coin_market_cap_hourly_ohlc()
+    where exists (
+        select 1
+        from streamline.all_unknown_coin_market_cap_asset_ohlc_hourly
+        limit 1
+    );
+END;'
+{% endset %}
+{% do run_query(sql) %}
+
+{% if target.database == 'CROSSCHAIN' %}
+    {% set sql %}
+        alter task streamline.bulk_get_coin_market_cap_hourly_ohlc resume;
+    {% endset %}
+    {% do run_query(sql) %}
+{% endif %}
+{% endmacro %}

macros/streamline/get_coin_market_cap_hourly_ohlc/udf_get_coin_market_cap_hourly_ohlc.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_market_cap_hourly_ohlc() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_market_cap_hourly_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_market_cap_hourly_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_hourly_ohlc'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_hourly_ohlc'

macros/streamline/get_coin_market_cap_prices/udf_bulk_get_coin_market_cap_prices.sql

@@ -1,6 +1,6 @@
 {% macro udf_bulk_get_coin_market_cap_prices() %}
     CREATE
-    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
+    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_coin_market_cap_prices() returns text api_integration = aws_crosschain_api_dev AS {% if target.database == "CROSSCHAIN" -%}
         'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_prices'
     {% else %}
         'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_prices'