diff --git a/macros/sp_create_copy_coin_gecko_price_data_to_legacy.sql b/macros/sp_create_copy_coin_gecko_price_data_to_legacy.sql index 378b5d8..51d7a70 100644 --- a/macros/sp_create_copy_coin_gecko_price_data_to_legacy.sql +++ b/macros/sp_create_copy_coin_gecko_price_data_to_legacy.sql @@ -29,7 +29,7 @@ $$ d.value[4]::float as price, md.name, md.symbol - from crosschain.bronze.asset_ohlc_coin_gecko_api a + from streamline.{{ target.database }}.asset_ohlc_coin_gecko_api a left outer join base_metadata md on md.asset_id = a.id left join table(flatten(data)) d where recorded_at >= current_date - 1 diff --git a/macros/streamline/sp_refresh_external_table_by_recent_date.sql b/macros/streamline/sp_refresh_external_table_by_recent_date.sql index 56c728c..c823d79 100644 --- a/macros/streamline/sp_refresh_external_table_by_recent_date.sql +++ b/macros/streamline/sp_refresh_external_table_by_recent_date.sql @@ -20,7 +20,7 @@ $$ for row_variable in c1 do path := row_variable.path; end for; - refresh_stmt := 'alter external table bronze.' || :external_table_name || ' refresh \'' || :PATH || '\''; + refresh_stmt := 'alter external table streamline.{{ target.database }}.' || :external_table_name || ' refresh \'' || :PATH || '\''; res := (execute immediate :refresh_stmt); res := (execute immediate :select_stmt2); @@ -28,7 +28,7 @@ $$ for row_variable in c2 do path := row_variable.path; end for; - refresh_stmt2 := 'alter external table bronze.' || :external_table_name || ' refresh \'' || :PATH || '\''; + refresh_stmt2 := 'alter external table streamline.{{ target.database }}.' || :external_table_name || ' refresh \'' || :PATH || '\''; res := (execute immediate :refresh_stmt2); return 'table refreshed with ' || :refresh_stmt || ' and ' || :refresh_stmt2; end; diff --git a/models/bronze/bronze__asset_metadata_coin_gecko.sql b/models/bronze/bronze__asset_metadata_coin_gecko.sql index da5b284..fe395df 100644 --- a/models/bronze/bronze__asset_metadata_coin_gecko.sql +++ b/models/bronze/bronze__asset_metadata_coin_gecko.sql @@ -7,6 +7,6 @@ SELECT TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_gecko_api' ) }} \ No newline at end of file diff --git a/models/bronze/bronze__asset_metadata_coin_market_cap.sql b/models/bronze/bronze__asset_metadata_coin_market_cap.sql index 9694742..63b94eb 100644 --- a/models/bronze/bronze__asset_metadata_coin_market_cap.sql +++ b/models/bronze/bronze__asset_metadata_coin_market_cap.sql @@ -7,6 +7,6 @@ SELECT TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_market_cap_api' ) }} \ No newline at end of file diff --git a/models/bronze/bronze__hourly_prices_coin_gecko.sql b/models/bronze/bronze__hourly_prices_coin_gecko.sql index 8219ca6..8f2c39b 100644 --- a/models/bronze/bronze__hourly_prices_coin_gecko.sql +++ b/models/bronze/bronze__hourly_prices_coin_gecko.sql @@ -7,6 +7,6 @@ SELECT TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_ohlc_coin_gecko_api' ) }} diff --git a/models/bronze/bronze__hourly_prices_coin_market_cap.sql b/models/bronze/bronze__hourly_prices_coin_market_cap.sql index 149d01f..d2563cf 100644 --- a/models/bronze/bronze__hourly_prices_coin_market_cap.sql +++ b/models/bronze/bronze__hourly_prices_coin_market_cap.sql @@ -7,6 +7,6 @@ SELECT TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_ohlc_coin_market_cap_api' ) }} diff --git a/models/sources.yml b/models/sources.yml index 4ef0dd1..a6ec988 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -99,129 +99,12 @@ sources: tables: - name: swaps - name: liquidity_actions - - name: crosschain_external - schema: bronze - loader: S3 + - name: bronze_streamline + database: streamline + schema: crosschain tables: - name: asset_metadata_coin_market_cap_api - description: "all coins supported by provider" - external: - location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_METADATA_API/coinmarketcap" - file_format: "( type = json, strip_outer_array = TRUE )" - auto_refresh: true - partitions: - - name: provider - data_type: string - expression: split_part(METADATA$FILENAME,'/',3) - - name: _inserted_date - data_type: string - expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10) - columns: - - name: id - data_type: string - description: "" - - name: symbol - data_type: string - description: "" - - name: name - data_type: string - description: "" - name: asset_metadata_coin_gecko_api - description: "all coins supported by provider" - external: - location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_METADATA_API/coingecko" - file_format: "( type = json, strip_outer_array = TRUE )" - auto_refresh: true - partitions: - - name: provider - data_type: string - expression: split_part(METADATA$FILENAME,'/',3) - - name: _inserted_date - data_type: string - expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10) - columns: - - name: id - data_type: string - description: "" - - name: symbol - data_type: string - description: "" - - name: name - data_type: string - description: "" - name: asset_ohlc_coin_market_cap_api - description: "all coins supported by provider" - external: - location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_OHLC_API/coinmarketcap" - file_format: "( type = json, strip_outer_array = TRUE )" - auto_refresh: true - partitions: - - name: _inserted_date - data_type: string - expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10) - columns: - - name: id - data_type: number - description: "" - - name: api_start_time - data_type: number - - name: api_end_time - data_type: number - - name: metadata - data_type: string - description: "" - - name: data - data_type: variant - description: "" - name: asset_historical_hourly_market_data_coin_gecko_api - description: "all coins supported by provider" - external: - location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_HISTORICAL_HOURLY_MARKET_DATA_API/coingecko" - file_format: "( type = json, strip_outer_array = TRUE )" - auto_refresh: true - partitions: - - name: _inserted_date - data_type: string - expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10) - columns: - - name: id - data_type: string - description: "" - - name: currency - data_type: string - - name: metadata - data_type: variant - description: "" - - name: data - data_type: variant - description: "" - - name: error - data_type: string - description: "" - - name: asset_ohlc_coin_gecko_api - description: "all coins supported by provider" - external: - location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_OHLC_API/coingecko" - file_format: "( type = json, strip_outer_array = TRUE )" - auto_refresh: true - partitions: - - name: _inserted_date - data_type: string - expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10) - columns: - - name: id - data_type: string - description: "" - - name: run_time - data_type: timestamp_ntz - - name: metadata - data_type: string - description: "" - - name: data - data_type: variant - description: "" - - name: error - data_type: string - description: "" - - \ No newline at end of file + - name: asset_ohlc_coin_gecko_api \ No newline at end of file diff --git a/models/streamline/streamline__all_unknown_coin_gecko_asset_ohlc.sql b/models/streamline/streamline__all_unknown_coin_gecko_asset_ohlc.sql index 2d24a10..14154f8 100644 --- a/models/streamline/streamline__all_unknown_coin_gecko_asset_ohlc.sql +++ b/models/streamline/streamline__all_unknown_coin_gecko_asset_ohlc.sql @@ -7,7 +7,7 @@ SELECT date_trunc('hour',current_timestamp) AS run_time FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_gecko_api' ) }} WHERE @@ -17,7 +17,7 @@ WHERE MAX(_inserted_date) FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_gecko_api' ) }} WHERE @@ -29,6 +29,6 @@ SELECT run_time FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_ohlc_coin_gecko_api' ) }} diff --git a/models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql b/models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql index 8def05c..f471e9f 100644 --- a/models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql +++ b/models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql @@ -31,7 +31,7 @@ cmc_active_assets AS ( VALUE :first_historical_data :: timestamp_ntz AS genesis_recorded_time FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_market_cap_api' ) }} WHERE @@ -42,7 +42,7 @@ cmc_active_assets AS ( MAX(_inserted_date) FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_market_cap_api' ) }} WHERE @@ -69,7 +69,7 @@ base AS ( id FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_ohlc_coin_market_cap_api' ) }} WHERE diff --git a/models/streamline/streamline__coin_gecko_historical_asset_market_data_hourly.sql b/models/streamline/streamline__coin_gecko_historical_asset_market_data_hourly.sql index 6bac331..2ebef29 100644 --- a/models/streamline/streamline__coin_gecko_historical_asset_market_data_hourly.sql +++ b/models/streamline/streamline__coin_gecko_historical_asset_market_data_hourly.sql @@ -7,7 +7,7 @@ SELECT 90 AS days FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_gecko_api' ) }} WHERE @@ -17,7 +17,7 @@ WHERE MAX(_inserted_date) FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_gecko_api' ) }} WHERE @@ -29,6 +29,6 @@ SELECT 90 FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_historical_hourly_market_data_coin_gecko_api' ) }} diff --git a/models/streamline/streamline__coin_market_cap_historical_asset_ohlc_hourly.sql b/models/streamline/streamline__coin_market_cap_historical_asset_ohlc_hourly.sql index 0dedd1d..69b7ee5 100644 --- a/models/streamline/streamline__coin_market_cap_historical_asset_ohlc_hourly.sql +++ b/models/streamline/streamline__coin_market_cap_historical_asset_ohlc_hourly.sql @@ -22,7 +22,7 @@ WITH base AS ( ) AS historical_load_end_time FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_market_cap_api' ) }} WHERE @@ -32,7 +32,7 @@ WITH base AS ( MAX(_inserted_date) FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_metadata_coin_market_cap_api' ) }} WHERE @@ -52,7 +52,7 @@ WITH base AS ( id FROM {{ source( - 'crosschain_external', + 'bronze_streamline', 'asset_ohlc_coin_market_cap_api' ) }} WHERE