change model references to new streamline sources location (#44)

2026-02-06 13:26:52 +00:00 · 2022-09-20 11:48:59 -07:00 · 2022-09-20 11:48:59 -07:00 · e6fe9cd707
commit e6fe9cd707
parent a5a62f1290
11 changed files with 23 additions and 140 deletions
--- a/macros/sp_create_copy_coin_gecko_price_data_to_legacy.sql
+++ b/macros/sp_create_copy_coin_gecko_price_data_to_legacy.sql
@ -29,7 +29,7 @@ $$
                        d.value[4]::float as price,
                        md.name, 
                        md.symbol
-                    from crosschain.bronze.asset_ohlc_coin_gecko_api a
+                    from streamline.{{ target.database }}.asset_ohlc_coin_gecko_api a
                    left outer join base_metadata md on md.asset_id = a.id
                    left join table(flatten(data)) d
                    where recorded_at >= current_date - 1
--- a/macros/streamline/sp_refresh_external_table_by_recent_date.sql
+++ b/macros/streamline/sp_refresh_external_table_by_recent_date.sql
@ -20,7 +20,7 @@ $$
        for row_variable in c1 do
            path := row_variable.path;
        end for;
-        refresh_stmt := 'alter external table bronze.' || :external_table_name || ' refresh \'' || :PATH || '\'';
+        refresh_stmt := 'alter external table streamline.{{ target.database }}.' || :external_table_name || ' refresh \'' || :PATH || '\'';
        res := (execute immediate :refresh_stmt);

        res := (execute immediate :select_stmt2);
@ -28,7 +28,7 @@ $$
        for row_variable in c2 do
            path := row_variable.path;
        end for;
-        refresh_stmt2 := 'alter external table bronze.' || :external_table_name || ' refresh \'' || :PATH || '\'';
+        refresh_stmt2 := 'alter external table streamline.{{ target.database }}.' || :external_table_name || ' refresh \'' || :PATH || '\'';
        res := (execute immediate :refresh_stmt2);
        return 'table refreshed with ' || :refresh_stmt || ' and ' || :refresh_stmt2;
    end;
--- a/models/bronze/bronze__asset_metadata_coin_gecko.sql
+++ b/models/bronze/bronze__asset_metadata_coin_gecko.sql
@ -7,6 +7,6 @@ SELECT
    TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_metadata_coin_gecko_api'
    ) }}
--- a/models/bronze/bronze__asset_metadata_coin_market_cap.sql
+++ b/models/bronze/bronze__asset_metadata_coin_market_cap.sql
@ -7,6 +7,6 @@ SELECT
    TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_metadata_coin_market_cap_api'
    ) }}
--- a/models/bronze/bronze__hourly_prices_coin_gecko.sql
+++ b/models/bronze/bronze__hourly_prices_coin_gecko.sql
@ -7,6 +7,6 @@ SELECT
    TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_ohlc_coin_gecko_api'
    ) }}
--- a/models/bronze/bronze__hourly_prices_coin_market_cap.sql
+++ b/models/bronze/bronze__hourly_prices_coin_market_cap.sql
@ -7,6 +7,6 @@ SELECT
    TO_TIMESTAMP_NTZ(SUBSTR(SPLIT_PART(metadata$filename, '/', 5), 1, 10) :: NUMBER, 0) AS _inserted_timestamp
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_ohlc_coin_market_cap_api'
    ) }}
--- a/models/sources.yml
+++ b/models/sources.yml
@ -99,129 +99,12 @@ sources:
    tables:
      - name: swaps
      - name: liquidity_actions
-  - name: crosschain_external
-    schema: bronze
-    loader: S3
+  - name: bronze_streamline
+    database: streamline
+    schema: crosschain
    tables:
      - name: asset_metadata_coin_market_cap_api
-        description: "all coins supported by provider"
-        external:
-          location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_METADATA_API/coinmarketcap"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-          partitions:
-            - name: provider
-              data_type: string 
-              expression: split_part(METADATA$FILENAME,'/',3)
-            - name: _inserted_date
-              data_type: string
-              expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10)
-        columns:
-          - name: id
-            data_type: string
-            description: ""
-          - name: symbol
-            data_type: string
-            description: ""
-          - name: name
-            data_type: string
-            description: ""
      - name: asset_metadata_coin_gecko_api
-        description: "all coins supported by provider"
-        external:
-          location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_METADATA_API/coingecko"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-          partitions:
-            - name: provider
-              data_type: string 
-              expression: split_part(METADATA$FILENAME,'/',3)
-            - name: _inserted_date
-              data_type: string
-              expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10)
-        columns:
-          - name: id
-            data_type: string
-            description: ""
-          - name: symbol
-            data_type: string
-            description: ""
-          - name: name
-            data_type: string
-            description: ""
      - name: asset_ohlc_coin_market_cap_api
-        description: "all coins supported by provider"
-        external:
-          location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_OHLC_API/coinmarketcap"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-          partitions:
-            - name: _inserted_date
-              data_type: string
-              expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10)
-        columns:
-          - name: id
-            data_type: number
-            description: ""
-          - name: api_start_time
-            data_type: number
-          - name: api_end_time
-            data_type: number
-          - name: metadata
-            data_type: string
-            description: ""
-          - name: data
-            data_type: variant
-            description: ""
      - name: asset_historical_hourly_market_data_coin_gecko_api
-        description: "all coins supported by provider"
-        external:
-          location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_HISTORICAL_HOURLY_MARKET_DATA_API/coingecko"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-          partitions:
-            - name: _inserted_date
-              data_type: string
-              expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10)
-        columns:
-          - name: id
-            data_type: string
-            description: ""
-          - name: currency
-            data_type: string
-          - name: metadata
-            data_type: variant
-            description: ""
-          - name: data
-            data_type: variant
-            description: ""
-          - name: error
-            data_type: string
-            description: ""
-      - name: asset_ohlc_coin_gecko_api
-        description: "all coins supported by provider"
-        external:
-          location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_OHLC_API/coingecko"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-          partitions:
-            - name: _inserted_date
-              data_type: string
-              expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10)
-        columns:
-          - name: id
-            data_type: string
-            description: ""
-          - name: run_time
-            data_type: timestamp_ntz
-          - name: metadata
-            data_type: string
-            description: ""
-          - name: data
-            data_type: variant
-            description: ""
-          - name: error
-            data_type: string
-            description: ""
-      
-  
+      - name: asset_ohlc_coin_gecko_api
--- a/models/streamline/streamline__all_unknown_coin_gecko_asset_ohlc.sql
+++ b/models/streamline/streamline__all_unknown_coin_gecko_asset_ohlc.sql
@ -7,7 +7,7 @@ SELECT
    date_trunc('hour',current_timestamp) AS run_time
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_metadata_coin_gecko_api'
    ) }}
 WHERE
@ -17,7 +17,7 @@ WHERE
            MAX(_inserted_date)
        FROM
            {{ source(
-                'crosschain_external',
+                'bronze_streamline',
                'asset_metadata_coin_gecko_api'
            ) }}
        WHERE
@ -29,6 +29,6 @@ SELECT
    run_time
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_ohlc_coin_gecko_api'
    ) }}
--- a/models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql
+++ b/models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql
@ -31,7 +31,7 @@ cmc_active_assets AS (
        VALUE :first_historical_data :: timestamp_ntz AS genesis_recorded_time
    FROM
        {{ source(
-            'crosschain_external',
+            'bronze_streamline',
            'asset_metadata_coin_market_cap_api'
        ) }}
    WHERE
@ -42,7 +42,7 @@ cmc_active_assets AS (
                MAX(_inserted_date)
            FROM
                {{ source(
-                    'crosschain_external',
+                    'bronze_streamline',
                    'asset_metadata_coin_market_cap_api'
                ) }}
            WHERE
@ -69,7 +69,7 @@ base AS (
        id
    FROM
        {{ source(
-            'crosschain_external',
+            'bronze_streamline',
            'asset_ohlc_coin_market_cap_api'
        ) }}
    WHERE
--- a/models/streamline/streamline__coin_gecko_historical_asset_market_data_hourly.sql
+++ b/models/streamline/streamline__coin_gecko_historical_asset_market_data_hourly.sql
@ -7,7 +7,7 @@ SELECT
    90 AS days
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_metadata_coin_gecko_api'
    ) }}
 WHERE
@ -17,7 +17,7 @@ WHERE
            MAX(_inserted_date)
        FROM
            {{ source(
-                'crosschain_external',
+                'bronze_streamline',
                'asset_metadata_coin_gecko_api'
            ) }}
        WHERE
@ -29,6 +29,6 @@ SELECT
    90
 FROM
    {{ source(
-        'crosschain_external',
+        'bronze_streamline',
        'asset_historical_hourly_market_data_coin_gecko_api'
    ) }}
--- a/models/streamline/streamline__coin_market_cap_historical_asset_ohlc_hourly.sql
+++ b/models/streamline/streamline__coin_market_cap_historical_asset_ohlc_hourly.sql
@ -22,7 +22,7 @@ WITH base AS (
        ) AS historical_load_end_time
    FROM
        {{ source(
-            'crosschain_external',
+            'bronze_streamline',
            'asset_metadata_coin_market_cap_api'
        ) }}
    WHERE
@ -32,7 +32,7 @@ WITH base AS (
                MAX(_inserted_date)
            FROM
                {{ source(
-                    'crosschain_external',
+                    'bronze_streamline',
                    'asset_metadata_coin_market_cap_api'
                ) }}
            WHERE
@ -52,7 +52,7 @@ WITH base AS (
        id
    FROM
        {{ source(
-            'crosschain_external',
+            'bronze_streamline',
            'asset_ohlc_coin_market_cap_api'
        ) }}
    WHERE