An 2461/use latest streamline endpoints (#81)

* use streamline db location for source data * use new streamline endpoints * remove limits, now controlled from streamline at runtime
2026-02-06 11:26:55 +00:00 · 2022-12-12 13:49:43 -08:00 · 2022-12-12 13:49:43 -08:00 · a9c76a7046
commit a9c76a7046
parent e1145d9200
11 changed files with 18 additions and 170 deletions
--- a/macros/streamline/bulk_get_asset_metadata/udf_bulk_get_asset_metadata.sql
+++ b/macros/streamline/bulk_get_asset_metadata/udf_bulk_get_asset_metadata.sql
@ -1,8 +1,8 @@
 {% macro udf_bulk_get_asset_metadata() %}
    CREATE
    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_asset_metadata() returns text api_integration = aws_osmosis_api_dev AS {% if target.database == 'OSMOSIS' %}
-        'https://k7jc1bnb8i.execute-api.us-east-1.amazonaws.com/prod/get_asset_metadata'
+        'https://r9o2ijlhfc.execute-api.us-east-1.amazonaws.com/prod/get_asset_metadata'
    {% else %}
-        'https://auacbjh2tj.execute-api.us-east-1.amazonaws.com/dev/get_asset_metadata'
+        'https://tirqv0b7h7.execute-api.us-east-1.amazonaws.com/dev/get_asset_metadata'
    {%- endif %}
 {% endmacro %}
--- a/macros/streamline/bulk_get_balances/udf_bulk_get_balances.sql
+++ b/macros/streamline/bulk_get_balances/udf_bulk_get_balances.sql
@ -1,8 +1,8 @@
 {% macro udf_bulk_get_balances() %}
    CREATE
    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_balances() returns text api_integration = aws_osmosis_api_dev AS {% if target.database == 'OSMOSIS' %}
-        'https://k7jc1bnb8i.execute-api.us-east-1.amazonaws.com/prod/bulk_get_balances'
+        'https://r9o2ijlhfc.execute-api.us-east-1.amazonaws.com/prod/bulk_get_balances'
    {% else %}
-        'https://auacbjh2tj.execute-api.us-east-1.amazonaws.com/dev/bulk_get_balances'
+        'https://tirqv0b7h7.execute-api.us-east-1.amazonaws.com/dev/bulk_get_balances'
    {%- endif %}
 {% endmacro %}
--- a/macros/streamline/bulk_get_pool_balances/udf_bulk_get_pool_balances.sql
+++ b/macros/streamline/bulk_get_pool_balances/udf_bulk_get_pool_balances.sql
@ -1,8 +1,8 @@
 {% macro udf_bulk_get_pool_balances() %}
    CREATE
    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_pool_balances() returns text api_integration = aws_osmosis_api_dev AS {% if target.database == 'OSMOSIS' %}
-        'https://k7jc1bnb8i.execute-api.us-east-1.amazonaws.com/prod/bulk_get_pool_balances'
+        'https://r9o2ijlhfc.execute-api.us-east-1.amazonaws.com/prod/bulk_get_pool_balances'
    {% else %}
-        'https://auacbjh2tj.execute-api.us-east-1.amazonaws.com/dev/bulk_get_pool_balances'
+        'https://tirqv0b7h7.execute-api.us-east-1.amazonaws.com/dev/bulk_get_pool_balances'
    {%- endif %}
 {% endmacro %}
--- a/macros/streamline/bulk_get_validator_metadata/udf_bulk_get_validator_metadata.sql
+++ b/macros/streamline/bulk_get_validator_metadata/udf_bulk_get_validator_metadata.sql
@ -1,8 +1,8 @@
 {% macro udf_bulk_get_validator_metadata() %}
    CREATE
    OR REPLACE EXTERNAL FUNCTION silver.udf_bulk_get_validator_metadata() returns text api_integration = aws_osmosis_api_dev AS {% if target.database == 'OSMOSIS' %}
-        'https://k7jc1bnb8i.execute-api.us-east-1.amazonaws.com/prod/get_validator_metadata'
+        'https://r9o2ijlhfc.execute-api.us-east-1.amazonaws.com/prod/get_validator_metadata'
    {% else %}
-        'https://auacbjh2tj.execute-api.us-east-1.amazonaws.com/dev/get_validator_metadata'
+        'https://tirqv0b7h7.execute-api.us-east-1.amazonaws.com/dev/get_validator_metadata'
    {%- endif %}
 {% endmacro %}
--- a/models/silver/silver__all_undecoded_assets.sql
+++ b/models/silver/silver__all_undecoded_assets.sql
@ -28,6 +28,6 @@ SELECT
  base AS address
 FROM
  {{ source(
-    'osmosis_external',
+    'bronze_streamline',
    'asset_metadata_api'
  ) }}
--- a/models/silver/silver__all_unknown_balances.sql
+++ b/models/silver/silver__all_unknown_balances.sql
@ -141,10 +141,8 @@ SELECT
    address
 FROM
    {{ source(
-        'osmosis_external',
+        'bronze_streamline',
        'balances_api'
    ) }}
 ORDER BY
    block_id
-LIMIT
-    25000
--- a/models/silver/silver__all_unknown_pool_balances.sql
+++ b/models/silver/silver__all_unknown_pool_balances.sql
@ -28,8 +28,6 @@ SELECT
    block_id
 FROM
    {{ source(
-        'osmosis_external',
+        'bronze_streamline',
        'pool_balances_api'
    ) }}
-LIMIT
-    1000
--- a/models/silver/silver__asset_metadata.sql
+++ b/models/silver/silver__asset_metadata.sql
@ -11,7 +11,7 @@ WITH base AS (
    denom_units AS raw_metadata
  FROM
    {{ source(
-      'osmosis_external',
+      'bronze_streamline',
      'asset_metadata_api'
    ) }}
  GROUP BY
--- a/models/silver/silver__liquid_balances.sql
+++ b/models/silver/silver__liquid_balances.sql
@ -20,7 +20,7 @@ WITH base AS (
        ) AS _inserted_timestamp
    FROM
        {{ source(
-            'osmosis_external',
+            'bronze_streamline',
            'balances_api'
        ) }}
        bal
--- a/models/silver/silver__validator_metadata.sql
+++ b/models/silver/silver__validator_metadata.sql
@ -27,6 +27,6 @@ SELECT
    ) AS _unique_key
 FROM
  {{ source(
-    'osmosis_external',
+    'bronze_streamline',
    'validator_metadata_api'
  ) }}
--- a/models/sources.yml
+++ b/models/sources.yml
@ -21,159 +21,11 @@ sources:
    schema: bronze_api
    tables: 
      - name: blockchain
-  - name: osmosis_external
-    schema: bronze
-    loader: S3
+  - name: bronze_streamline
+    database: streamline
+    schema: osmosis
    tables:
      - name: validator_metadata_api
-        description: "External table of osmosis validator metadata files"
-        external:
-          location: "@osmosis.bronze.analytics_external_tables/{{target.database}}/VALIDATOR_METADATA_API"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-        columns:
-          - name: rank
-            data_type: number
-            description: ""
-            expression: value:rank::number
-          - name: account_address
-            data_type: string
-            description: ""
-            expression: value:account_address::string
-          - name: operator_address
-            data_type: string
-            description: ""
-            expression: value:operator_address::string
-          - name: consensus_pubkey
-            data_type: string
-            description: ""
-            expression: value:consensus_pubkey::string
-          - name: jailed
-            data_type: boolean
-            description: ""
-            expression: value:jailed::boolean
-          - name: status
-            data_type: number
-            description: ""
-            expression: value:status::number
-          - name: tokens
-            data_type: number
-            description: ""
-            expression: value:tokens::number
-          - name: delegator_shares
-            data_type: float
-            description: ""
-            expression: value:delegator_shares::float
-          - name: moniker
-            data_type: string
-            description: ""
-            expression: value:moniker::string
-          - name: identity
-            data_type: string
-            description: ""
-            expression: value:identity::string
-          - name: website
-            data_type: string
-            description: ""
-            expression: value:website::string
-          - name: details
-            data_type: string
-            description: ""
-            expression: value:details::string
-          - name: unbonding_height
-            data_type: number
-            description: ""
-            expression: value:unbonding_height::number
-          - name: unbonding_time
-            data_type: timestamp_ntz
-            description: ""
-            expression: value:unbonding_time::string::timestamp_ntz
-          - name: rate
-            data_type: float
-            description: ""
-            expression: value:rate::float
-          - name: max_rate
-            data_type: float
-            description: ""
-            expression: value:max_rate::float
-          - name: max_change_rate
-            data_type: float
-            description: ""
-            expression: value:max_change_rate::float
-          - name: uptime
-            data_type: variant
-            description: ""
-            expression: value:update::variant
-            # (1-(value:uptime:missed::number/value:uptime:over_blocks::number))
-          - name: min_self_delegation
-            data_type: number
-            description: ""
-            expression: value:min_self_delegation::number
      - name: asset_metadata_api
-        description: "External table of osmosis asset metadata files"
-        external:
-          location: "@osmosis.bronze.analytics_external_tables/{{target.database}}/ASSET_METADATA_API"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-        columns:
-          - name: base
-            data_type: string
-            description: ""
-            expression: value:base::string
-          - name: coingecko_id
-            data_type: string
-            description: ""
-            expression: value:coingecko_id::string
-          - name: denom_units
-            data_type: array
-            description: ""
-            expression: value:denom_units::array
-          - name: description
-            data_type: string
-            description: ""
-            expression: "value:description::string"
-          - name: ibc
-            data_type: variant
-            description: ""
-            expression: value:ibc::variant
-          - name: name
-            data_type: string
-            description: ""
-            expression: value:name::string
-          - name: symbol
-            data_type: string
-            description: ""
-            expression: value:symbol::string
      - name: balances_api
-        description: "Wallet balances from osmosis api"
-        external:
-          location: "@osmosis.bronze.analytics_external_tables/OSMOSIS/BALANCES_API"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-          partitions:
-            - name: _inserted_date
-              data_type: string
-              expression: substr((split_part(METADATA$FILENAME,'/',3)),16,10)
-        columns:
-          - name: block_id
-            data_type: number
-            description: ""
-          - name: address
-            data_type: string
-            description: ""
-          - name: balances
-            data_type: array
-            description: ""
-      - name: pool_balances_api
-        description: "pool balances from osmosis api"
-        external:
-          location: "@osmosis.bronze.analytics_external_tables/{{target.database}}/POOL_BALANCES_API"
-          file_format: "( type = json, strip_outer_array = TRUE )"
-          auto_refresh: true
-        columns:
-          - name: block_id
-            data_type: number
-            description: ""
-          - name: pools
-            data_type: variant
-            description: ""
+      - name: pool_balances_api