From d8be29f0aec56befe7f84a6ce4c46dd7785b0ac4 Mon Sep 17 00:00:00 2001 From: mattromano Date: Wed, 12 Jun 2024 10:25:03 -0700 Subject: [PATCH] add prices bronze and silver --- dbt_project.yml | 6 ++- ...bronze__complete_native_asset_metadata.sql | 26 +++++++++++ .../prices/bronze__complete_native_prices.sql | 29 ++++++++++++ ...onze__complete_provider_asset_metadata.sql | 26 +++++++++++ .../bronze__complete_provider_prices.sql | 24 ++++++++++ .../bronze__complete_token_asset_metadata.sql | 28 +++++++++++ .../prices/bronze__complete_token_prices.sql | 31 +++++++++++++ ...silver__complete_native_asset_metadata.sql | 37 +++++++++++++++ ...silver__complete_native_asset_metadata.yml | 24 ++++++++++ .../prices/silver__complete_native_prices.sql | 40 ++++++++++++++++ .../prices/silver__complete_native_prices.yml | 37 +++++++++++++++ ...lver__complete_provider_asset_metadata.sql | 37 +++++++++++++++ ...lver__complete_provider_asset_metadata.yml | 26 +++++++++++ .../silver__complete_provider_prices.sql | 45 ++++++++++++++++++ .../silver__complete_provider_prices.yml | 25 ++++++++++ .../silver__complete_token_asset_metadata.sql | 42 +++++++++++++++++ .../silver__complete_token_asset_metadata.yml | 28 +++++++++++ .../prices/silver__complete_token_prices.sql | 46 +++++++++++++++++++ .../prices/silver__complete_token_prices.yml | 41 +++++++++++++++++ models/sources.yml | 10 ++++ 20 files changed, 607 insertions(+), 1 deletion(-) create mode 100644 models/bronze/prices/bronze__complete_native_asset_metadata.sql create mode 100644 models/bronze/prices/bronze__complete_native_prices.sql create mode 100644 models/bronze/prices/bronze__complete_provider_asset_metadata.sql create mode 100644 models/bronze/prices/bronze__complete_provider_prices.sql create mode 100644 models/bronze/prices/bronze__complete_token_asset_metadata.sql create mode 100644 models/bronze/prices/bronze__complete_token_prices.sql create mode 100644 models/silver/prices/silver__complete_native_asset_metadata.sql create mode 100644 models/silver/prices/silver__complete_native_asset_metadata.yml create mode 100644 models/silver/prices/silver__complete_native_prices.sql create mode 100644 models/silver/prices/silver__complete_native_prices.yml create mode 100644 models/silver/prices/silver__complete_provider_asset_metadata.sql create mode 100644 models/silver/prices/silver__complete_provider_asset_metadata.yml create mode 100644 models/silver/prices/silver__complete_provider_prices.sql create mode 100644 models/silver/prices/silver__complete_provider_prices.yml create mode 100644 models/silver/prices/silver__complete_token_asset_metadata.sql create mode 100644 models/silver/prices/silver__complete_token_asset_metadata.yml create mode 100644 models/silver/prices/silver__complete_token_prices.sql create mode 100644 models/silver/prices/silver__complete_token_prices.yml diff --git a/dbt_project.yml b/dbt_project.yml index 45b3c0b..5c789a7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -66,4 +66,8 @@ vars: WAIT: 0 HEAL_MODEL: False HEAL_CURATED_MODEL: [] - START_GHA_TASKS: False \ No newline at end of file + START_GHA_TASKS: False + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] if var("config")[target.name] else var("config")["dev"]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' + ROLES: | + ["INTERNAL_DEV"] \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_native_asset_metadata.sql b/models/bronze/prices/bronze__complete_native_asset_metadata.sql new file mode 100644 index 0000000..68e73ac --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_asset_metadata.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_asset_metadata' + ) }} +WHERE + blockchain = 'klaytn' + AND symbol = 'KLAY' diff --git a/models/bronze/prices/bronze__complete_native_prices.sql b/models/bronze/prices/bronze__complete_native_prices.sql new file mode 100644 index 0000000..3aae3a3 --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_prices.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_prices' + ) }} +WHERE + blockchain = 'klaytn' + AND symbol = 'KLAY' diff --git a/models/bronze/prices/bronze__complete_provider_asset_metadata.sql b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..ed97334 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_asset_metadata' + ) }} +WHERE + platform = 'Klaytn' +-- platforms specific to Blast \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_provider_prices.sql b/models/bronze/prices/bronze__complete_provider_prices.sql new file mode 100644 index 0000000..7d85636 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_prices.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_prices' + ) }} +-- prices for all ids \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_token_asset_metadata.sql b/models/bronze/prices/bronze__complete_token_asset_metadata.sql new file mode 100644 index 0000000..0e3f380 --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_asset_metadata.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_asset_metadata' + ) }} +WHERE + blockchain = 'klaytn' \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_token_prices.sql b/models/bronze/prices/bronze__complete_token_prices.sql new file mode 100644 index 0000000..00245bc --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_prices.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_prices' + ) }} +WHERE + blockchain = 'klaytn' diff --git a/models/silver/prices/silver__complete_native_asset_metadata.sql b/models/silver/prices/silver__complete_native_asset_metadata.sql new file mode 100644 index 0000000..f06d593 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_asset_metadata_id', + tags = ['non_realtime'] +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_asset_metadata.yml b/models/silver/prices/silver__complete_native_asset_metadata.yml new file mode 100644 index 0000000..1c1fdf1 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__complete_native_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - SYMBOL + + columns: + - name: PROVIDER + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_native_prices.sql b/models/silver/prices/silver__complete_native_prices.sql new file mode 100644 index 0000000..375cbed --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.sql @@ -0,0 +1,40 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_prices_id', + tags = ['non_realtime'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_prices' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_prices.yml b/models/silver/prices/silver__complete_native_prices.yml new file mode 100644 index 0000000..bb72433 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__complete_native_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + + columns: + - name: HOUR + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.sql b/models/silver/prices/silver__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..a22da44 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_asset_metadata_id', + tags = ['non_realtime'] +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_provider_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.yml b/models/silver/prices/silver__complete_provider_asset_metadata.yml new file mode 100644 index 0000000..8fa56fd --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__complete_provider_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - TOKEN_ADDRESS + - NAME + - SYMBOL + - PLATFORM + - PLATFORM_ID + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_prices.sql b/models/silver/prices/silver__complete_provider_prices.sql new file mode 100644 index 0000000..c6aa062 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.sql @@ -0,0 +1,45 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_prices_id', + tags = ['non_realtime'] +) }} + +SELECT + p.asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + p.provider, + p.source, + p._inserted_timestamp, + p.inserted_timestamp, + p.modified_timestamp, + p.complete_provider_prices_id, + p._invocation_id +FROM + {{ ref( + 'bronze__complete_provider_prices' + ) }} + p + INNER JOIN {{ ref('bronze__complete_provider_asset_metadata') }} + m + ON p.asset_id = m.asset_id + +{% if is_incremental() %} +WHERE + p.modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider +ORDER BY + p.modified_timestamp DESC)) = 1 diff --git a/models/silver/prices/silver__complete_provider_prices.yml b/models/silver/prices/silver__complete_provider_prices.yml new file mode 100644 index 0000000..8db23c7 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__complete_provider_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - RECORDED_HOUR + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: RECORDED_HOUR + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_asset_metadata.sql b/models/silver/prices/silver__complete_token_asset_metadata.sql new file mode 100644 index 0000000..2dfc8a3 --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_asset_metadata_id', + tags = ['non_realtime'] +) }} + +SELECT + LOWER( + A.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_asset_metadata' + ) }} A + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_asset_metadata.yml b/models/silver/prices/silver__complete_token_asset_metadata.yml new file mode 100644 index 0000000..3eea590 --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__complete_token_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: PROVIDER + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql new file mode 100644 index 0000000..f343d5a --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_prices_id', + tags = ['non_realtime'] +) }} + +SELECT + HOUR, + LOWER( + p.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_prices' + ) }} + p + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_prices.yml b/models/silver/prices/silver__complete_token_prices.yml new file mode 100644 index 0000000..d2ac57f --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: silver__complete_token_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: HOUR + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 7fc562b..f20ef2e 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -17,3 +17,13 @@ sources: schema: github_actions tables: - name: workflows + - name: silver_crosschain + database: "{{ 'crosschain' if target.database == 'KAIA' else 'crosschain_dev' }}" + schema: silver + tables: + - name: complete_token_asset_metadata + - name: complete_token_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_native_asset_metadata + - name: complete_native_prices \ No newline at end of file