diff --git a/.cursor/mcp.json b/.cursor/mcp.json new file mode 100644 index 00000000..2a173b87 --- /dev/null +++ b/.cursor/mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "datapilot": { + "url": "http://localhost:7703/sse", + "type": "sse", + "updatedAt": "2025-06-09T01:36:54.579Z" + } + } +} \ No newline at end of file diff --git a/.cursor/rules/patprefs.mdc b/.cursor/rules/patprefs.mdc new file mode 100644 index 00000000..0c220e55 --- /dev/null +++ b/.cursor/rules/patprefs.mdc @@ -0,0 +1,17 @@ +--- +description: +globs: +alwaysApply: false +--- +# Coding pattern preferences +– Always prefer simple solutions +– Avoid duplication of code whenever possible, which means checking for other areas of the codebase that might already have similar code and functionality +– Write code that takes into account the different environments: dev, test, and prod +– You are careful to only make changes that are requested or you are confident are well understood and related to the change being requested +– When fixing an issue or bug, do not introduce a new pattern or technology without first exhausting all options for the existing implementation. And if you finally do this, make sure to remove the old implementation afterwards so we don’t have duplicate logic. +– Keep the codebase very clean and organized +– Avoid writing scripts in files if possible, especially if the script is likely only to be run once +– Avoid having files over 200–300 lines of code. Refactor at that point. +– Mocking data is only needed for tests, never mock data for dev or prod +– Never add stubbing or fake data patterns to code that affects the dev or prod environments +– Never overwrite my .env file without first asking and confirming \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1ebc2457..0a9db215 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ target/ dbt_modules/ # newer versions of dbt use this directory instead of dbt_modules for test dependencies -dbt_packages/ +dbt_packages/* +!dbt_packages/fsc_evm/ logs/ .venv/ diff --git a/VELO_DELETE.sql b/VELO_DELETE.sql new file mode 100644 index 00000000..e68a8c2d --- /dev/null +++ b/VELO_DELETE.sql @@ -0,0 +1,8 @@ +SELECT +'optimism_dev' as database_name, +'DROP VIEW ' || upper(database_name) || '.' || table_schema || '.' || table_name || ';' as drop_statement +FROM optimism_dev.information_schema.views +WHERE table_schema = 'STREAMLINE' +AND table_name LIKE 'VELODROME%' +AND table_name NOT IN ('VELODROME_SWAPS', 'VELODROME_POOLS') +order by 1 asc; diff --git a/VELO_DELETE_SF.sql b/VELO_DELETE_SF.sql new file mode 100644 index 00000000..b90442ae --- /dev/null +++ b/VELO_DELETE_SF.sql @@ -0,0 +1,18 @@ + select * from OPTIMISM.not_null_silver.exactly_liquidations_COLLATERAL_SYMBOL; + + select + * + from + OPTIMISM.silver.contracts + where + contract_address = '0x6926b434cce9b5b7966ae1bfeef6d0a7dcf3a8bb'; + +select * from OPTIMISM_dev.silver.exactly_asset_details; + +select * from OPTIMISM.silver.traces where tx_hash = '0x441c2c5cb943d43e8685d8f3d43f62b62912126de9f1ed1457609007f7abf6a7'; +select * from OPTIMISM.silver.traces where tx_hash = '0xc1a5ace2b5433bf9bb66f6d2eed00c9f5229ff386f0898500ab59282f9e6e0e1'; +select count(*) +from OPTIMISM.silver.logs +where contract_address = '0x6926b434cce9b5b7966ae1bfeef6d0a7dcf3a8bb'; + +select * from crosschain.silver.function_sig limit 10; \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index cf91c514..9f012cd3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -28,6 +28,7 @@ clean-targets: # directories to be removed by `dbt clean` tests: +store_failures: true # all tests + +where: "__timestamp_filter__" on-run-start: - "{{ create_sps() }}" diff --git a/decode_models.sql b/decode_models.sql new file mode 100644 index 00000000..e69de29b diff --git a/decode_models_sf.sql b/decode_models_sf.sql new file mode 100644 index 00000000..fa1f180b --- /dev/null +++ b/decode_models_sf.sql @@ -0,0 +1,56 @@ +create or replace view streamline.decoded_logs_history_2021_11 as ( + WITH target_blocks AS ( + SELECT + block_number + FROM OPTIMISM_DEV.core.fact_blocks + WHERE date_trunc('month', block_timestamp) = '2021-11-01'::timestamp + ), + new_abis AS ( + SELECT + abi, + parent_contract_address, + event_signature, + start_block, + end_block + FROM OPTIMISM_DEV.silver.complete_event_abis + + ), + existing_logs_to_exclude AS ( + SELECT _log_id + FROM OPTIMISM_DEV.streamline.complete_decode_logs l + INNER JOIN target_blocks b using (block_number) + ), + candidate_logs AS ( + SELECT + l.block_number, + l.tx_hash, + l.event_index, + l.contract_address, + l.topics, + l.data, + concat(l.tx_hash::string, '-', l.event_index::string) as _log_id + FROM target_blocks b + INNER JOIN OPTIMISM_DEV.core.fact_event_logs l using (block_number) + WHERE l.tx_status = 'SUCCESS' and date_trunc('month', l.block_timestamp) = '2021-11-01'::timestamp + ) + SELECT + l.block_number, + l._log_id, + A.abi, + OBJECT_CONSTRUCT( + 'topics', l.topics, + 'data', l.data, + 'address', l.contract_address + ) AS data + FROM candidate_logs l + INNER JOIN new_abis A + ON A.parent_contract_address = l.contract_address + AND A.event_signature = l.topics[0]::STRING + AND l.block_number BETWEEN A.start_block AND A.end_block + WHERE NOT EXISTS ( + SELECT 1 + FROM existing_logs_to_exclude e + WHERE e._log_id = l._log_id + ) + LIMIT 7500000 + ) diff --git a/macros/tests/dynamic_tests.sql b/macros/tests/dynamic_tests.sql new file mode 100644 index 00000000..55db4dbe --- /dev/null +++ b/macros/tests/dynamic_tests.sql @@ -0,0 +1,80 @@ +{% macro get_where_subquery(relation) -%} + {%- set where = config.get('where') -%} + + {%- set interval_vars = namespace( + interval_type = none, + interval_value = none + ) -%} + + {% set intervals = { + 'minutes': var('minutes', none), + 'hours': var('hours', none), + 'days': var('days', none), + 'weeks': var('weeks', none), + 'months': var('months', none), + 'years': var('years', none) + } %} + + {% for type, value in intervals.items() %} + {% if value is not none %} + {% set interval_vars.interval_type = type[:-1] %} + {% set interval_vars.interval_value = value %} + {% break %} + {% endif %} + {% endfor %} + + {% if 'dbt_expectations_expect_column_values_to_be_in_type_list' in this | string %} + {% do return(relation) %} + {% endif %} + + {%- set ts_vars = namespace( + timestamp_column = none, + filter_condition = none + ) -%} + + {% if where and interval_vars.interval_type is not none and interval_vars.interval_value is not none %} + {% if "__timestamp_filter__" in where %} + {% set columns = adapter.get_columns_in_relation(relation) %} + {% set column_names = columns | map(attribute='name') | list %} + + {% for column in columns %} + {% if column.name == 'MODIFIED_TIMESTAMP' %} + {% set ts_vars.timestamp_column = 'MODIFIED_TIMESTAMP' %} + {% break %} + {% endif %} + {% endfor %} + + {% if not ts_vars.timestamp_column %} + {% for column in columns %} + {% if column.name == '_INSERTED_TIMESTAMP' %} + {% set ts_vars.timestamp_column = '_INSERTED_TIMESTAMP' %} + {% break %} + {% endif %} + {% endfor %} + {% endif %} + + {% if not ts_vars.timestamp_column %} + {% for column in columns %} + {% if column.name == 'BLOCK_TIMESTAMP' %} + {% set ts_vars.timestamp_column = 'BLOCK_TIMESTAMP' %} + {% break %} + {% endif %} + {% endfor %} + {% endif %} + + {% if ts_vars.timestamp_column is not none %} + {% set ts_vars.filter_condition = ts_vars.timestamp_column ~ " >= dateadd(" ~ + interval_vars.interval_type ~ ", -" ~ + interval_vars.interval_value ~ ", current_timestamp())" %} + {% set where = where | replace("__timestamp_filter__", ts_vars.filter_condition) %} + {% endif %} + {% endif %} + + {%- set filtered -%} + (select * from {{ relation }} where {{ where }}) dbt_subquery + {%- endset -%} + {% do return(filtered) %} + {%- else -%} + {% do return(relation) %} + {%- endif -%} +{%- endmacro %} \ No newline at end of file diff --git a/macros/utils/context_explorer.sql b/macros/utils/context_explorer.sql new file mode 100644 index 00000000..60627110 --- /dev/null +++ b/macros/utils/context_explorer.sql @@ -0,0 +1,65 @@ +{# -- Save this as macros/utils/explore_context.sql +{% macro explore_context(var_name, detailed=false) %} + +{# This will explore a specific variable or the entire context #} +{% if var_name == 'all' %} + {% do log('=== EXPLORING ALL AVAILABLE CONTEXT VARIABLES ===', info=true) %} + {% for key in context %} + {% do log('VARIABLE: ' ~ key, info=true) %} + {% if detailed %} + {% if (context[key] is mapping) or (context[key] is iterable and context[key] is not string) %} + {% do log(' TYPE: ' ~ context[key].__class__.__name__ if context[key].__class__ is defined else 'Complex type', info=true) %} + {% do log(' STRUCTURE: Cannot display full structure (complex type)', info=true) %} + + {# Try to get some basic info for specific types #} + {% if context[key] is mapping %} + {% do log(' KEYS: ' ~ context[key].keys() | list, info=true) %} + {% endif %} + {% else %} + {% do log(' VALUE: ' ~ context[key], info=true) %} + {% endif %} + {% endif %} + {% endfor %} +{% else %} + {# This will explore a specific variable #} + {% if context[var_name] is defined %} + {% do log('=== EXPLORING VARIABLE: ' ~ var_name ~ ' ===', info=true) %} + + {# Try to determine the type #} + {% do log('TYPE: ' ~ context[var_name].__class__.__name__ if context[var_name].__class__ is defined else 'Unknown type', info=true) %} + + {# If it's a mapping (dictionary-like), show keys #} + {% if context[var_name] is mapping %} + {% do log('KEYS: ' ~ context[var_name].keys() | list, info=true) %} + + {# If detailed is true, try to show values for each key #} + {% if detailed %} + {% for key in context[var_name].keys() %} + {% do log(' ' ~ key ~ ': ' ~ context[var_name][key], info=true) %} + {% endfor %} + {% endif %} + {% endif %} + + {# If it's another iterable but not a string, list items #} + {% if context[var_name] is iterable and context[var_name] is not string and context[var_name] is not mapping %} + {% do log('ITEMS: ' ~ context[var_name] | list, info=true) %} + {% endif %} + + {# If it's a simple value, show it #} + {% if context[var_name] is not mapping and (context[var_name] is not iterable or context[var_name] is string) %} + {% do log('VALUE: ' ~ context[var_name], info=true) %} + {% endif %} + + {# Try to show common attributes for objects #} + {% do log('ATTRIBUTES:', info=true) %} + {% for attr in ['name', 'schema', 'database', 'identifier', 'alias', 'original_file_path', 'package_name', 'path', 'unique_id'] %} + {% if context[var_name][attr] is defined %} + {% do log(' ' ~ attr ~ ': ' ~ context[var_name][attr], info=true) %} + {% endif %} + {% endfor %} + {% else %} + {% do log('Variable "' ~ var_name ~ '" not found in context', info=true) %} + {% endif %} +{% endif %} + +{% endmacro %} #} \ No newline at end of file diff --git a/macros/utils/get_path_tags.sql b/macros/utils/get_path_tags.sql new file mode 100644 index 00000000..6917ff5c --- /dev/null +++ b/macros/utils/get_path_tags.sql @@ -0,0 +1,60 @@ +{%- macro load_tag_mapping() -%} + {% set tag_mapping = get_tag_dictionary() %} + {{ log("Loaded tag mappings: " ~ tag_mapping, info=True) }} + {{ return(tag_mapping) }} +{%- endmacro -%} + +{%- macro get_path_tags(model, additional_tags=[]) -%} + {% set tags = [] %} + + {{ log(model.original_file_path, info=True) }} + + {% set path_str = model.original_file_path | string %} + {% set path = path_str.split('/') %} + + {# Skip 'models' directory if it exists #} + {% set start_index = 1 if path[0] == 'models' else 0 %} + + {# Process each directory in the path #} + {% for part in path[start_index:-1] %} + {% do tags.append(part) %} + {% endfor %} + + {# Process the filename without extension #} + {% set filename = path[-1] | replace('.sql', '') | replace('.yml', '') %} + + {# Split on __ and take the last part as the model name #} + {% set name_parts = filename.split('__') %} + {% if name_parts|length > 1 %} + {# Add the prefix as a tag #} + {% do tags.append(name_parts[0]) %} + {# Add the actual model name #} + {% do tags.append(name_parts[-1]) %} + {% else %} + {% do tags.append(filename) %} + {% endif %} + + {# Add any additional tags provided #} + {% if additional_tags is not none %} + {% do tags.extend(additional_tags) %} + {% endif %} + + {{ log("Initial tags: " ~ tags, info=True) }} + + {# Load tag mapping from YAML #} + {% set tag_mapping = load_tag_mapping() %} + + {# Apply tag mapping rules #} + {% set final_tags = tags.copy() %} + {% for tag in tags %} + {% if tag in tag_mapping %} + {% do final_tags.extend(tag_mapping[tag]) %} + {{ log("Added tags from '" ~ tag ~ "': " ~ tag_mapping[tag], info=True) }} + {% endif %} + {% endfor %} + + {{ log("Final tags: " ~ final_tags | unique | list, info=True) }} + + {# Return unique tags to avoid duplicates #} + {{ return(final_tags | unique | list) }} +{%- endmacro -%} \ No newline at end of file diff --git a/macros/utils/tag_mapping.sql b/macros/utils/tag_mapping.sql new file mode 100644 index 00000000..51c69a48 --- /dev/null +++ b/macros/utils/tag_mapping.sql @@ -0,0 +1,13 @@ +{%- macro get_tag_dictionary() -%} + {% set tag_mapping = { + + 'defi': ['curated', 'reorg'], + 'protocols': ['curated', 'reorg'], + + 'silver': ['raw_data'], + 'gold': ['analytics_ready'], + 'core': ['core_tables'] + } %} + + {{ return(tag_mapping) }} +{%- endmacro -%} \ No newline at end of file diff --git a/models/silver/defi/lending/compound/silver__comp_asset_details.sql b/models/silver/defi/lending/compound/silver__comp_asset_details.sql index 52a38be6..f116b3ce 100644 --- a/models/silver/defi/lending/compound/silver__comp_asset_details.sql +++ b/models/silver/defi/lending/compound/silver__comp_asset_details.sql @@ -1,114 +1,96 @@ - {{ config( - materialized = 'incremental', - unique_key = "compound_market_address", - tags = ['silver','defi','lending','curated','asset_details'] - ) }} +{{ config( + materialized = 'incremental', + unique_key = "compound_market_address", + tags = ['silver','defi','lending','curated','asset_details'] +) }} - WITH contracts_dim AS ( - SELECT - address, - name, - symbol, - decimals - FROM - {{ ref('core__dim_contracts') }} - ), - - comp_v3_base AS ( - SELECT - contract_address, - block_number, - live.udf_api( - 'POST', - '{URL}', - OBJECT_CONSTRUCT( - 'Content-Type', 'application/json', - 'fsc-quantum-state', 'livequery' - ), - utils.udf_json_rpc_call( - 'eth_call', - [ - { - 'to': contract_address, - 'from': null, - 'data': RPAD('0xc55dae63', 64, '0') - }, - utils.udf_int_to_hex(block_number) - ], - concat_ws('-', contract_address, '0xc55dae63', block_number) - ), - 'Vault/prod/evm/quicknode/optimism/mainnet' - ) AS api_response - FROM - {{ ref('core__fact_event_logs') }} - WHERE - topic_0 = '0xbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b' - AND origin_from_address IN ( - LOWER('0x6103DB328d4864dc16BD2F0eE1B9A92e3F87f915'), - LOWER('0x2501713A67a3dEdde090E42759088A7eF37D4EAb') - ) - - {% if is_incremental() %} - AND modified_timestamp >= ( - SELECT MAX(modified_timestamp) - INTERVAL '12 hours' FROM {{ this }} - ) - AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' - {% endif %} - - QUALIFY ROW_NUMBER() OVER ( - PARTITION BY contract_address - ORDER BY block_number ASC - ) = 1 - ), - - comp_v3_data AS ( - SELECT - l.contract_address AS ctoken_address, - c1.symbol AS ctoken_symbol, - c1.name AS ctoken_name, - c1.decimals AS ctoken_decimals, - LOWER( - CONCAT( - '0x', - SUBSTR( - l.api_response:data:result :: STRING, - -40 - ) - ) - ) AS underlying_address, - c2.name AS underlying_name, - c2.symbol AS underlying_symbol, - c2.decimals AS underlying_decimals, - l.block_number AS created_block, - 'Compound V3' AS compound_version - FROM comp_v3_base l - LEFT JOIN contracts_dim c1 ON l.contract_address = c1.address - LEFT JOIN contracts_dim c2 ON LOWER( +WITH contracts_dim AS ( + SELECT + address, + name, + symbol, + decimals + FROM {{ ref('core__dim_contracts') }} +), +comp_v3_base AS ( + SELECT + contract_address, + block_number, + live.udf_api( + 'POST', + '{URL}', + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', 'livequery' + ), + utils.udf_json_rpc_call( + 'eth_call', + [ + { + 'to': contract_address, + 'from': null, + 'data': RPAD('0xc55dae63', 64, '0') + }, + utils.udf_int_to_hex(block_number) + ], + concat_ws('-', contract_address, '0xc55dae63', block_number) + ), + 'Vault/prod/evm/quicknode/optimism/mainnet' + ) AS api_response + FROM {{ ref('core__fact_event_logs') }} + WHERE topic_0 = '0xbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b' + AND origin_from_address IN ( + LOWER('0x6103DB328d4864dc16BD2F0eE1B9A92e3F87f915'), + LOWER('0x2501713A67a3dEdde090E42759088A7eF37D4EAb') + ) + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT MAX(modified_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY contract_address + ORDER BY block_number ASC + ) = 1 +), +comp_v3_data AS ( + SELECT + l.contract_address AS ctoken_address, + c1.symbol AS ctoken_symbol, + c1.name AS ctoken_name, + c1.decimals AS ctoken_decimals, + LOWER( CONCAT( '0x', - SUBSTR( - l.api_response:data:result :: STRING, - -40 - ) + SUBSTR(l.api_response:data:result :: STRING, -40) ) - ) = c2.address - WHERE c1.name IS NOT NULL - ) - - SELECT - ctoken_address AS compound_market_address, - ctoken_symbol AS compound_market_symbol, - ctoken_name AS compound_market_name, - ctoken_decimals AS compound_market_decimals, - underlying_address AS underlying_asset_address, - underlying_name AS underlying_asset_name, - underlying_symbol AS underlying_asset_symbol, - underlying_decimals AS underlying_asset_decimals, - created_block AS created_block_number, - compound_version, - {{ dbt_utils.generate_surrogate_key(['compound_market_address']) }} AS comp_asset_details_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - comp_v3_data + ) AS underlying_address, + c2.name AS underlying_name, + c2.symbol AS underlying_symbol, + c2.decimals AS underlying_decimals, + l.block_number AS created_block, + 'Compound V3' AS compound_version + FROM comp_v3_base l + LEFT JOIN contracts_dim c1 ON l.contract_address = c1.address + LEFT JOIN contracts_dim c2 ON LOWER( + CONCAT('0x', SUBSTR(l.api_response:data:result :: STRING, -40)) + ) = c2.address + WHERE c1.name IS NOT NULL +) +SELECT + ctoken_address AS compound_market_address, + ctoken_symbol AS compound_market_symbol, + ctoken_name AS compound_market_name, + ctoken_decimals AS compound_market_decimals, + underlying_address AS underlying_asset_address, + underlying_name AS underlying_asset_name, + underlying_symbol AS underlying_asset_symbol, + underlying_decimals AS underlying_asset_decimals, + created_block AS created_block_number, + compound_version, + {{ dbt_utils.generate_surrogate_key(['compound_market_address']) }} AS comp_asset_details_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM comp_v3_data diff --git a/models/silver/defi/lending/compound/silver__comp_borrows.sql b/models/silver/defi/lending/compound/silver__comp_borrows.sql index 7b57639f..2e79a536 100644 --- a/models/silver/defi/lending/compound/silver__comp_borrows.sql +++ b/models/silver/defi/lending/compound/silver__comp_borrows.sql @@ -7,7 +7,6 @@ ) }} WITH comp_assets AS ( - SELECT compound_market_address, compound_market_name, @@ -17,27 +16,24 @@ WITH comp_assets AS ( underlying_asset_name, underlying_asset_symbol, underlying_asset_decimals - FROM - {{ ref('silver__comp_asset_details') }} + FROM {{ ref('silver__comp_asset_details') }} ), borrow AS ( SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - contract_address AS asset, - CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS src_address, - CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS to_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: INTEGER AS borrow_amount, - origin_from_address AS borrower_address, + l.tx_hash, + l.block_number, + l.block_timestamp, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, + l.contract_address, + regexp_substr_all(SUBSTR(l.DATA, 3, len(l.DATA)), '.{64}') AS segmented_data, + l.contract_address AS asset, + CONCAT('0x', SUBSTR(l.topics[1] :: STRING, 27, 40)) AS src_address, + CONCAT('0x', SUBSTR(l.topics[2] :: STRING, 27, 40)) AS to_address, + utils.udf_hex_to_int(segmented_data[0] :: STRING) :: INTEGER AS borrow_amount, + l.origin_from_address AS borrower_address, 'Compound V3' AS compound_version, C.compound_market_name AS NAME, C.compound_market_symbol AS symbol, @@ -45,36 +41,21 @@ borrow AS ( C.underlying_asset_address, C.underlying_asset_symbol, 'optimism' AS blockchain, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, - modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - l - LEFT JOIN comp_assets C - ON asset = C.compound_market_address - WHERE - topics [0] = '0x9b1bfa7fa9ee420a16e124f794c35ac9f90472acc99140eb2f6447c714cad8eb' --withdrawl - AND l.contract_address IN ( - SELECT - DISTINCT(compound_market_address) - FROM - comp_assets - ) - AND tx_succeeded - -{% if is_incremental() %} -AND l.modified_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + CONCAT(l.tx_hash :: STRING, '-', l.event_index :: STRING) AS _log_id, + l.modified_timestamp AS _inserted_timestamp + FROM {{ ref('core__fact_event_logs') }} l + LEFT JOIN comp_assets C ON l.contract_address = C.compound_market_address + WHERE l.topics[0] = '0x9b1bfa7fa9ee420a16e124f794c35ac9f90472acc99140eb2f6447c714cad8eb' + AND l.contract_address IN ( + SELECT DISTINCT compound_market_address FROM comp_assets + ) + AND l.tx_succeeded + {% if is_incremental() %} + AND l.modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ) SELECT tx_hash, @@ -85,21 +66,16 @@ SELECT origin_to_address, origin_function_signature, contract_address, - w.asset AS compound_market, + asset AS compound_market, borrower_address AS borrower, - w.underlying_asset_address AS token_address, - w.underlying_asset_symbol AS token_symbol, + underlying_asset_address AS token_address, + underlying_asset_symbol AS token_symbol, borrow_amount AS amount_unadj, - borrow_amount / pow( - 10, - w.decimals - ) AS amount, - w.symbol AS itoken_symbol, + borrow_amount / pow(10, decimals) AS amount, + symbol AS itoken_symbol, compound_version, blockchain, _log_id, _inserted_timestamp -FROM - borrow w qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM borrow +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/compound/silver__comp_deposits.sql b/models/silver/defi/lending/compound/silver__comp_deposits.sql index 72948e58..d7ea9b95 100644 --- a/models/silver/defi/lending/compound/silver__comp_deposits.sql +++ b/models/silver/defi/lending/compound/silver__comp_deposits.sql @@ -7,7 +7,6 @@ ) }} WITH comp_assets AS ( - SELECT compound_market_address, compound_market_name, @@ -17,62 +16,44 @@ WITH comp_assets AS ( underlying_asset_name, underlying_asset_symbol, underlying_asset_decimals - FROM - {{ ref('silver__comp_asset_details') }} + FROM {{ ref('silver__comp_asset_details') }} ), supply AS ( SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, + l.tx_hash, + l.block_number, + l.block_timestamp, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, l.contract_address, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + regexp_substr_all(SUBSTR(l.DATA, 3, len(l.DATA)), '.{64}') AS segmented_data, l.contract_address AS compound_market, - CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS asset, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: INTEGER AS supply_amount, - origin_from_address AS depositor_address, + CONCAT('0x', SUBSTR(l.topics[3] :: STRING, 27, 40)) AS asset, + utils.udf_hex_to_int(segmented_data[0] :: STRING) :: INTEGER AS supply_amount, + l.origin_from_address AS depositor_address, 'Compound V3' AS compound_version, C.contract_address AS underlying_asset_address, C.token_name, C.token_symbol, C.token_decimals, 'optimism' AS blockchain, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, + CONCAT(l.tx_hash :: STRING, '-', l.event_index :: STRING) AS _log_id, l.modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - l - LEFT JOIN {{ ref('silver__contracts') }} C - ON asset = C.contract_address - WHERE - topics [0] = '0xfa56f7b24f17183d81894d3ac2ee654e3c26388d17a28dbd9549b8114304e1f4' --SupplyCollateral - AND l.contract_address IN ( - SELECT - DISTINCT(compound_market_address) - FROM - comp_assets - ) - AND tx_succeeded - -{% if is_incremental() %} -AND l.modified_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + FROM {{ ref('core__fact_event_logs') }} l + LEFT JOIN {{ ref('silver__contracts') }} C ON asset = C.contract_address + WHERE l.topics[0] = '0xfa56f7b24f17183d81894d3ac2ee654e3c26388d17a28dbd9549b8114304e1f4' + AND l.contract_address IN ( + SELECT DISTINCT compound_market_address FROM comp_assets + ) + AND l.tx_succeeded + {% if is_incremental() %} + AND l.modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ) SELECT tx_hash, @@ -86,17 +67,12 @@ SELECT compound_market, depositor_address, asset AS token_address, - token_symbol AS token_symbol, + token_symbol, supply_amount AS amount_unadj, - supply_amount / pow( - 10, - w.token_decimals - ) AS amount, + supply_amount / pow(10, token_decimals) AS amount, compound_version, blockchain, _log_id, _inserted_timestamp -FROM - supply w qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM supply +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/compound/silver__comp_liquidations.sql b/models/silver/defi/lending/compound/silver__comp_liquidations.sql index 4d70beb2..9d6645c5 100644 --- a/models/silver/defi/lending/compound/silver__comp_liquidations.sql +++ b/models/silver/defi/lending/compound/silver__comp_liquidations.sql @@ -7,7 +7,6 @@ ) }} WITH comp_assets AS ( - SELECT compound_market_address, compound_market_name, @@ -17,75 +16,55 @@ WITH comp_assets AS ( underlying_asset_name, underlying_asset_symbol, underlying_asset_decimals - FROM - {{ ref('silver__comp_asset_details') }} + FROM {{ ref('silver__comp_asset_details') }} ), liquidations AS ( SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, + l.tx_hash, + l.block_number, + l.block_timestamp, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, l.contract_address, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + regexp_substr_all(SUBSTR(l.DATA, 3, len(l.DATA)), '.{64}') AS segmented_data, l.contract_address AS compound_market, - CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS asset, - CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS absorber, - CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS borrower, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: INTEGER AS collateral_absorbed, - utils.udf_hex_to_int( - segmented_data [1] :: STRING - ) :: INTEGER AS usd_value, - origin_from_address AS depositor_address, + CONCAT('0x', SUBSTR(l.topics[3] :: STRING, 27, 40)) AS asset, + CONCAT('0x', SUBSTR(l.topics[1] :: STRING, 27, 40)) AS absorber, + CONCAT('0x', SUBSTR(l.topics[2] :: STRING, 27, 40)) AS borrower, + utils.udf_hex_to_int(segmented_data[0] :: STRING) :: INTEGER AS collateral_absorbed, + utils.udf_hex_to_int(segmented_data[1] :: STRING) :: INTEGER AS usd_value, + l.origin_from_address AS depositor_address, 'Compound V3' AS compound_version, C.token_name, C.token_symbol, C.token_decimals, 'optimism' AS blockchain, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, + CONCAT(l.tx_hash :: STRING, '-', l.event_index :: STRING) AS _log_id, l.modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - l - LEFT JOIN {{ ref('silver__contracts') }} C - ON asset = C.contract_address - WHERE - topics [0] = '0x9850ab1af75177e4a9201c65a2cf7976d5d28e40ef63494b44366f86b2f9412e' --AbsorbCollateral - AND l.contract_address IN ( - SELECT - DISTINCT(compound_market_address) - FROM - comp_assets - ) - AND tx_succeeded - -{% if is_incremental() %} -AND l.modified_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + FROM {{ ref('core__fact_event_logs') }} l + LEFT JOIN {{ ref('silver__contracts') }} C ON asset = C.contract_address + WHERE l.topics[0] = '0x9850ab1af75177e4a9201c65a2cf7976d5d28e40ef63494b44366f86b2f9412e' + AND l.contract_address IN ( + SELECT DISTINCT compound_market_address FROM comp_assets + ) + AND l.tx_succeeded + {% if is_incremental() %} + AND l.modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ) SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, + l.tx_hash, + l.block_number, + l.block_timestamp, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, l.contract_address, compound_market, absorber, @@ -94,23 +73,14 @@ SELECT asset AS token_address, token_symbol, collateral_absorbed AS amount_unadj, - collateral_absorbed / pow( - 10, - token_decimals - ) AS amount, - usd_value / pow( - 10, - 8 - ) AS amount_usd, + collateral_absorbed / pow(10, token_decimals) AS amount, + usd_value / pow(10, 8) AS amount_usd, A.underlying_asset_address AS debt_asset, A.underlying_asset_symbol AS debt_asset_symbol, compound_version, blockchain, l._log_id, l._inserted_timestamp -FROM - liquidations l - LEFT JOIN comp_assets A - ON l.compound_market = A.compound_market_address qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM liquidations l +LEFT JOIN comp_assets A ON l.compound_market = A.compound_market_address +QUALIFY ROW_NUMBER() OVER (PARTITION BY l._log_id ORDER BY l._inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/compound/silver__comp_repayments.sql b/models/silver/defi/lending/compound/silver__comp_repayments.sql index ceaf9852..53309179 100644 --- a/models/silver/defi/lending/compound/silver__comp_repayments.sql +++ b/models/silver/defi/lending/compound/silver__comp_repayments.sql @@ -7,7 +7,6 @@ ) }} WITH comp_assets AS ( - SELECT compound_market_address, compound_market_name, @@ -17,67 +16,47 @@ WITH comp_assets AS ( underlying_asset_name, underlying_asset_symbol, underlying_asset_decimals - FROM - {{ ref('silver__comp_asset_details') }} + FROM {{ ref('silver__comp_asset_details') }} ), repayments AS ( SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, + l.tx_hash, + l.block_number, + l.block_timestamp, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, l.contract_address, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + regexp_substr_all(SUBSTR(l.DATA, 3, len(l.DATA)), '.{64}') AS segmented_data, l.contract_address AS asset, - CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS repayer, - CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS borrower, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: INTEGER AS amount, - utils.udf_hex_to_int( - segmented_data [1] :: STRING - ) :: INTEGER AS usd_value, - origin_from_address AS depositor, + CONCAT('0x', SUBSTR(l.topics[1] :: STRING, 27, 40)) AS repayer, + CONCAT('0x', SUBSTR(l.topics[2] :: STRING, 27, 40)) AS borrower, + utils.udf_hex_to_int(segmented_data[0] :: STRING) :: INTEGER AS amount, + utils.udf_hex_to_int(segmented_data[1] :: STRING) :: INTEGER AS usd_value, + l.origin_from_address AS depositor, 'Compound V3' AS compound_version, - compound_market_name, - compound_market_symbol, - compound_market_decimals, + C.compound_market_name, + C.compound_market_symbol, + C.compound_market_decimals, C.underlying_asset_address AS underlying_asset, C.underlying_asset_symbol, 'optimism' AS blockchain, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, + CONCAT(l.tx_hash :: STRING, '-', l.event_index :: STRING) AS _log_id, l.modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - l - LEFT JOIN comp_assets C - ON contract_address = C.compound_market_address - WHERE - topics [0] = '0xd1cf3d156d5f8f0d50f6c122ed609cec09d35c9b9fb3fff6ea0959134dae424e' --Supply - AND l.contract_address IN ( - SELECT - DISTINCT(compound_market_address) - FROM - comp_assets - ) - AND tx_succeeded - -{% if is_incremental() %} -AND l.modified_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + FROM {{ ref('core__fact_event_logs') }} l + LEFT JOIN comp_assets C ON l.contract_address = C.compound_market_address + WHERE l.topics[0] = '0xd1cf3d156d5f8f0d50f6c122ed609cec09d35c9b9fb3fff6ea0959134dae424e' + AND l.contract_address IN ( + SELECT DISTINCT compound_market_address FROM comp_assets + ) + AND l.tx_succeeded + {% if is_incremental() %} + AND l.modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ) SELECT tx_hash, @@ -88,22 +67,17 @@ SELECT origin_to_address, origin_function_signature, contract_address, - w.asset AS compound_market, + asset AS compound_market, repayer, borrower, depositor, underlying_asset AS token_address, - w.underlying_asset_symbol AS token_symbol, + underlying_asset_symbol AS token_symbol, amount AS amount_unadj, - amount / pow( - 10, - w.compound_market_decimals - ) AS amount, + amount / pow(10, compound_market_decimals) AS amount, compound_version, blockchain, _log_id, _inserted_timestamp -FROM - repayments w qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM repayments +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/compound/silver__comp_withdraws.sql b/models/silver/defi/lending/compound/silver__comp_withdraws.sql index 4d805356..a6636318 100644 --- a/models/silver/defi/lending/compound/silver__comp_withdraws.sql +++ b/models/silver/defi/lending/compound/silver__comp_withdraws.sql @@ -7,7 +7,6 @@ ) }} WITH comp_assets AS ( - SELECT compound_market_address, compound_market_name, @@ -17,61 +16,43 @@ WITH comp_assets AS ( underlying_asset_name, underlying_asset_symbol, underlying_asset_decimals - FROM - {{ ref('silver__comp_asset_details') }} + FROM {{ ref('silver__comp_asset_details') }} ), withdraw AS ( SELECT - tx_hash, - block_number, - block_timestamp, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, + l.tx_hash, + l.block_number, + l.block_timestamp, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, l.contract_address, - regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, + regexp_substr_all(SUBSTR(l.DATA, 3, len(l.DATA)), '.{64}') AS segmented_data, l.contract_address AS compound_market, - CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS token_address, - utils.udf_hex_to_int( - segmented_data [0] :: STRING - ) :: INTEGER AS withdraw_amount, - origin_from_address AS depositor_address, + CONCAT('0x', SUBSTR(l.topics[3] :: STRING, 27, 40)) AS token_address, + utils.udf_hex_to_int(segmented_data[0] :: STRING) :: INTEGER AS withdraw_amount, + l.origin_from_address AS depositor_address, 'Compound V3' AS compound_version, C.token_name, C.token_symbol, C.token_decimals, 'optimism' AS blockchain, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id, + CONCAT(l.tx_hash :: STRING, '-', l.event_index :: STRING) AS _log_id, l.modified_timestamp AS _inserted_timestamp - FROM - {{ ref('core__fact_event_logs') }} - l - LEFT JOIN {{ ref('silver__contracts') }} C - ON token_address = C.contract_address - WHERE - topics [0] = '0xd6d480d5b3068db003533b170d67561494d72e3bf9fa40a266471351ebba9e16' --WithdrawCollateral - AND l.contract_address IN ( - SELECT - DISTINCT(compound_market_address) - FROM - comp_assets - ) - AND tx_succeeded - -{% if is_incremental() %} -AND l.modified_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + FROM {{ ref('core__fact_event_logs') }} l + LEFT JOIN {{ ref('silver__contracts') }} C ON token_address = C.contract_address + WHERE l.topics[0] = '0xd6d480d5b3068db003533b170d67561494d72e3bf9fa40a266471351ebba9e16' + AND l.contract_address IN ( + SELECT DISTINCT compound_market_address FROM comp_assets + ) + AND l.tx_succeeded + {% if is_incremental() %} + AND l.modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND l.modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ) SELECT tx_hash, @@ -84,18 +65,13 @@ SELECT contract_address, compound_market, depositor_address, - w.token_address, - w.token_symbol, + token_address, + token_symbol, withdraw_amount AS amount_unadj, - withdraw_amount / pow( - 10, - w.token_decimals - ) AS amount, + withdraw_amount / pow(10, token_decimals) AS amount, compound_version, blockchain, _log_id, _inserted_timestamp -FROM - withdraw w qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM withdraw +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql b/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql index 0c445c73..a8d228e4 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_asset_details.sql @@ -7,58 +7,35 @@ ) }} WITH log_pull AS ( - SELECT tx_hash, block_number, block_timestamp, contract_address, modified_timestamp AS _inserted_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id - FROM - {{ ref('core__fact_event_logs') }} - WHERE - topics [0] :: STRING = '0x7ac369dbd14fa5ea3f473ed67cc9d598964a77501540ba6751eb0b3decf5870d' - AND origin_from_address = LOWER('0xFb59Ce8986943163F14C590755b29dB2998F2322') - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + CONCAT(tx_hash :: STRING, '-', event_index :: STRING) AS _log_id + FROM {{ ref('core__fact_event_logs') }} + WHERE topics[0] :: STRING = '0x7ac369dbd14fa5ea3f473ed67cc9d598964a77501540ba6751eb0b3decf5870d' + AND origin_from_address = LOWER('0xFb59Ce8986943163F14C590755b29dB2998F2322') + {% if is_incremental() %} + AND _inserted_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ), traces_pull AS ( SELECT from_address AS token_address, to_address AS underlying_asset - FROM - {{ ref('core__fact_traces') }} - WHERE - tx_hash IN ( - SELECT - tx_hash - FROM - log_pull + FROM {{ ref('core__fact_traces') }} + WHERE tx_hash IN ( + SELECT tx_hash FROM log_pull ) - AND concat_ws( - '_', - TYPE, - trace_address - ) = 'STATICCALL_2' + AND concat_ws('_', TYPE, trace_address) = 'STATICCALL_2' ), contracts AS ( - SELECT - * - FROM - {{ ref('silver__contracts') }} + SELECT * FROM {{ ref('silver__contracts') }} ), contract_pull AS ( SELECT @@ -72,14 +49,10 @@ contract_pull AS ( t.underlying_asset, l._inserted_timestamp, l._log_id - FROM - log_pull l - LEFT JOIN traces_pull t - ON l.contract_address = t.token_address - LEFT JOIN contracts C - ON C.contract_address = l.contract_address qualify(ROW_NUMBER() over(PARTITION BY l.contract_address - ORDER BY - block_timestamp ASC)) = 1 + FROM log_pull l + LEFT JOIN traces_pull t ON l.contract_address = t.token_address + LEFT JOIN contracts C ON C.contract_address = l.contract_address + QUALIFY ROW_NUMBER() OVER (PARTITION BY l.contract_address ORDER BY block_timestamp ASC) = 1 ) SELECT l.tx_hash, @@ -95,10 +68,7 @@ SELECT C.token_decimals AS underlying_decimals, l._inserted_timestamp, l._log_id -FROM - contract_pull l - LEFT JOIN contracts C - ON C.contract_address = l.underlying_asset -WHERE - underlying_asset IS NOT NULL - AND l.token_name IS NOT NULL +FROM contract_pull l +LEFT JOIN contracts C ON C.contract_address = l.underlying_asset +WHERE underlying_asset IS NOT NULL + AND l.token_name IS NOT NULL diff --git a/models/silver/defi/lending/sonne/silver__sonne_borrows.sql b/models/silver/defi/lending/sonne/silver__sonne_borrows.sql index 344482fe..99fea4da 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_borrows.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_borrows.sql @@ -3,11 +3,10 @@ incremental_strategy = 'delete+insert', unique_key = "block_number", cluster_by = ['block_timestamp::DATE'], - tags = ['silver','defi','lending','curated'] + tags = ['silver','defi','lending','curated'] ) }} --- pull all token addresses and corresponding name -WITH asset_details AS ( +WITH asset_details AS ( SELECT token_address, token_symbol, @@ -17,8 +16,7 @@ WITH asset_details AS ( underlying_name, underlying_symbol, underlying_decimals - FROM - {{ ref('silver__sonne_asset_details') }} + FROM {{ ref('silver__sonne_asset_details') }} ), sonne_borrows AS ( SELECT @@ -31,70 +29,49 @@ sonne_borrows AS ( origin_function_signature, contract_address, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS borrower, - utils.udf_hex_to_int( - segmented_data [1] :: STRING - ) :: INTEGER AS loan_amount_raw, - utils.udf_hex_to_int( - segmented_data [2] :: STRING - ) :: INTEGER AS accountBorrows, - utils.udf_hex_to_int( - segmented_data [3] :: STRING - ) :: INTEGER AS totalBorrows, + CONCAT('0x', SUBSTR(segmented_data[0] :: STRING, 25, 40)) AS borrower, + utils.udf_hex_to_int(segmented_data[1] :: STRING) :: INTEGER AS loan_amount_raw, + utils.udf_hex_to_int(segmented_data[2] :: STRING) :: INTEGER AS accountBorrows, + utils.udf_hex_to_int(segmented_data[3] :: STRING) :: INTEGER AS totalBorrows, contract_address AS token, 'Sonne' AS platform, modified_timestamp AS _inserted_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id - FROM - {{ ref('core__fact_event_logs') }} - WHERE - contract_address IN ( - SELECT - token_address - FROM - asset_details + CONCAT(tx_hash :: STRING, '-', event_index :: STRING) AS _log_id + FROM {{ ref('core__fact_event_logs') }} + WHERE contract_address IN ( + SELECT token_address FROM asset_details ) - AND topics [0] :: STRING = '0x13ed6866d4e1ee6da46f845c46d7e54120883d75c5ea9a2dacc1c4ca8984ab80' + AND topics[0] :: STRING = '0x13ed6866d4e1ee6da46f845c46d7e54120883d75c5ea9a2dacc1c4ca8984ab80' AND tx_succeeded - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ), sonne_combine AS ( SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - borrower, - loan_amount_raw, + b.block_number, + b.block_timestamp, + b.tx_hash, + b.event_index, + b.origin_from_address, + b.origin_to_address, + b.origin_function_signature, + b.contract_address, + b.borrower, + b.loan_amount_raw, C.underlying_asset_address AS borrows_contract_address, C.underlying_symbol AS borrows_contract_symbol, - token, + b.token, C.token_symbol, C.underlying_decimals, b.platform, b._log_id, b._inserted_timestamp - FROM - sonne_borrows b - LEFT JOIN asset_details C - ON b.token = C.token_address + FROM sonne_borrows b + LEFT JOIN asset_details C ON b.token = C.token_address ) SELECT block_number, @@ -108,17 +85,12 @@ SELECT borrower, borrows_contract_address, borrows_contract_symbol, - token as token_address, + token AS token_address, token_symbol, loan_amount_raw AS amount_unadj, - loan_amount_raw / pow( - 10, - underlying_decimals - ) AS amount, + loan_amount_raw / pow(10, underlying_decimals) AS amount, platform, _inserted_timestamp, _log_id -FROM - sonne_combine qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM sonne_combine +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/sonne/silver__sonne_deposits.sql b/models/silver/defi/lending/sonne/silver__sonne_deposits.sql index 5f89041a..288d78b2 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_deposits.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_deposits.sql @@ -3,11 +3,10 @@ incremental_strategy = 'delete+insert', unique_key = "block_number", cluster_by = ['block_timestamp::DATE'], - tags = ['silver','defi','lending','curated'] + tags = ['silver','defi','lending','curated'] ) }} --- pull all token addresses and corresponding name -WITH asset_details AS ( +WITH asset_details AS ( SELECT token_address, token_symbol, @@ -17,8 +16,7 @@ WITH asset_details AS ( underlying_name, underlying_symbol, underlying_decimals - FROM - {{ ref('silver__sonne_asset_details') }} + FROM {{ ref('silver__sonne_asset_details') }} ), sonne_deposits AS ( SELECT @@ -32,55 +30,38 @@ sonne_deposits AS ( contract_address, contract_address AS token_address, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - utils.udf_hex_to_int( - segmented_data [2] :: STRING - ) :: INTEGER AS minttokens_raw, - utils.udf_hex_to_int( - segmented_data [1] :: STRING - ) :: INTEGER AS mintAmount_raw, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS supplier, + utils.udf_hex_to_int(segmented_data[2] :: STRING) :: INTEGER AS minttokens_raw, + utils.udf_hex_to_int(segmented_data[1] :: STRING) :: INTEGER AS mintAmount_raw, + CONCAT('0x', SUBSTR(segmented_data[0] :: STRING, 25, 40)) AS supplier, 'Sonne' AS platform, modified_timestamp AS _inserted_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id - FROM - {{ ref('core__fact_event_logs') }} - WHERE - contract_address IN ( - SELECT - token_address - FROM - asset_details + CONCAT(tx_hash :: STRING, '-', event_index :: STRING) AS _log_id + FROM {{ ref('core__fact_event_logs') }} + WHERE contract_address IN ( + SELECT token_address FROM asset_details ) - AND topics [0] :: STRING = '0x4c209b5fc8ad50758f13e2e1088ba56a560dff690a1c6fef26394f4c03821c4f' + AND topics[0] :: STRING = '0x4c209b5fc8ad50758f13e2e1088ba56a560dff690a1c6fef26394f4c03821c4f' AND tx_succeeded - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ), sonne_combine AS ( SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - supplier, - minttokens_raw, - mintAmount_raw, + b.block_number, + b.block_timestamp, + b.tx_hash, + b.event_index, + b.origin_from_address, + b.origin_to_address, + b.origin_function_signature, + b.contract_address, + b.supplier, + b.minttokens_raw, + b.mintAmount_raw, C.underlying_asset_address AS supplied_contract_addr, C.underlying_symbol AS supplied_symbol, C.token_address, @@ -90,10 +71,8 @@ sonne_combine AS ( b.platform, b._log_id, b._inserted_timestamp - FROM - sonne_deposits b - LEFT JOIN asset_details C - ON b.token_address = C.token_address + FROM sonne_deposits b + LEFT JOIN asset_details C ON b.token_address = C.token_address ) SELECT block_number, @@ -106,22 +85,14 @@ SELECT contract_address, token_address, token_symbol, - minttokens_raw / pow( - 10, - token_decimals - ) AS issued_tokens, + minttokens_raw / pow(10, token_decimals) AS issued_tokens, mintAmount_raw AS amount_unadj, - mintAmount_raw / pow( - 10, - underlying_decimals - ) AS amount, + mintAmount_raw / pow(10, underlying_decimals) AS amount, supplied_contract_addr, supplied_symbol, supplier, platform, _inserted_timestamp, _log_id -FROM - sonne_combine qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM sonne_combine +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql b/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql index e834d389..96efe653 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_liquidations.sql @@ -3,11 +3,10 @@ incremental_strategy = 'delete+insert', unique_key = "block_number", cluster_by = ['block_timestamp::DATE'], - tags = ['silver','defi','lending','curated'] + tags = ['silver','defi','lending','curated'] ) }} WITH asset_details AS ( - SELECT token_address, token_symbol, @@ -17,8 +16,7 @@ WITH asset_details AS ( underlying_name, underlying_symbol, underlying_decimals - FROM - {{ ref('silver__sonne_asset_details') }} + FROM {{ ref('silver__sonne_asset_details') }} ), sonne_liquidations AS ( SELECT @@ -31,88 +29,60 @@ sonne_liquidations AS ( origin_function_signature, contract_address, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS borrower, + CONCAT('0x', SUBSTR(segmented_data[1] :: STRING, 25, 40)) AS borrower, contract_address AS token, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS liquidator, - utils.udf_hex_to_int( - segmented_data [4] :: STRING - ) :: INTEGER AS seizeTokens_raw, - utils.udf_hex_to_int( - segmented_data [2] :: STRING - ) :: INTEGER AS repayAmount_raw, - CONCAT('0x', SUBSTR(segmented_data [3] :: STRING, 25, 40)) AS tokenCollateral, + CONCAT('0x', SUBSTR(segmented_data[0] :: STRING, 25, 40)) AS liquidator, + utils.udf_hex_to_int(segmented_data[4] :: STRING) :: INTEGER AS seizeTokens_raw, + utils.udf_hex_to_int(segmented_data[2] :: STRING) :: INTEGER AS repayAmount_raw, + CONCAT('0x', SUBSTR(segmented_data[3] :: STRING, 25, 40)) AS tokenCollateral, 'Sonne' AS platform, modified_timestamp AS _inserted_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id - FROM - {{ ref('core__fact_event_logs') }} - WHERE - contract_address IN ( - SELECT - token_address - FROM - asset_details + CONCAT(tx_hash :: STRING, '-', event_index :: STRING) AS _log_id + FROM {{ ref('core__fact_event_logs') }} + WHERE contract_address IN ( + SELECT token_address FROM asset_details ) - AND topics [0] :: STRING = '0x298637f684da70674f26509b10f07ec2fbc77a335ab1e7d6215a4b2484d8bb52' + AND topics[0] :: STRING = '0x298637f684da70674f26509b10f07ec2fbc77a335ab1e7d6215a4b2484d8bb52' AND tx_succeeded - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ), liquidation_union AS ( SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - borrower, - token, + l.block_number, + l.block_timestamp, + l.tx_hash, + l.event_index, + l.origin_from_address, + l.origin_to_address, + l.origin_function_signature, + l.contract_address, + l.borrower, + l.token, asd1.token_symbol AS token_symbol, - liquidator, - seizeTokens_raw / pow( - 10, - asd2.token_decimals - ) AS tokens_seized, - tokenCollateral AS protocol_market, + l.liquidator, + seizeTokens_raw / pow(10, asd2.token_decimals) AS tokens_seized, + l.tokenCollateral AS protocol_market, asd2.token_symbol AS collateral_token_symbol, asd2.underlying_asset_address AS collateral_token, asd2.underlying_symbol AS collateral_symbol, - repayAmount_raw AS amount_unadj, - repayAmount_raw / pow( - 10, - asd1.underlying_decimals - ) AS amount, + l.repayAmount_raw AS amount_unadj, + l.repayAmount_raw / pow(10, asd1.underlying_decimals) AS amount, asd1.underlying_decimals, asd1.underlying_asset_address AS liquidation_contract_address, asd1.underlying_symbol AS liquidation_contract_symbol, l.platform, l._inserted_timestamp, l._log_id - FROM - sonne_liquidations l - LEFT JOIN asset_details asd1 - ON l.token = asd1.token_address - LEFT JOIN asset_details asd2 - ON l.tokenCollateral = asd2.token_address + FROM sonne_liquidations l + LEFT JOIN asset_details asd1 ON l.token = asd1.token_address + LEFT JOIN asset_details asd2 ON l.tokenCollateral = asd2.token_address ) SELECT * -FROM - liquidation_union qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM liquidation_union +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/sonne/silver__sonne_repayments.sql b/models/silver/defi/lending/sonne/silver__sonne_repayments.sql index 9f55476e..344576b6 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_repayments.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_repayments.sql @@ -3,11 +3,10 @@ incremental_strategy = 'delete+insert', unique_key = "block_number", cluster_by = ['block_timestamp::DATE'], - tags = ['silver','defi','lending','curated'] + tags = ['silver','defi','lending','curated'] ) }} --- pull all token addresses and corresponding name -WITH asset_details AS ( +WITH asset_details AS ( SELECT token_address, token_symbol, @@ -17,8 +16,7 @@ WITH asset_details AS ( underlying_name, underlying_symbol, underlying_decimals - FROM - {{ ref('silver__sonne_asset_details') }} + FROM {{ ref('silver__sonne_asset_details') }} ), sonne_repayments AS ( SELECT @@ -31,66 +29,49 @@ sonne_repayments AS ( origin_function_signature, contract_address, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - CONCAT('0x', SUBSTR(segmented_data [1] :: STRING, 25, 40)) AS borrower, + CONCAT('0x', SUBSTR(segmented_data[1] :: STRING, 25, 40)) AS borrower, contract_address AS token, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS payer, - utils.udf_hex_to_int( - segmented_data [2] :: STRING - ) :: INTEGER AS repayed_amount_raw, + CONCAT('0x', SUBSTR(segmented_data[0] :: STRING, 25, 40)) AS payer, + utils.udf_hex_to_int(segmented_data[2] :: STRING) :: INTEGER AS repayed_amount_raw, 'Sonne' AS platform, modified_timestamp AS _inserted_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id - FROM - {{ ref('core__fact_event_logs') }} - WHERE - contract_address IN ( - SELECT - token_address - FROM - asset_details + CONCAT(tx_hash :: STRING, '-', event_index :: STRING) AS _log_id + FROM {{ ref('core__fact_event_logs') }} + WHERE contract_address IN ( + SELECT token_address FROM asset_details ) - AND topics [0] :: STRING = '0x1a2a22cb034d26d1854bdc6666a5b91fe25efbbb5dcad3b0355478d6f5c362a1' + AND topics[0] :: STRING = '0x1a2a22cb034d26d1854bdc6666a5b91fe25efbbb5dcad3b0355478d6f5c362a1' AND tx_succeeded - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ), sonne_combine AS ( SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - borrower, - token, + b.block_number, + b.block_timestamp, + b.tx_hash, + b.event_index, + b.origin_from_address, + b.origin_to_address, + b.origin_function_signature, + b.contract_address, + b.borrower, + b.token, C.token_symbol, - payer, - repayed_amount_raw, + b.payer, + b.repayed_amount_raw, C.underlying_asset_address AS repay_contract_address, C.underlying_symbol AS repay_contract_symbol, C.underlying_decimals, b.platform, b._log_id, b._inserted_timestamp - FROM - sonne_repayments b - LEFT JOIN asset_details C - ON b.token = C.token_address + FROM sonne_repayments b + LEFT JOIN asset_details C ON b.token = C.token_address ) SELECT block_number, @@ -102,20 +83,15 @@ SELECT origin_function_signature, contract_address, borrower, - token as token_address, + token AS token_address, token_symbol, payer, repay_contract_address, repay_contract_symbol, repayed_amount_raw AS amount_unadj, - repayed_amount_raw / pow( - 10, - underlying_decimals - ) AS amount, + repayed_amount_raw / pow(10, underlying_decimals) AS amount, platform, _inserted_timestamp, _log_id -FROM - sonne_combine qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM sonne_combine +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql b/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql index 2cf21bdc..8f4a8201 100644 --- a/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql +++ b/models/silver/defi/lending/sonne/silver__sonne_withdraws.sql @@ -5,9 +5,8 @@ cluster_by = ['block_timestamp::DATE'], tags = ['silver','defi','lending','curated'] ) }} --- pull all token addresses and corresponding name -WITH asset_details AS ( +WITH asset_details AS ( SELECT token_address, token_symbol, @@ -17,8 +16,7 @@ WITH asset_details AS ( underlying_name, underlying_symbol, underlying_decimals - FROM - {{ ref('silver__sonne_asset_details') }} + FROM {{ ref('silver__sonne_asset_details') }} ), sonne_redemptions AS ( SELECT @@ -32,56 +30,39 @@ sonne_redemptions AS ( contract_address, contract_address AS token, regexp_substr_all(SUBSTR(DATA, 3, len(DATA)), '.{64}') AS segmented_data, - utils.udf_hex_to_int( - segmented_data [1] :: STRING - ) :: INTEGER AS received_amount_raw, - utils.udf_hex_to_int( - segmented_data [3] :: STRING - ) :: INTEGER AS redeemed_token_raw, - CONCAT('0x', SUBSTR(segmented_data [0] :: STRING, 25, 40)) AS redeemer, + utils.udf_hex_to_int(segmented_data[1] :: STRING) :: INTEGER AS received_amount_raw, + utils.udf_hex_to_int(segmented_data[3] :: STRING) :: INTEGER AS redeemed_token_raw, + CONCAT('0x', SUBSTR(segmented_data[0] :: STRING, 25, 40)) AS redeemer, 'Sonne' AS platform, modified_timestamp AS _inserted_timestamp, - CONCAT( - tx_hash :: STRING, - '-', - event_index :: STRING - ) AS _log_id - FROM - {{ ref('core__fact_event_logs') }} - WHERE - contract_address IN ( - SELECT - token_address - FROM - asset_details + CONCAT(tx_hash :: STRING, '-', event_index :: STRING) AS _log_id + FROM {{ ref('core__fact_event_logs') }} + WHERE contract_address IN ( + SELECT token_address FROM asset_details ) - AND topics [0] :: STRING = '0xe5b754fb1abb7f01b499791d0b820ae3b6af3424ac1c59768edb53f4ec31a929' + AND topics[0] :: STRING = '0xe5b754fb1abb7f01b499791d0b820ae3b6af3424ac1c59768edb53f4ec31a929' AND tx_succeeded - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - INTERVAL '12 hours' - FROM - {{ this }} -) -AND _inserted_timestamp >= SYSDATE() - INTERVAL '7 day' -{% endif %} + {% if is_incremental() %} + AND modified_timestamp >= ( + SELECT MAX(_inserted_timestamp) - INTERVAL '12 hours' FROM {{ this }} + ) + AND modified_timestamp >= SYSDATE() - INTERVAL '7 day' + {% endif %} ), sonne_combine AS ( SELECT - block_number, - block_timestamp, - tx_hash, - event_index, - origin_from_address, - origin_to_address, - origin_function_signature, - contract_address, - token, - redeemer, - received_amount_raw, - redeemed_token_raw, + b.block_number, + b.block_timestamp, + b.tx_hash, + b.event_index, + b.origin_from_address, + b.origin_to_address, + b.origin_function_signature, + b.contract_address, + b.token, + b.redeemer, + b.received_amount_raw, + b.redeemed_token_raw, C.underlying_asset_address AS received_contract_address, C.underlying_symbol AS received_contract_symbol, C.token_symbol, @@ -90,10 +71,8 @@ sonne_combine AS ( b.platform, b._log_id, b._inserted_timestamp - FROM - sonne_redemptions b - LEFT JOIN asset_details C - ON b.token = C.token_address + FROM sonne_redemptions b + LEFT JOIN asset_details C ON b.token = C.token_address ) SELECT block_number, @@ -104,24 +83,16 @@ SELECT origin_to_address, origin_function_signature, contract_address, - token as token_address, + token AS token_address, token_symbol, received_amount_raw AS amount_unadj, - received_amount_raw / pow( - 10, - underlying_decimals - ) AS amount, + received_amount_raw / pow(10, underlying_decimals) AS amount, received_contract_address, received_contract_symbol, - redeemed_token_raw / pow( - 10, - token_decimals - ) AS redeemed_token, + redeemed_token_raw / pow(10, token_decimals) AS redeemed_token, redeemer, platform, _inserted_timestamp, _log_id -FROM - sonne_combine ee qualify(ROW_NUMBER() over(PARTITION BY _log_id -ORDER BY - _inserted_timestamp DESC)) = 1 +FROM sonne_combine +QUALIFY ROW_NUMBER() OVER (PARTITION BY _log_id ORDER BY _inserted_timestamp DESC) = 1 diff --git a/models/test/test_definitions.yml b/models/test/test_definitions.yml new file mode 100644 index 00000000..271c84eb --- /dev/null +++ b/models/test/test_definitions.yml @@ -0,0 +1,18 @@ +test_tags: + recency: &recency_tags + - recency + + finance: &finance_tags + - finance + - daily + + +default_test_config: &default_test_config + severity: error + tags: ['critical', 'data_quality'] + enabled: true + +common_tags: &common_tags + - non_realtime + - critical + - production \ No newline at end of file diff --git a/packages.yml b/packages.yml index 5e55889f..4bcf8c5b 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v4.0.0-beta.89 \ No newline at end of file + revision: update-dynamic-test-macro \ No newline at end of file diff --git a/tests/test.txt b/tests/test.txt new file mode 100644 index 00000000..3d38cbc8 --- /dev/null +++ b/tests/test.txt @@ -0,0 +1,103 @@ +0x60806040523661001357610011610017565b005b6100115b61001f6100ee565b6001600160a01b031633036100e45760606001600160e01b03195f35166364d3180d60e11b810161005957610052610120565b91506100dc565b63587086bd60e11b6001600160e01b031982160161007957610052610173565b63070d7c6960e41b6001600160e01b0319821601610099576100526101b7565b621eb96f60e61b6001600160e01b03198216016100b8576100526101e7565b63a39f25e560e01b6001600160e01b03198216016100d857610052610226565b5f80fd5b815160208301f35b6100ec610239565b565b5f7fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d61035b546001600160a01b0316919050565b606061012a610249565b5f61013836600481846104e3565b8101906101459190610525565b90506101608160405180602001604052805f8152505f610253565b505060408051602081019091525f815290565b60605f8061018436600481846104e3565b8101906101919190610552565b915091506101a182826001610253565b60405180602001604052805f8152509250505090565b60606101c1610249565b5f6101cf36600481846104e3565b8101906101dc9190610525565b90506101608161027e565b60606101f1610249565b5f6101fa6100ee565b604080516001600160a01b03831660208201529192500160405160208183030381529060405291505090565b6060610230610249565b5f6101fa6102d5565b6100ec6102446102d5565b6102e3565b34156100ec575f80fd5b61025c83610301565b5f825111806102685750805b15610279576102778383610340565b505b505050565b7f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c9798f6102a76100ee565b604080516001600160a01b03928316815291841660208301520160405180910390a16102d28161036c565b50565b5f6102de6103c2565b905090565b365f80375f80365f845af43d5f803e8080156102fd573d5ff35b3d5ffd5b61030a816103e9565b6040516001600160a01b038216907fbc7cd75a20ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b905f90a250565b6060610365838360405180606001604052806027815260200161062560279139610423565b9392505050565b6001600160a01b03811661037e575f80fd5b807fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d61035b80546001600160a01b0319166001600160a01b039290921691909117905550565b5f7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc610111565b6001600160a01b0381163b6103fc575f80fd5b807f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc6103a1565b60605f80856001600160a01b03168560405161043f919061060e565b5f60405180830381855af49150503d805f8114610477576040519150601f19603f3d011682016040523d82523d5f602084013e61047c565b606091505b509150915061048d86838387610497565b9695505050505050565b606083156104c15782515f036104ba576001600160a01b0385163b6104ba575f80fd5b50816104cb565b6104cb83836104d3565b949350505050565b8151156100d85781518083602001fd5b5f80858511156104f1575f80fd5b838611156104fd575f80fd5b5050820193919092039150565b80356001600160a01b0381168114610520575f80fd5b919050565b5f60208284031215610535575f80fd5b6103658261050a565b634e487b7160e01b5f52604160045260245ffd5b5f8060408385031215610563575f80fd5b61056c8361050a565b9150602083013567ffffffffffffffff80821115610588575f80fd5b818501915085601f83011261059b575f80fd5b8135818111156105ad576105ad61053e565b604051601f8201601f19908116603f011681019083821181831017156105d5576105d561053e565b816040528281528860208487010111156105ed575f80fd5b826020860160208301375f6020848301015280955050505050509250929050565b5f82518060208501845e5f92019182525091905056fe416464726573733a206c6f772d6c6576656c2064656c65676174652063616c6c206661696c6564a26469706673582212207adc130af683e5732b81908080a45f6134fe389cd5cd3dde740f5a5249cbeeaf64736f6c63430008190033 + +[ + "52604051610a6d380380610a6d833981016040819052610022916102b7565b82", + "8161002f82825f610043565b5061003b90508261006e565b505050610398565b", + "61004c836100db565b5f825111806100585750805b1561006957610067838361", + "011a565b505b505050565b7f7e644d79422f17c01e4894b5f4f588d331ebfa28", + "653d42ae832dc59e38c9798f6100ad5f80516020610a26833981519152546001", + "600160a01b031690565b604080516001600160a01b0392831681529184166020", + "8301520160405180910390a16100d881610146565b50565b6100e48161018956", + "5b6040516001600160a01b038216907fbc7cd75a20ee27fd9adebab32041f755", + "214dbc6bffa90cc0225b39da2e5c2d3b905f90a250565b606061013f83836040", + "51806060016040528060278152602001610a46602791396101c3565b93925050", + "50565b6001600160a01b038116610158575f80fd5b805f80516020610a268339", + "815191525b80546001600160a01b0319166001600160a01b0392909216919091", + "17905550565b6001600160a01b0381163b61019c575f80fd5b807f360894a13b", + "a1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc610168565b", + "60605f80856001600160a01b0316856040516101df9190610382565b5f604051", + "80830381855af49150503d805f8114610217576040519150601f19603f3d0116", + "82016040523d82523d5f602084013e61021c565b606091505b50909250905061", + "022e86838387610238565b9695505050505050565b606083156102625782515f", + "0361025b576001600160a01b0385163b61025b575f80fd5b508161026c565b61", + "026c8383610274565b949350505050565b8151156102845781518083602001fd", + "5b5f80fd5b80516001600160a01b038116811461029e575f80fd5b919050565b", + "634e487b7160e01b5f52604160045260245ffd5b5f805f606084860312156102", + "c9575f80fd5b6102d284610288565b92506102e060208501610288565b604085", + "01519092506001600160401b03808211156102fc575f80fd5b81860191508660", + "1f83011261030f575f80fd5b815181811115610321576103216102a3565b6040", + "51601f8201601f19908116603f01168101908382118183101715610349576103", + "496102a3565b81604052828152896020848701011115610361575f80fd5b8260", + "208601602083015e5f6020848301015280955050505050509250925092565b5f", + "82518060208501845e5f920191825250919050565b610681806103a55f395ff3", + "fe60806040523661001357610011610017565b005b6100115b61001f6100ee56", + "5b6001600160a01b031633036100e45760606001600160e01b03195f35166364", + "d3180d60e11b810161005957610052610120565b91506100dc565b63587086bd", + "60e11b6001600160e01b031982160161007957610052610173565b63070d7c69", + "60e41b6001600160e01b0319821601610099576100526101b7565b621eb96f60", + "e61b6001600160e01b03198216016100b8576100526101e7565b63a39f25e560", + "e01b6001600160e01b03198216016100d857610052610226565b5f80fd5b8151", + "60208301f35b6100ec610239565b565b5f7fb53127684a568b3173ae13b9f8a6", + "016e243e63b6e8ee1178d6a717850b5d61035b546001600160a01b0316919050", + "565b606061012a610249565b5f61013836600481846104e3565b810190610145", + "9190610525565b90506101608160405180602001604052805f8152505f610253", + "565b505060408051602081019091525f815290565b60605f8061018436600481", + "846104e3565b8101906101919190610552565b915091506101a1828260016102", + "53565b60405180602001604052805f8152509250505090565b60606101c16102", + "49565b5f6101cf36600481846104e3565b8101906101dc9190610525565b9050", + "6101608161027e565b60606101f1610249565b5f6101fa6100ee565b60408051", + "6001600160a01b03831660208201529192500160405160208183030381529060", + "405291505090565b6060610230610249565b5f6101fa6102d5565b6100ec6102", + "446102d5565b6102e3565b34156100ec575f80fd5b61025c83610301565b5f82", + "5111806102685750805b15610279576102778383610340565b505b505050565b", + "7f7e644d79422f17c01e4894b5f4f588d331ebfa28653d42ae832dc59e38c979", + "8f6102a76100ee565b604080516001600160a01b039283168152918416602083", + "01520160405180910390a16102d28161036c565b50565b5f6102de6103c2565b", + "905090565b365f80375f80365f845af43d5f803e8080156102fd573d5ff35b3d", + "5ffd5b61030a816103e9565b6040516001600160a01b038216907fbc7cd75a20", + "ee27fd9adebab32041f755214dbc6bffa90cc0225b39da2e5c2d3b905f90a250", + "565b606061036583836040518060600160405280602781526020016106256027", + "9139610423565b9392505050565b6001600160a01b03811661037e575f80fd5b", + "807fb53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d", + "61035b80546001600160a01b0319166001600160a01b03929092169190911790", + "5550565b5f7f360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3", + "ca505d382bbc610111565b6001600160a01b0381163b6103fc575f80fd5b807f", + "360894a13ba1a3210667c828492db98dca3e2076cc3735a920a3ca505d382bbc", + "6103a1565b60605f80856001600160a01b03168560405161043f919061060e56", + "5b5f60405180830381855af49150503d805f8114610477576040519150601f19", + "603f3d011682016040523d82523d5f602084013e61047c565b606091505b5091", + "50915061048d86838387610497565b9695505050505050565b606083156104c1", + "5782515f036104ba576001600160a01b0385163b6104ba575f80fd5b50816104", + "cb565b6104cb83836104d3565b949350505050565b8151156100d85781518083", + "602001fd5b5f80858511156104f1575f80fd5b838611156104fd575f80fd5b50", + "50820193919092039150565b80356001600160a01b0381168114610520575f80", + "fd5b919050565b5f60208284031215610535575f80fd5b6103658261050a565b", + "634e487b7160e01b5f52604160045260245ffd5b5f8060408385031215610563", + "575f80fd5b61056c8361050a565b9150602083013567ffffffffffffffff8082", + "1115610588575f80fd5b818501915085601f83011261059b575f80fd5b813581", + "8111156105ad576105ad61053e565b604051601f8201601f19908116603f0116", + "81019083821181831017156105d5576105d561053e565b816040528281528860", + "208487010111156105ed575f80fd5b826020860160208301375f602084830101", + "5280955050505050509250929050565b5f82518060208501845e5f9201918252", + "5091905056fe416464726573733a206c6f772d6c6576656c2064656c65676174", + "652063616c6c206661696c6564a26469706673582212207adc130af683e5732b", + "81908080a45f6134fe389cd5cd3dde740f5a5249cbeeaf64736f6c6343000819", + "0033b53127684a568b3173ae13b9f8a6016e243e63b6e8ee1178d6a717850b5d", + "6103416464726573733a206c6f772d6c6576656c2064656c6567617465206361", + "6c6c206661696c656400000000000000000000000087581e1540ae45056c0ed5", + "9530c783a6e0a9c53e000000000000000000000000fba759bcd1a99a7724c506", + "8feddb4f5b844b941a0000000000000000000000000000000000000000000000", + "0000000000000000600000000000000000000000000000000000000000000000", + "00000000000000016497bce8d000000000000000000000000000000000000000", + "0000000000000000000000012000000000000000000000000000000000000000", + "0000000000000000000000000700000000000000000000000000000000000000", + "00000000001bc16d674ec8000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00000000000000000c206898d500000000000000000000000000000000000000", + "0000000000016345785d8a000000000000000000000000000000000000000000", + "000000000000b1a2bc2ec5000000000000000000000000000000000000000000", + "0000000000001057acf5f7800000000000000000000000000000000000000000", + "0000000000058d15e17628000000000000000000000000000000000000000000", + "0000000000000000000000000455534443000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000" +] \ No newline at end of file