From 03a23a79d173bd9ca12429c5bfee79344f3264c4 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:02:33 -0800 Subject: [PATCH] Streamline realtime view optimizations (#423) --- analysis/udf_transform_test.py | 143 ++++++++++++++++++ dbt_project.yml | 3 +- .../streamline__contract_abis_realtime.sql | 31 ++-- .../streamline/streamline__reads_realtime.sql | 10 +- 4 files changed, 158 insertions(+), 29 deletions(-) create mode 100644 analysis/udf_transform_test.py diff --git a/analysis/udf_transform_test.py b/analysis/udf_transform_test.py new file mode 100644 index 00000000..ed6b7ce5 --- /dev/null +++ b/analysis/udf_transform_test.py @@ -0,0 +1,143 @@ +# %% +from textwrap import indent +import simplejson as json +from pprint import pprint +from copy import deepcopy + +decoded_data = { + "address": "0x00000000006c3852cbEf3e08E8dF289169EdE581", + "data": [ + { + "decoded": True, + "name": "orderHash", + "type": "bytes32", + "value": "0x71c2d239d193ead00563eb4c1aa2a0b67b472bd1ec07e30e1cc5305847f6ed71", + }, + { + "decoded": True, + "name": "offerer", + "type": "address", + "value": "0x412426adf9dc080cc4e2d0d90f12166ef9d75d84", + }, + { + "decoded": True, + "name": "zone", + "type": "address", + "value": "0x004c00500000ad104d7dbd00e3ae0a5c00560c00", + }, + { + "decoded": True, + "name": "recipient", + "type": "address", + "value": "0x22034382da3f3e76ed455f96faf186e84e1541a0", + }, + { + "components": [ + {"internalType": "enum ItemType", "name": "itemType", "type": "uint8"}, + {"internalType": "address", "name": "token", "type": "address"}, + {"internalType": "uint256", "name": "identifier", "type": "uint256"}, + {"internalType": "uint256", "name": "amount", "type": "uint256"}, + ], + "decoded": True, + "name": "offer", + "type": "tuple[]", + "value": [[2, "0xa35aa193f94a90eca0ae2a3fb5616e53c1f35193", 8272, 1]], + }, + { + "components": [ + {"internalType": "enum ItemType", "name": "itemType", "type": "uint8"}, + {"internalType": "address", "name": "token", "type": "address"}, + {"internalType": "uint256", "name": "identifier", "type": "uint256"}, + {"internalType": "uint256", "name": "amount", "type": "uint256"}, + { + "internalType": "address payable", + "name": "recipient", + "type": "address", + }, + ], + "decoded": True, + "name": "consideration", + "type": "tuple[]", + "value": [ + [ + 0, + "0x0000000000000000000000000000000000000000", + 0, + "23493000000000000", + "0x412426adf9dc080cc4e2d0d90f12166ef9d75d84", + ], + [ + 0, + "0x0000000000000000000000000000000000000000", + 0, + 615000000000000, + "0x0000a26b00c1f0df003000390027140000faa719", + ], + [ + 0, + "0x0000000000000000000000000000000000000000", + 0, + 492000000000000, + "0xe718904d67cba3c65e4895fb27247b0fd96d733a", + ], + ], + }, + ], + "decoded": True, + "name": "OrderFulfilled", +} +decoded_data2 = { + "address": "0xA35aa193f94A90eca0AE2a3fB5616E53C1F35193", + "data": [ + { + "decoded": True, + "name": "from", + "type": "address", + "value": "0x412426adf9dc080cc4e2d0d90f12166ef9d75d84", + }, + { + "decoded": True, + "name": "to", + "type": "address", + "value": "0x22034382da3f3e76ed455f96faf186e84e1541a0", + }, + {"decoded": True, "name": "tokenId", "type": "uint256", "value": 8272}, + ], + "decoded": True, + "name": "Transfer", +} + + +def transform_event(event: dict): + new_event = deepcopy(event) + if new_event.get("components"): + components = new_event.get("components") + for iy, y in enumerate(new_event["value"]): + for i, c in enumerate(components): + y[i] = {"value": y[i], **c} + new_event["value"][iy] = {z["name"]: z["value"] for z in y} + return new_event + else: + return event + + +def transform(events: list): + try: + results = [ + transform_event(event) if event["decoded"] else event + for event in events["data"] + ] + events["data"] = results + return events + except: + return events + + +# %% + +# if __name__ == "__main__": +results = transform(decoded_data) + +print(json.dumps(results, indent=4)) + +# %% diff --git a/dbt_project.yml b/dbt_project.yml index 5d5b3ad0..802fc6b7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -38,7 +38,7 @@ on-run-start: - "{{ create_udfs() }}" on-run-end: - - '{{ apply_meta_as_tags(results) }}' + - "{{ apply_meta_as_tags(results) }}" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models @@ -53,3 +53,4 @@ vars: STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False UPDATE_UDFS_AND_SPS: False UPDATE_SNOWFLAKE_TAGS: True + WAIT: 0 diff --git a/models/silver/streamline/streamline__contract_abis_realtime.sql b/models/silver/streamline/streamline__contract_abis_realtime.sql index ca4c3eed..88762fec 100644 --- a/models/silver/streamline/streamline__contract_abis_realtime.sql +++ b/models/silver/streamline/streamline__contract_abis_realtime.sql @@ -5,27 +5,20 @@ target = "{{this.schema}}.{{this.identifier}}" ) ) }} -( - SELECT - contract_address, - block_number - FROM - {{ ref("streamline__contract_addresses") }} - WHERE - block_number > 15000000 - EXCEPT - SELECT - contract_address, - block_number - FROM - {{ ref("streamline__complete_contract_abis") }} - WHERE - block_number > 15000000 -) -UNION ALL + SELECT contract_address, block_number FROM - {{ ref("streamline__contract_abis_history") }} + {{ ref("streamline__contract_addresses") }} +WHERE + block_number > 15000000 +EXCEPT +SELECT + contract_address, + block_number +FROM + {{ ref("streamline__complete_contract_abis") }} +WHERE + block_number > 15000000 diff --git a/models/silver/streamline/streamline__reads_realtime.sql b/models/silver/streamline/streamline__reads_realtime.sql index 7efaaae6..37146427 100644 --- a/models/silver/streamline/streamline__reads_realtime.sql +++ b/models/silver/streamline/streamline__reads_realtime.sql @@ -53,12 +53,4 @@ WHERE last_3_days ) {# TODO: OR can be removed once historical load is complete #} OR block_number > 15000000 -UNION ALL -SELECT - contract_address, - function_signature, - call_name, - function_input, - block_number -FROM - {{ ref("streamline__reads_history") }} +