From cd5e75ddde66113db0ca95252aa84aa3e631357e Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:21:15 -0600 Subject: [PATCH 1/4] logs for macro (#243) --- .github/workflows/dbt_run_operation_reorg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_operation_reorg.yml b/.github/workflows/dbt_run_operation_reorg.yml index 024350a..3d7a013 100644 --- a/.github/workflows/dbt_run_operation_reorg.yml +++ b/.github/workflows/dbt_run_operation_reorg.yml @@ -49,5 +49,5 @@ jobs: - name: Execute block_reorg macro run: | - dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" + dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && grep -E "SQL status|/* {" logs/dbt.log \ No newline at end of file From 57c3e3ca126654d67f6d82d18246ce539044a87e Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Fri, 6 Oct 2023 11:39:45 -0400 Subject: [PATCH 2/4] lookback (#245) --- macros/streamline/models.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql index 4fe0e64..3b99ca9 100644 --- a/macros/streamline/models.sql +++ b/macros/streamline/models.sql @@ -12,7 +12,7 @@ FROM TABLE( information_schema.external_table_file_registration_history( - start_time => DATEADD('day', -7, CURRENT_TIMESTAMP()), + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), table_name => '{{ source( "bronze_streamline", model) }}') ) A ) From 7e6456851886fc15c78a21f99bbb561a48c8da32 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Fri, 6 Oct 2023 15:39:23 -0400 Subject: [PATCH 3/4] retry (#246) --- models/streamline/silver/core/retry/_unconfirmed_blocks.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/streamline/silver/core/retry/_unconfirmed_blocks.sql b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql index a6f6a02..8e2507c 100644 --- a/models/streamline/silver/core/retry/_unconfirmed_blocks.sql +++ b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql @@ -29,4 +29,6 @@ WHERE lookback ) AND cb._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) - AND txs._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + AND ( + txs._inserted_timestamp >= DATEADD('hour', -84, SYSDATE()) + OR txs._inserted_timestamp IS NULL) From e02a4d74cca6896f1f7661c2c70e14767dd67c33 Mon Sep 17 00:00:00 2001 From: Austin <93135983+austinFlipside@users.noreply.github.com> Date: Tue, 10 Oct 2023 08:23:13 -0400 Subject: [PATCH 4/4] fix (#247) --- models/silver/core/silver__contracts.sql | 126 ++++++++++++----------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/models/silver/core/silver__contracts.sql b/models/silver/core/silver__contracts.sql index d25a11d..213d50e 100644 --- a/models/silver/core/silver__contracts.sql +++ b/models/silver/core/silver__contracts.sql @@ -36,64 +36,68 @@ token_names AS ( function_signature, read_output, utils.udf_hex_to_string( - SUBSTR(read_output,(64*2+3),LEN(read_output))) AS token_name - FROM - base_metadata - WHERE - function_signature = '0x06fdde03' - AND token_name IS NOT NULL -), -token_symbols AS ( - SELECT - contract_address, - block_number, - function_signature, - read_output, - utils.udf_hex_to_string( - SUBSTR(read_output,(64*2+3),LEN(read_output))) AS token_symbol - FROM - base_metadata - WHERE - function_signature = '0x95d89b41' - AND token_symbol IS NOT NULL -), -token_decimals AS ( - SELECT - contract_address, - utils.udf_hex_to_int( - read_output :: STRING - ) AS token_decimals, - LENGTH(token_decimals) AS dec_length - FROM - base_metadata - WHERE - function_signature = '0x313ce567' - AND read_output IS NOT NULL - AND read_output <> '0x' -), -contracts AS ( - SELECT - contract_address, - MAX(_inserted_timestamp) AS _inserted_timestamp - FROM - base_metadata - GROUP BY - 1 -) -SELECT - c1.contract_address :: STRING AS contract_address, - token_name, - TRY_TO_NUMBER(token_decimals) AS token_decimals, - token_symbol, - _inserted_timestamp -FROM - contracts c1 - LEFT JOIN token_names - ON c1.contract_address = token_names.contract_address - LEFT JOIN token_symbols - ON c1.contract_address = token_symbols.contract_address - LEFT JOIN token_decimals - ON c1.contract_address = token_decimals.contract_address - AND dec_length < 3 qualify(ROW_NUMBER() over(PARTITION BY c1.contract_address -ORDER BY - _inserted_timestamp DESC)) = 1 + SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_name + FROM + base_metadata + WHERE + function_signature = '0x06fdde03' + AND token_name IS NOT NULL + ), + token_symbols AS ( + SELECT + contract_address, + block_number, + function_signature, + read_output, + utils.udf_hex_to_string( + SUBSTR(read_output,(64 * 2 + 3), len(read_output))) AS token_symbol + FROM + base_metadata + WHERE + function_signature = '0x95d89b41' + AND token_symbol IS NOT NULL + ), + token_decimals AS ( + SELECT + contract_address, + CASE + WHEN LENGTH( + read_output :: STRING + ) <= 4300 + AND read_output IS NOT NULL THEN utils.udf_hex_to_int(LEFT(read_output :: STRING, 66)) + ELSE NULL + END AS token_decimals, + LENGTH(token_decimals) AS dec_length + FROM + base_metadata + WHERE + function_signature = '0x313ce567' + AND read_output IS NOT NULL + AND read_output <> '0x' + ), + contracts AS ( + SELECT + contract_address, + MAX(_inserted_timestamp) AS _inserted_timestamp + FROM + base_metadata + GROUP BY + 1 + ) + SELECT + c1.contract_address :: STRING AS contract_address, + token_name, + TRY_TO_NUMBER(token_decimals) AS token_decimals, + token_symbol, + _inserted_timestamp + FROM + contracts c1 + LEFT JOIN token_names + ON c1.contract_address = token_names.contract_address + LEFT JOIN token_symbols + ON c1.contract_address = token_symbols.contract_address + LEFT JOIN token_decimals + ON c1.contract_address = token_decimals.contract_address + AND dec_length < 3 qualify(ROW_NUMBER() over(PARTITION BY c1.contract_address + ORDER BY + _inserted_timestamp DESC)) = 1