From fee0084c80688de5e4bfaeabdcfa26f09ee69edd Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 3 Mar 2025 21:13:12 +0800 Subject: [PATCH 01/16] change request --- models/bronze/api_udf/bronze_api__contract_abis.sql | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/models/bronze/api_udf/bronze_api__contract_abis.sql b/models/bronze/api_udf/bronze_api__contract_abis.sql index 3069832..b94c104 100644 --- a/models/bronze/api_udf/bronze_api__contract_abis.sql +++ b/models/bronze/api_udf/bronze_api__contract_abis.sql @@ -47,7 +47,16 @@ row_nos AS ( batched AS ({% for item in range(150) %} SELECT rn.contract_address, - live.udf_api(concat('https://api-cypress.klaytnscope.com/v2/accounts/',contract_address)) as abi_data, + live.udf_api( + 'GET', + concat('https://mainnet-oapi.kaiascan.io/api/v1/contracts/',rn.contract_address, '/abi?apikey={key}'), + OBJECT_CONSTRUCT( + 'Content-Type', 'application/json', + 'fsc-quantum-state', 'livequery' + ), + NULL, + 'Vault/prod/block_explorers/kaia_scan' + ) as abi_data, SYSDATE() AS _inserted_timestamp FROM row_nos rn From 90321fa96ae947a04b2b82e6ef6732bc8ff05464 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 4 Mar 2025 10:27:41 +0800 Subject: [PATCH 02/16] update url --- models/bronze/api_udf/bronze_api__contract_abis.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/bronze/api_udf/bronze_api__contract_abis.sql b/models/bronze/api_udf/bronze_api__contract_abis.sql index b94c104..7270b9e 100644 --- a/models/bronze/api_udf/bronze_api__contract_abis.sql +++ b/models/bronze/api_udf/bronze_api__contract_abis.sql @@ -49,7 +49,7 @@ SELECT rn.contract_address, live.udf_api( 'GET', - concat('https://mainnet-oapi.kaiascan.io/api/v1/contracts/',rn.contract_address, '/abi?apikey={key}'), + CONCAT('https://mainnet-oapi.kaiascan.io/api?module=contract&action=getabi&address=', rn.contract_address, '&apikey={key}'), OBJECT_CONSTRUCT( 'Content-Type', 'application/json', 'fsc-quantum-state', 'livequery' From e9fbddbddf27d0f751ca0fdc340cd3f3694e227f Mon Sep 17 00:00:00 2001 From: mattromano Date: Fri, 7 Mar 2025 13:26:41 -0800 Subject: [PATCH 03/16] add backfill logic --- models/silver/core/silver__traces2.sql | 67 +++++++++++++++++--------- models/sources.yml | 5 ++ 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index d6c0aee..9a0eeba 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -9,6 +9,8 @@ ) }} WITH bronze_traces AS ( + + {% if is_incremental() and not var('full_reload_mode') %} SELECT block_number, partition_key, @@ -16,8 +18,6 @@ WITH bronze_traces AS ( DATA :result AS full_traces, _inserted_timestamp FROM - - {% if is_incremental() and not full_reload_mode %} {{ ref('bronze__streamline_traces') }} WHERE _inserted_timestamp >= ( @@ -27,30 +27,53 @@ WITH bronze_traces AS ( {{ this }} ) AND DATA :result IS NOT NULL + and block_number > 160000000 + and partition_key > 160000000 - {% elif is_incremental() and full_reload_mode %} - {{ ref('bronze__streamline_fr_traces') }} + {% elif is_incremental() and var('full_reload_mode') and not var('initial_load') %} + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + partition_key, + VALUE :array_index :: INT AS tx_position, + DATA :result AS full_traces, + _inserted_timestamp + FROM + {{ source('klaytn_bronze', 'streamline_fr_traces') }} WHERE - partition_key BETWEEN ( - SELECT - MAX(partition_key) - 100000 - FROM - {{ this }} - ) - AND ( - SELECT - MAX(partition_key) + 5000000 - FROM - {{ this }} - ) + partition_key BETWEEN ( + SELECT + MAX(partition_key) + FROM + {{ this }} + WHERE + partition_key <= 74000000 + AND _inserted_timestamp >= '2024-10-01' + ) - 1000000 + AND ( + SELECT + MAX(partition_key) + FROM + {{ this }} + WHERE + partition_key <= 74000000 + AND _inserted_timestamp >= '2024-10-01' + ) + 5000000 + AND DATA :result IS NOT NULL {% else %} - {{ ref('bronze__streamline_fr_traces') }} - WHERE partition_key <= 149500000 - {% endif %} - - and block_number > 160000000 - and partition_key > 160000000 + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + partition_key, + VALUE :array_index :: INT AS tx_position, + DATA :result AS full_traces, + _inserted_timestamp + FROM + {{ source('klaytn_bronze', 'streamline_fr_traces') }} + WHERE + block_number BETWEEN 0 AND 5000000 + AND DATA :result IS NOT NULL + AND partition_key <= 5000000 + {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position ORDER BY diff --git a/models/sources.yml b/models/sources.yml index c969612..4125cbd 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,6 +1,11 @@ version: 2 sources: + - name: klaytn_bronze + database: klaytn + schema: bronze + tables: + - name: streamline_fr_traces - name: klaytn_silver database: klaytn schema: silver From 9e9c2433b927bb37c37a7e4c5bb5389ab59f06e8 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 09:59:46 -0700 Subject: [PATCH 04/16] update trace models with load logic --- models/silver/core/silver__fact_traces2.sql | 30 +++++++++++---------- models/silver/core/silver__traces2.sql | 26 ++++++++++++------ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index da2cd39..7efca29 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -23,27 +23,29 @@ WITH silver_traces AS ( {{ ref('silver__traces2') }} WHERE 1 = 1 - AND block_number > 160000000 - {% if is_incremental() and not full_reload_mode %} + {% if is_incremental() and not var('full_reload_mode') %} + AND block_number > 160000000 AND modified_timestamp > ( SELECT MAX(modified_timestamp) FROM {{ this }} + WHERE + block_number > 160000000 ) - {% elif is_incremental() and full_reload_mode %} - AND block_number BETWEEN ( - SELECT - MAX(block_number) - FROM - {{ this }} - ) - AND ( - SELECT - MAX(block_number) + 5000000 - FROM - {{ this }} + {% elif is_incremental() and var('full_reload_mode') %} + AND block_number < 80000000 + AND modified_timestamp > coalesce( + ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + WHERE + block_number < 80000000 + ), + '2024-01-01' ) {% else %} AND block_number <= 149500000 diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 9a0eeba..035e741 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -46,8 +46,7 @@ WITH bronze_traces AS ( FROM {{ this }} WHERE - partition_key <= 74000000 - AND _inserted_timestamp >= '2024-10-01' + partition_key < 80000000 ) - 1000000 AND ( SELECT @@ -55,11 +54,23 @@ WITH bronze_traces AS ( FROM {{ this }} WHERE - partition_key <= 74000000 - AND _inserted_timestamp >= '2024-10-01' - ) + 5000000 + partition_key < 80000000 + ) + 4000000 AND DATA :result IS NOT NULL + {% elif var('initial_load') %} + SELECT + VALUE :BLOCK_NUMBER :: INT AS block_number, + partition_key, + VALUE :array_index :: INT AS tx_position, + DATA :result AS full_traces, + _inserted_timestamp + FROM + {{ source('klaytn_bronze', 'streamline_fr_traces') }} + WHERE + DATA :result IS NOT NULL + AND partition_key BETWEEN 0 AND 5000000 + {% else %} SELECT VALUE :BLOCK_NUMBER :: INT AS block_number, @@ -70,9 +81,8 @@ WITH bronze_traces AS ( FROM {{ source('klaytn_bronze', 'streamline_fr_traces') }} WHERE - block_number BETWEEN 0 AND 5000000 - AND DATA :result IS NOT NULL - AND partition_key <= 5000000 + DATA :result IS NOT NULL + AND block_number < 149000000 {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position From 9020b360874a91a9ebc762c5276e2f3dd9dc793e Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 10:27:50 -0700 Subject: [PATCH 05/16] update var config and add workflow --- .github/workflows/dbt_run_trace_backfill.yml | 50 ++++++++++++++++++++ models/silver/core/silver__fact_traces2.sql | 4 +- models/silver/core/silver__traces2.sql | 4 +- 3 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dbt_run_trace_backfill.yml diff --git a/.github/workflows/dbt_run_trace_backfill.yml b/.github/workflows/dbt_run_trace_backfill.yml new file mode 100644 index 0000000..f0d202f --- /dev/null +++ b/.github/workflows/dbt_run_trace_backfill.yml @@ -0,0 +1,50 @@ +name: dbt_run_trace_backfill +run-name: dbt_run_trace_backfill + +on: + workflow_dispatch: + schedule: + # Runs “At 17:00 on day-of-month 1.” (see https://crontab.guru) + - cron: '50 * * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_2xl + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 1 --vars '{"full_reload_mode": true}' -m silver__traces2 silver__fact_traces2 + + + diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index 7efca29..59204cf 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -24,7 +24,7 @@ WITH silver_traces AS ( WHERE 1 = 1 - {% if is_incremental() and not var('full_reload_mode') %} + {% if is_incremental() and not var('full_reload_mode', false) %} AND block_number > 160000000 AND modified_timestamp > ( SELECT @@ -34,7 +34,7 @@ WITH silver_traces AS ( WHERE block_number > 160000000 ) - {% elif is_incremental() and var('full_reload_mode') %} + {% elif is_incremental() and var('full_reload_mode', false) %} AND block_number < 80000000 AND modified_timestamp > coalesce( ( diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 035e741..30d0bd0 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -30,7 +30,7 @@ WITH bronze_traces AS ( and block_number > 160000000 and partition_key > 160000000 - {% elif is_incremental() and var('full_reload_mode') and not var('initial_load') %} + {% elif is_incremental() and var('full_reload_mode', false) and not var('initial_load', false) %} SELECT VALUE :BLOCK_NUMBER :: INT AS block_number, partition_key, @@ -58,7 +58,7 @@ WITH bronze_traces AS ( ) + 4000000 AND DATA :result IS NOT NULL - {% elif var('initial_load') %} + {% elif var('initial_load', false) %} SELECT VALUE :BLOCK_NUMBER :: INT AS block_number, partition_key, From a4a706268e039951d09b5e90ea7a411ba74a5128 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 10:36:43 -0700 Subject: [PATCH 06/16] update sources --- models/silver/core/silver__fact_traces2.sql | 2 +- models/silver/core/silver__traces2.sql | 19 +++++-------------- models/sources.yml | 5 ----- 3 files changed, 6 insertions(+), 20 deletions(-) diff --git a/models/silver/core/silver__fact_traces2.sql b/models/silver/core/silver__fact_traces2.sql index 59204cf..c196b32 100644 --- a/models/silver/core/silver__fact_traces2.sql +++ b/models/silver/core/silver__fact_traces2.sql @@ -36,7 +36,7 @@ WITH silver_traces AS ( ) {% elif is_incremental() and var('full_reload_mode', false) %} AND block_number < 80000000 - AND modified_timestamp > coalesce( + AND modified_timestamp > COALESCE( ( SELECT MAX(modified_timestamp) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 30d0bd0..7297ca4 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -38,7 +38,7 @@ WITH bronze_traces AS ( DATA :result AS full_traces, _inserted_timestamp FROM - {{ source('klaytn_bronze', 'streamline_fr_traces') }} + {{ ref('bronze__streamline_fr_traces') }} WHERE partition_key BETWEEN ( SELECT @@ -66,24 +66,15 @@ WITH bronze_traces AS ( DATA :result AS full_traces, _inserted_timestamp FROM - {{ source('klaytn_bronze', 'streamline_fr_traces') }} + {{ ref('bronze__streamline_fr_traces') }} WHERE DATA :result IS NOT NULL AND partition_key BETWEEN 0 AND 5000000 {% else %} - SELECT - VALUE :BLOCK_NUMBER :: INT AS block_number, - partition_key, - VALUE :array_index :: INT AS tx_position, - DATA :result AS full_traces, - _inserted_timestamp - FROM - {{ source('klaytn_bronze', 'streamline_fr_traces') }} - WHERE - DATA :result IS NOT NULL - AND block_number < 149000000 - {% endif %} + {{ ref('bronze__streamline_fr_traces') }} + WHERE partition_key <= 149500000 + {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position ORDER BY diff --git a/models/sources.yml b/models/sources.yml index 4125cbd..c969612 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -1,11 +1,6 @@ version: 2 sources: - - name: klaytn_bronze - database: klaytn - schema: bronze - tables: - - name: streamline_fr_traces - name: klaytn_silver database: klaytn schema: silver From ae03ab54c2ecb8b109a888da89fc43e6d12a2d46 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 10:49:27 -0700 Subject: [PATCH 07/16] update selects --- models/silver/core/silver__traces2.sql | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 7297ca4..e2a94b8 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -10,13 +10,13 @@ WITH bronze_traces AS ( - {% if is_incremental() and not var('full_reload_mode') %} SELECT block_number, partition_key, VALUE :array_index :: INT AS tx_position, DATA :result AS full_traces, _inserted_timestamp + {% if is_incremental() and not var('full_reload_mode', false) %} FROM {{ ref('bronze__streamline_traces') }} WHERE @@ -31,12 +31,6 @@ WITH bronze_traces AS ( and partition_key > 160000000 {% elif is_incremental() and var('full_reload_mode', false) and not var('initial_load', false) %} - SELECT - VALUE :BLOCK_NUMBER :: INT AS block_number, - partition_key, - VALUE :array_index :: INT AS tx_position, - DATA :result AS full_traces, - _inserted_timestamp FROM {{ ref('bronze__streamline_fr_traces') }} WHERE @@ -59,12 +53,6 @@ WITH bronze_traces AS ( AND DATA :result IS NOT NULL {% elif var('initial_load', false) %} - SELECT - VALUE :BLOCK_NUMBER :: INT AS block_number, - partition_key, - VALUE :array_index :: INT AS tx_position, - DATA :result AS full_traces, - _inserted_timestamp FROM {{ ref('bronze__streamline_fr_traces') }} WHERE From d6c53bed3ba4db292430e8c0f7170b4368c981c4 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 11:00:18 -0700 Subject: [PATCH 08/16] block_number increment --- models/silver/core/silver__traces2.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index e2a94b8..b5fd4ae 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -34,21 +34,21 @@ WITH bronze_traces AS ( FROM {{ ref('bronze__streamline_fr_traces') }} WHERE - partition_key BETWEEN ( + block_number BETWEEN ( SELECT - MAX(partition_key) + MAX(block_number) FROM {{ this }} WHERE - partition_key < 80000000 + block_number < 80000000 ) - 1000000 AND ( SELECT - MAX(partition_key) + MAX(block_number) FROM {{ this }} WHERE - partition_key < 80000000 + block_number < 80000000 ) + 4000000 AND DATA :result IS NOT NULL @@ -57,11 +57,11 @@ WITH bronze_traces AS ( {{ ref('bronze__streamline_fr_traces') }} WHERE DATA :result IS NOT NULL - AND partition_key BETWEEN 0 AND 5000000 + AND block_number BETWEEN 0 AND 5000000 {% else %} {{ ref('bronze__streamline_fr_traces') }} - WHERE partition_key <= 149500000 + WHERE block_number <= 149500000 {% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number, tx_position From df9b64e3e9fcc85eca6b70a751414d56006ab9f3 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 11:02:04 -0700 Subject: [PATCH 09/16] decrease lookback --- models/silver/core/silver__traces2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index b5fd4ae..e260bd5 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -41,7 +41,7 @@ WITH bronze_traces AS ( {{ this }} WHERE block_number < 80000000 - ) - 1000000 + ) - 10000 AND ( SELECT MAX(block_number) From 93a5cc90e6696ef528d00a1a8e90445dc22f6e2d Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 11:12:20 -0700 Subject: [PATCH 10/16] format from --- models/silver/core/silver__traces2.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index e260bd5..2dee038 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -16,8 +16,8 @@ WITH bronze_traces AS ( VALUE :array_index :: INT AS tx_position, DATA :result AS full_traces, _inserted_timestamp - {% if is_incremental() and not var('full_reload_mode', false) %} FROM + {% if is_incremental() and not var('full_reload_mode', false) %} {{ ref('bronze__streamline_traces') }} WHERE _inserted_timestamp >= ( @@ -31,7 +31,6 @@ WITH bronze_traces AS ( and partition_key > 160000000 {% elif is_incremental() and var('full_reload_mode', false) and not var('initial_load', false) %} - FROM {{ ref('bronze__streamline_fr_traces') }} WHERE block_number BETWEEN ( @@ -53,7 +52,6 @@ WITH bronze_traces AS ( AND DATA :result IS NOT NULL {% elif var('initial_load', false) %} - FROM {{ ref('bronze__streamline_fr_traces') }} WHERE DATA :result IS NOT NULL From 3a02a41714c22850b16739f028d745fc3dba8c88 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 11:13:50 -0700 Subject: [PATCH 11/16] cron comment --- .github/workflows/dbt_run_trace_backfill.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_trace_backfill.yml b/.github/workflows/dbt_run_trace_backfill.yml index f0d202f..3b298d4 100644 --- a/.github/workflows/dbt_run_trace_backfill.yml +++ b/.github/workflows/dbt_run_trace_backfill.yml @@ -4,7 +4,7 @@ run-name: dbt_run_trace_backfill on: workflow_dispatch: schedule: - # Runs “At 17:00 on day-of-month 1.” (see https://crontab.guru) + # Runs “At Minute 50 every hour” (see https://crontab.guru) - cron: '50 * * * *' env: From 38305319120151f8b4d3bf63930081623eeb47e5 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 13:05:56 -0700 Subject: [PATCH 12/16] update to 2M blocks --- models/silver/core/silver__traces2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 2dee038..ab961bf 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -48,7 +48,7 @@ WITH bronze_traces AS ( {{ this }} WHERE block_number < 80000000 - ) + 4000000 + ) + 2000000 AND DATA :result IS NOT NULL {% elif var('initial_load', false) %} From 8da9da6db584b2ed32f516a8fde99b40133b3985 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 15:22:31 -0700 Subject: [PATCH 13/16] update with partition key filter --- models/silver/core/silver__traces2.sql | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index ab961bf..d1c11ef 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -33,7 +33,16 @@ WITH bronze_traces AS ( {% elif is_incremental() and var('full_reload_mode', false) and not var('initial_load', false) %} {{ ref('bronze__streamline_fr_traces') }} WHERE - block_number BETWEEN ( + partition_key < ( + SELECT + MAX(block_number) + FROM + {{ this }} + WHERE + block_number < 80000000 + ) + 2500000 + AND DATA :result IS NOT NULL + AND block_number BETWEEN ( SELECT MAX(block_number) FROM @@ -49,13 +58,13 @@ WITH bronze_traces AS ( WHERE block_number < 80000000 ) + 2000000 - AND DATA :result IS NOT NULL {% elif var('initial_load', false) %} {{ ref('bronze__streamline_fr_traces') }} WHERE DATA :result IS NOT NULL AND block_number BETWEEN 0 AND 5000000 + and partition_key < 5500000 {% else %} {{ ref('bronze__streamline_fr_traces') }} From 0bb3ee69f34188e029034e30af634819b48b9fb2 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 15:29:03 -0700 Subject: [PATCH 14/16] decrease part increment --- models/silver/core/silver__traces2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index d1c11ef..7d9c37a 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -40,7 +40,7 @@ WITH bronze_traces AS ( {{ this }} WHERE block_number < 80000000 - ) + 2500000 + ) + 2100000 AND DATA :result IS NOT NULL AND block_number BETWEEN ( SELECT From d85e46373abb387fc4cadf4ad21b994b042b6478 Mon Sep 17 00:00:00 2001 From: mattromano Date: Tue, 11 Mar 2025 16:12:56 -0700 Subject: [PATCH 15/16] add partition key --- models/silver/core/silver__traces2.sql | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 7d9c37a..9bc26d3 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -33,18 +33,10 @@ WITH bronze_traces AS ( {% elif is_incremental() and var('full_reload_mode', false) and not var('initial_load', false) %} {{ ref('bronze__streamline_fr_traces') }} WHERE - partition_key < ( + DATA :result IS NOT NULL + AND partition_key BETWEEN ( SELECT - MAX(block_number) - FROM - {{ this }} - WHERE - block_number < 80000000 - ) + 2100000 - AND DATA :result IS NOT NULL - AND block_number BETWEEN ( - SELECT - MAX(block_number) + ROUND(MAX(block_number),-3) FROM {{ this }} WHERE @@ -52,7 +44,7 @@ WITH bronze_traces AS ( ) - 10000 AND ( SELECT - MAX(block_number) + ROUND(MAX(block_number),-3) FROM {{ this }} WHERE From 3cb0eaa5300aaf58605cf952881a504ef36bff75 Mon Sep 17 00:00:00 2001 From: mattromano Date: Wed, 12 Mar 2025 07:23:43 -0700 Subject: [PATCH 16/16] increase partition lookback --- models/silver/core/silver__traces2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/silver/core/silver__traces2.sql b/models/silver/core/silver__traces2.sql index 9bc26d3..eb32be2 100644 --- a/models/silver/core/silver__traces2.sql +++ b/models/silver/core/silver__traces2.sql @@ -41,7 +41,7 @@ WITH bronze_traces AS ( {{ this }} WHERE block_number < 80000000 - ) - 10000 + ) - 100000 AND ( SELECT ROUND(MAX(block_number),-3)