flip/near-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/near-models.git synced 2026-02-06 14:11:54 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

incr fixes

Browse Source
This commit is contained in:
forgash_ 2024-05-01 23:46:37 -06:00
parent 061a161271
commit 2223ae077d
26 changed files with 90 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
macros/autosync/get_s3_filenames.sql
View File

@ -1,56 +0,0 @@
{% macro get_s3_filenames() %}
{% set create_table %}
CREATE schema if NOT EXISTS {{ target.database }}.bronze;
CREATE TABLE if NOT EXISTS {{ target.database }}.bronze.s3_filenames(
NAME VARCHAR,
SIZE NUMBER,
MD5 VARCHAR,
last_modified VARCHAR
);
{% endset %}
{% do run_query(create_table) %}
{% set get_block %}
SELECT
LPAD(
COALESCE(MAX(SPLIT(NAME, '/') [3] :: NUMBER), 9820210) :: STRING,
12,
'0'
) AS block_id,
FLOOR(
COALESCE(MAX(SPLIT(NAME, '/') [3] :: NUMBER), 9820210),
-4
) AS _partition_by_block_number
FROM
{{ target.database }}.bronze.s3_filenames;
{% endset %}
{% if execute %}
{% set max_block_id_indexed = run_query(get_block).columns [0].values() [0] %}
{% endif %}
{% set block_range_start_prefix = max_block_id_indexed [:-5] %}
{% set load_filenames %}
list @STREAMLINE.BRONZE.STG_US_EAST_1_SERVERLESS_NEAR_LAKE_MAINNET_FSC/{{ block_range_start_prefix }};
MERGE INTO {{ target.database }}.bronze.s3_filenames tgt USING (
SELECT
*
FROM
TABLE(RESULT_SCAN(LAST_QUERY_ID()))) src
ON tgt.name = src."name"
WHEN NOT matched THEN
INSERT
(
NAME,
SIZE,
MD5,
last_modified
)
VALUES
(
src."name",
src."size",
src."md5",
src."last_modified"
);
{% endset %}
{% do run_query(load_filenames) %}
{% endmacro %}

5
models/silver/actions/silver__actions_events_function_call_s3.sql
View File

@ -28,15 +28,18 @@ WITH action_events AS (
{{ ref('silver__actions_events_s3') }}
WHERE
action_name = 'FunctionCall'
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
WHERE _modified_timestamp >= (
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
FROM
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (

2
models/silver/actions/silver__actions_events_s3.sql
View File

@ -32,6 +32,7 @@ WITH receipts AS (
WHERE
{{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -39,6 +40,7 @@ WITH receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
),
flatten_actions AS (
SELECT

8
models/silver/curated/defi/bridge/silver__bridge_rainbow.sql
View File

@ -63,7 +63,7 @@ outbound_near_to_aurora AS (
'outbound' AS direction,
_inserted_timestamp,
_partition_by_block_number,
modified_timestamp AS _modified_timestamp
_modified_timestamp
FROM
functioncall
WHERE
@ -95,7 +95,7 @@ inbound_aurora_to_near AS (
_inserted_timestamp,
_partition_by_block_number,
args,
modified_timestamp AS _modified_timestamp
_modified_timestamp
FROM
functioncall
WHERE
@ -187,7 +187,7 @@ outbound_near_to_eth AS (
'outbound' AS direction,
_inserted_timestamp,
_partition_by_block_number,
modified_timestamp AS _modified_timestamp
_modified_timestamp
FROM
functioncall
WHERE
@ -224,7 +224,7 @@ inbound_eth_to_near AS (
booland_agg(receipt_succeeded) AS receipt_succeeded,
MIN(_inserted_timestamp) AS _inserted_timestamp,
MIN(_partition_by_block_number) AS _partition_by_block_number,
MIN(modified_timestamp) AS _modified_timestamp
MIN(_modified_timestamp) AS _modified_timestamp
FROM
functioncall
WHERE

2
models/silver/curated/defi/bridge/silver__bridge_wormhole.sql
View File

@ -31,6 +31,7 @@ WITH functioncall AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -38,6 +39,7 @@ WITH functioncall AS (
{{ this }}
)
{% endif %}
{% endif %}
),
outbound_near AS (
-- burn

2
models/silver/curated/defi/lending/burrow/silver__burrow_borrows.sql
View File

@ -32,6 +32,7 @@ borrows AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -39,6 +40,7 @@ borrows AS (
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (

6
models/silver/curated/defi/lending/burrow/silver__burrow_collaterals.sql
View File

@ -30,8 +30,11 @@ actions AS (
AND receipt_succeeded = TRUE
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
AND
{{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -39,6 +42,7 @@ actions AS (
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (
SELECT

4
models/silver/curated/defi/lending/burrow/silver__burrow_deposits.sql
View File

@ -33,6 +33,9 @@ deposits AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -40,6 +43,7 @@ deposits AS (
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (
SELECT

4
models/silver/curated/defi/lending/burrow/silver__burrow_repays.sql
View File

@ -28,10 +28,11 @@ actions AS (
WHERE
receiver_id = 'contract.main.burrow.near'
AND receipt_succeeded = TRUE
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -39,6 +40,7 @@ actions AS (
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (
SELECT

2
models/silver/curated/defi/lending/burrow/silver__burrow_withdraws.sql
View File

@ -33,6 +33,7 @@ withdraws AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -40,6 +41,7 @@ withdraws AS (
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (
SELECT

4
models/silver/curated/defi/silver__dex_swaps_v2.sql
View File

@ -30,6 +30,7 @@ WITH swap_logs AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -37,6 +38,7 @@ WITH swap_logs AS (
{{ this }}
)
{% endif %}
{% endif %}
),
receipts AS (
SELECT
@ -59,6 +61,7 @@ receipts AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -66,6 +69,7 @@ receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
),
swap_outcome AS (
SELECT

2
models/silver/curated/nft/silver__nft_transfers.sql
View File

@ -28,6 +28,7 @@ WITH actions_events AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -35,6 +36,7 @@ WITH actions_events AS (
{{ this }}
)
{% endif %}
{% endif %}
),
-------------------------------- NFT Transfers --------------------------------

6
models/silver/curated/nft/silver__standard_nft_mint_s3.sql
View File

@ -28,6 +28,7 @@ WITH logs AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -35,6 +36,7 @@ WITH logs AS (
{{ this }}
)
{% endif %}
{% endif %}
),
tx AS (
SELECT
@ -50,6 +52,7 @@ tx AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -57,6 +60,7 @@ tx AS (
{{ this }}
)
{% endif %}
{% endif %}
),
function_call AS (
SELECT
@ -71,6 +75,7 @@ function_call AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -78,6 +83,7 @@ function_call AS (
{{ this }}
)
{% endif %}
{% endif %}
),
standard_logs AS (
SELECT

6
models/silver/curated/silver__lockup_actions.sql
View File

@ -19,6 +19,7 @@ WITH txs AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -26,6 +27,7 @@ WITH txs AS (
{{ this }}
)
{% endif %}
{% endif %}
),
function_calls AS (
SELECT
@ -47,6 +49,7 @@ function_calls AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -54,6 +57,7 @@ function_calls AS (
{{ this }}
)
{% endif %}
{% endif %}
),
xfers AS (
SELECT
@ -71,6 +75,7 @@ xfers AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -78,6 +83,7 @@ xfers AS (
{{ this }}
)
{% endif %}
{% endif %}
),
lockup_actions AS (
SELECT

6
models/silver/curated/silver__logs_s3.sql
View File

@ -27,15 +27,17 @@ WITH receipts AS (
{{ ref('silver__streamline_receipts_final') }}
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
AND _modified_timestamp >= (
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
FROM
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (
SELECT

2
models/silver/curated/silver__prices_oracle_s3.sql
View File

@ -32,6 +32,7 @@ events_function_call AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -39,6 +40,7 @@ events_function_call AS (
{{ this }}
)
{% endif %}
{% endif %}
),
prices AS (
SELECT

6
models/silver/curated/silver__transfers_s3.sql
View File

@ -6,7 +6,7 @@
incremental_strategy = 'merge',
tags = ['curated']
) }}
{# TODO - time for a v2 #}
WITH action_events AS(
SELECT
@ -33,6 +33,7 @@ WITH action_events AS(
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -40,6 +41,7 @@ WITH action_events AS(
{{ this }}
)
{% endif %}
{% endif %}
),
txs AS (
SELECT
@ -61,6 +63,7 @@ txs AS (
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -68,6 +71,7 @@ txs AS (
{{ this }}
)
{% endif %}
{% endif %}
),
actions AS (
SELECT

2
models/silver/curated/staking/silver__staking_pools_s3.sql
View File

@ -62,6 +62,7 @@ function_calls AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -69,6 +70,7 @@ function_calls AS (
{{ this }}
)
{% endif %}
{% endif %}
),
add_addresses_from_tx AS (
SELECT

29
models/silver/horizon/silver_horizon__decoded_actions.sql
View File

@ -20,10 +20,11 @@ WITH all_horizon_receipts AS (
modified_timestamp AS _modified_timestamp
FROM
{{ ref('silver_horizon__receipts') }}
{% if var("MANUAL_FIX") %}
WHERE {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
WHERE _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -31,6 +32,7 @@ WITH all_horizon_receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
),
decoded_function_calls AS (
SELECT
@ -52,17 +54,7 @@ decoded_function_calls AS (
FROM
{{ ref('silver__actions_events_function_call_s3') }}
WHERE
{% if var("MANUAL_FIX") %}
{{ partition_load_manual('no_buffer') }}
{% else %}
{% if var('IS_MIGRATION') %}
{{ incremental_load_filter('_inserted_timestamp') }}
{% else %}
{{ incremental_load_filter('_modified_timestamp') }}
{% endif %}
{% endif %}
AND _partition_by_block_number >= 85000000
_partition_by_block_number >= 85000000
AND SPLIT(
action_id,
'-'
@ -72,6 +64,19 @@ decoded_function_calls AS (
FROM
all_horizon_receipts
)
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
FROM
{{ this }}
)
{% endif %}
{% endif %}
),
FINAL AS (
SELECT

2
models/silver/horizon/silver_horizon__receipts.sql
View File

@ -42,6 +42,7 @@ WITH all_horizon_receipts AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -49,6 +50,7 @@ WITH all_horizon_receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
)
SELECT

2
models/silver/social/silver_social__addkey.sql
View File

@ -22,6 +22,7 @@ WITH receipts AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -29,6 +30,7 @@ WITH receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
),
from_addkey_event AS (
SELECT

2
models/silver/social/silver_social__decoded_actions.sql
View File

@ -26,6 +26,7 @@ WITH all_social_receipts AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -33,6 +34,7 @@ WITH all_social_receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
),
decoded_function_calls AS (
SELECT

2
models/silver/social/silver_social__posts.sql
View File

@ -26,6 +26,7 @@ WITH decoded_actions AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -33,6 +34,7 @@ WITH decoded_actions AS (
{{ this }}
)
{% endif %}
{% endif %}
),
posts AS (
SELECT

2
models/silver/social/silver_social__profile_changes.sql
View File

@ -26,6 +26,7 @@ WITH decoded_actions AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -33,6 +34,7 @@ WITH decoded_actions AS (
{{ this }}
)
{% endif %}
{% endif %}
),
flatten_profile_json AS (
SELECT

2
models/silver/social/silver_social__receipts.sql
View File

@ -41,6 +41,7 @@ WITH all_social_receipts AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -48,6 +49,7 @@ WITH all_social_receipts AS (
{{ this }}
)
{% endif %}
{% endif %}
)
SELECT
*,

2
models/silver/social/silver_social__widgets.sql
View File

@ -27,6 +27,7 @@ WITH decoded_actions AS (
{% if var("MANUAL_FIX") %}
AND {{ partition_load_manual('no_buffer') }}
{% else %}
{% if is_incremental() %}
AND _modified_timestamp >= (
SELECT
MAX(modified_timestamp)
@ -34,6 +35,7 @@ WITH decoded_actions AS (
{{ this }}
)
{% endif %}
{% endif %}
),
widgets AS (
SELECT