From 1c43dba0ce95b08dd736ac0b0f9eabfe31e80a5e Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:56:53 -0600 Subject: [PATCH] update/snapshot-cmt (#37) --- models/snapshot/bronze/bronze__snapshot_proposals.sql | 3 +-- models/snapshot/bronze/bronze__snapshot_spaces.sql | 7 +++---- models/snapshot/bronze/bronze__snapshot_users.sql | 3 +-- models/snapshot/bronze/bronze__snapshot_votes.sql | 1 - 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/models/snapshot/bronze/bronze__snapshot_proposals.sql b/models/snapshot/bronze/bronze__snapshot_proposals.sql index 6f35ad6..3876892 100644 --- a/models/snapshot/bronze/bronze__snapshot_proposals.sql +++ b/models/snapshot/bronze/bronze__snapshot_proposals.sql @@ -1,10 +1,9 @@ {{ config( materialized = 'incremental', unique_key = 'proposal_id', + full_refresh = false, tags = ['snapshot'] ) }} --- full_refresh = false, --- backfill to complete in 24hrs WITH requests AS ({% for item in range(6) %} ( diff --git a/models/snapshot/bronze/bronze__snapshot_spaces.sql b/models/snapshot/bronze/bronze__snapshot_spaces.sql index 7cca3af..2d9b14c 100644 --- a/models/snapshot/bronze/bronze__snapshot_spaces.sql +++ b/models/snapshot/bronze/bronze__snapshot_spaces.sql @@ -2,12 +2,11 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'space_id', + full_refresh = false, tags = ['snapshot'] ) }} --- full_refresh = false, --- after initial fill, reduce range to 6 and change orderDirection to desc -WITH requests AS ({% for item in range(32) %} +WITH requests AS ({% for item in range(6) %} ( SELECT @@ -17,7 +16,7 @@ WITH requests AS ({% for item in range(32) %} FROM {{ source('crosschain_silver', 'apis_keys') }} WHERE - api_name = 'snapshot') },{ 'query': 'query { spaces(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ') { id name about network symbol admins members categories domain private treasuries { address name network } verified } }' }) AS resp, SYSDATE() AS _inserted_timestamp) {% if not loop.last %} + api_name = 'snapshot') },{ 'query': 'query { spaces(orderBy: "created", orderDirection: desc, first: 1000, skip: ' || {{ item * 1000 }} || ') { id name about network symbol admins members categories domain private treasuries { address name network } verified } }' }) AS resp, SYSDATE() AS _inserted_timestamp) {% if not loop.last %} UNION ALL {% endif %} {% endfor %}), diff --git a/models/snapshot/bronze/bronze__snapshot_users.sql b/models/snapshot/bronze/bronze__snapshot_users.sql index 73f3a51..59f1011 100644 --- a/models/snapshot/bronze/bronze__snapshot_users.sql +++ b/models/snapshot/bronze/bronze__snapshot_users.sql @@ -2,10 +2,9 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'address', + full_refresh = false, tags = ['snapshot'] ) }} --- full_refresh = false, --- backfill to complete in 1.5 weeks (estimate) WITH initial_request AS ({% for item in range(6) %} ( diff --git a/models/snapshot/bronze/bronze__snapshot_votes.sql b/models/snapshot/bronze/bronze__snapshot_votes.sql index cb1e21e..401ee76 100644 --- a/models/snapshot/bronze/bronze__snapshot_votes.sql +++ b/models/snapshot/bronze/bronze__snapshot_votes.sql @@ -4,7 +4,6 @@ full_refresh = false, tags = ['snapshot'] ) }} --- backfill from ethereum bronze_api__snapshot_votes WITH initial_votes_request AS ({% for item in range(6) %} (