Hotfix/reingest full parts to fix partial load (#204)

* temporarily reload larger range

* check complete blocks for max partition instead of arbitrary range
This commit is contained in:
desmond-hui 2023-01-11 09:08:38 -08:00 committed by GitHub
parent dba8e47c4f
commit d4ed00c3af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 14 deletions

View File

@ -47,18 +47,18 @@ WITH pre_final AS (
) <> 'Vote111111111111111111111111111111111111111'
{% if is_incremental() %}
AND _partition_id >= (
SELECT
MAX(_partition_id) -3
FROM
{{ this }}
)
AND _partition_id <= (
SELECT
MAX(_partition_id) + 10
FROM
{{ this }}
)
AND _partition_id >= (
SELECT
MAX(_partition_id) -3
FROM
{{ this }}
)
AND _partition_id <= (
SELECT
MAX(_partition_id)
FROM
{{ ref('streamline__complete_block_txs') }}
)
-- AND t._inserted_timestamp > (
-- SELECT
-- MAX(_inserted_timestamp)

View File

@ -46,8 +46,10 @@ WITH pre_final AS (
)
AND
_partition_id <= (
select max(_partition_id)+10
from {{this}}
SELECT
MAX(_partition_id)
FROM
{{ ref('streamline__complete_block_txs') }}
)
-- AND
-- t._inserted_timestamp > (