improving sushi borrowing

This commit is contained in:
Vahid 2022-09-09 10:53:36 -04:00
parent f187ca4120
commit 259d85bd24
2 changed files with 497 additions and 264 deletions

View File

@ -1,4 +1,3 @@
{{ config(
materialized = 'incremental',
incremental_strategy = 'delete+insert',
@ -8,292 +7,534 @@
cluster_by = ['block_timestamp::DATE']
) }}
with borrow_txns as (
select distinct tx_hash,contract_address
from {{ ref('silver__logs') }}
where topics [0]::string = '0x3a5151e57d3bc9798e7853034ac52293d1a0e12a2b44725e75b03b21f86477a6'
{% if is_incremental() %}
AND _inserted_timestamp::DATE >= (
WITH borrow_txns AS (
SELECT
MAX(_inserted_timestamp) ::DATE - 2
DISTINCT tx_hash,
contract_address
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x3a5151e57d3bc9798e7853034ac52293d1a0e12a2b44725e75b03b21f86477a6'
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
Repay_txns as (
select distinct tx_hash,contract_address
from {{ ref('silver__logs') }}
where topics [0]::string = '0xc8e512d8f188ca059984b5853d2bf653da902696b8512785b182b2c813789a6e'
{% if is_incremental() %}
AND _inserted_timestamp::DATE >= (
repay_txns AS (
SELECT
MAX(_inserted_timestamp) ::DATE - 2
DISTINCT tx_hash,
contract_address
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0xc8e512d8f188ca059984b5853d2bf653da902696b8512785b182b2c813789a6e'
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
Borrow0 as (
select block_timestamp,
block_number,
tx_hash,
'Borrow' as action,
origin_from_address,
origin_to_address,
origin_function_signature,
event_index,
CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS Lending_pool_address,
origin_from_address as Borrower,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) as Borrower2,
TRY_TO_NUMBER(
public.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA)))::integer
) as amount,
case when Borrower = Borrower2 then 'no'
else 'yes' end as Borrower_is_a_contract,
_log_id,
_inserted_timestamp
from {{ ref('silver__logs') }}
where topics [0]::string = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a' and tx_hash in (select tx_hash from borrow_txns)
and CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) in (select pair_address from {{ ref('sushi__dim_kashi_pairs') }} )
{% if is_incremental() %}
AND _inserted_timestamp::DATE >= (
add_asset AS (
SELECT
MAX(_inserted_timestamp) ::DATE - 2
DISTINCT tx_hash,
contract_address
FROM
{{ this }}
)
{% endif %}
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x30a8c4f9ab5af7e1309ca87c32377d1a83366c5990472dbf9d262450eae14e38'
),
pay_coll as (
select tx_hash,
concat ('0x', SUBSTR(topics [1] :: STRING, 27, 40)) as collateral,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) as Lending_pool_address,
TRY_TO_NUMBER(
public.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA)))::integer
) as collateral_amount,
_inserted_timestamp
from {{ ref('silver__logs') }}
where topics [0]::string = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a' and tx_hash in (select tx_hash from borrow_txns)
and CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) in (select pair_address from {{ ref('sushi__dim_kashi_pairs') }} )
{% if is_incremental() %}
AND _inserted_timestamp::DATE >= (
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) ::DATE - 2
FROM
{{ this }}
)
{% endif %}
),
Borrow as
(
select a.*, b.collateral_amount, b.collateral as collateral_address
from Borrow0 a
left join pay_coll b
on a.tx_hash = b.tx_hash and a.lending_pool_address = b.lending_pool_address
),
Repay0 as (
select block_timestamp,
block_number,
tx_hash,
'Repay' as action,
origin_from_address,
origin_to_address,
origin_function_signature,
event_index,
CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) as asset,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) as Lending_pool_address,
origin_from_address as Borrower,
CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) as Borrower2,
TRY_TO_NUMBER(
public.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA)))::integer
) as amount,
case when Borrower = Borrower2 then 'no'
else 'yes' end as Lender_is_a_contract,
_log_id,
_inserted_timestamp
from {{ ref('silver__logs') }}
where topics [0]::string = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a' and tx_hash in (select tx_hash from Repay_txns)
and CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) in (select pair_address from {{ ref('sushi__dim_kashi_pairs') }} )
{% if is_incremental() %}
AND _inserted_timestamp::DATE >= (
SELECT
MAX(_inserted_timestamp) ::DATE - 2
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
receive_coll as (
select tx_hash,
CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) as collateral,
CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) as Lending_pool_address,
TRY_TO_NUMBER(
public.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA)))::integer
) as collateral_amount,
_inserted_timestamp
from {{ ref('silver__logs') }}
where topics [0]::string = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a' and tx_hash in (select tx_hash from Repay_txns)
and CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) in (select pair_address from {{ ref('sushi__dim_kashi_pairs') }} )
{% if is_incremental() %}
AND _inserted_timestamp::DATE >= (
remove_asset AS (
SELECT
MAX(_inserted_timestamp) ::DATE - 2
DISTINCT tx_hash,
contract_address
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6e853a5fd6b51d773691f542ebac8513c9992a51380d4c342031056a64114228'
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
Repay as
(
select a.*, b.collateral_amount, b.collateral as collateral_address
from Repay0 a
left join receive_coll b
on a.tx_hash = b.tx_hash and a.lending_pool_address = b.lending_pool_address
),
Total as (
select * from Borrow
union all
select * from Repay
),
prices AS (
select
symbol,
date_trunc('hour',recorded_at) as hour,
avg(price) as price
from
{{ source('prices','prices_v2') }} a
join {{ ref('sushi__dim_kashi_pairs') }} b
on a.symbol = b.asset_symbol
WHERE
1 = 1
borrow AS (
SELECT
block_timestamp,
block_number,
tx_hash,
'Borrow' AS action,
origin_from_address,
origin_to_address,
origin_function_signature,
CONCAT ('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT ('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS lending_pool_address,
origin_from_address AS borrower,
CONCAT ('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS borrower2,
TRY_TO_NUMBER(
PUBLIC.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA))) :: INTEGER) AS amount,
CASE
WHEN borrower = borrower2 THEN 'no'
ELSE 'yes'
END AS borrower_is_a_contract,
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a'
AND tx_hash IN (
SELECT
tx_hash
FROM
borrow_txns
)
AND CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) IN (
SELECT
pair_address
FROM
{{ ref('sushi__dim_kashi_pairs') }}
)
{% if is_incremental() %}
AND hour :: DATE IN (
SELECT
DISTINCT block_timestamp :: DATE
FROM
borrow
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
add_coll_same_txn AS (
SELECT
block_timestamp,
block_number,
tx_hash,
'add collateral' AS action,
origin_from_address,
origin_to_address,
origin_function_signature,
CONCAT ('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS lending_pool_address,
origin_from_address AS borrower,
CONCAT ('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS borrower2,
TRY_TO_NUMBER(
PUBLIC.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA))) :: INTEGER) AS amount,
CASE
WHEN borrower = borrower2 THEN 'no'
ELSE 'yes'
END AS borrower_is_a_contract,
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a'
AND tx_hash IN (
SELECT
tx_hash
FROM
borrow_txns
)
AND CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) IN (
SELECT
pair_address
FROM
{{ ref('sushi__dim_kashi_pairs') }}
)
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
repay AS (
SELECT
block_timestamp,
block_number,
tx_hash,
'Repay' AS action,
origin_from_address,
origin_to_address,
origin_function_signature,
CONCAT ('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT ('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS lending_pool_address,
origin_from_address AS borrower,
CONCAT ('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS borrower2,
TRY_TO_NUMBER(
PUBLIC.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA))) :: INTEGER) AS amount,
CASE
WHEN borrower = borrower2 THEN 'no'
ELSE 'yes'
END AS lender_is_a_contract,
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a'
AND tx_hash IN (
SELECT
tx_hash
FROM
repay_txns
)
AND CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) IN (
SELECT
pair_address
FROM
{{ ref('sushi__dim_kashi_pairs') }}
)
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
remove_coll_same_txn AS (
SELECT
block_timestamp,
block_number,
tx_hash,
'Remove collateral' AS action,
origin_from_address,
origin_to_address,
origin_function_signature,
CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS lending_pool_address,
origin_from_address AS borrower,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS borrower2,
TRY_TO_NUMBER(
PUBLIC.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA))) :: INTEGER) AS amount,
CASE
WHEN borrower = borrower2 THEN 'no'
ELSE 'yes'
END AS borrower_is_a_contract,
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a'
AND tx_hash IN (
SELECT
tx_hash
FROM
repay_txns
)
AND CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) IN (
SELECT
pair_address
FROM
{{ ref('sushi__dim_kashi_pairs') }}
)
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
add_coll_in_separate_txn AS (
SELECT
block_timestamp,
block_number,
tx_hash,
'add collateral' AS action,
origin_from_address,
origin_to_address,
origin_function_signature,
CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS lending_pool_address,
origin_from_address AS borrower,
CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS borrower2,
TRY_TO_NUMBER(
PUBLIC.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA))) :: INTEGER) AS amount,
CASE
WHEN borrower = borrower2 THEN 'no'
ELSE 'yes'
END AS borrower_is_a_contract,
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a'
AND tx_hash NOT IN (
SELECT
tx_hash
FROM
borrow_txns
)
AND tx_hash NOT IN (
SELECT
tx_hash
FROM
repay_txns
)
AND tx_hash NOT IN (
SELECT
tx_hash
FROM
add_asset
)
AND CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) IN (
SELECT
pair_address
FROM
{{ ref('sushi__dim_kashi_pairs') }}
)
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
remove_coll_in_separate_txn AS (
SELECT
block_timestamp,
block_number,
tx_hash,
'Remove collateral' AS action,
origin_from_address,
origin_to_address,
origin_function_signature,
CONCAT('0x', SUBSTR(topics [1] :: STRING, 27, 40)) AS asset,
CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) AS lending_pool_address,
origin_from_address AS borrower,
CONCAT('0x', SUBSTR(topics [3] :: STRING, 27, 40)) AS borrower2,
TRY_TO_NUMBER(
PUBLIC.udf_hex_to_int(SUBSTR(DATA, 3, len(DATA))) :: INTEGER) AS amount,
CASE
WHEN borrower = borrower2 THEN 'no'
ELSE 'yes'
END AS borrower_is_a_contract,
_log_id,
_inserted_timestamp
FROM
{{ ref('silver__logs') }}
WHERE
topics [0] :: STRING = '0x6eabe333476233fd382224f233210cb808a7bc4c4de64f9d76628bf63c677b1a'
AND tx_hash NOT IN (
SELECT
tx_hash
FROM
borrow_txns
)
AND tx_hash NOT IN (
SELECT
tx_hash
FROM
repay_txns
)
AND tx_hash NOT IN (
SELECT
tx_hash
FROM
remove_asset
)
AND CONCAT('0x', SUBSTR(topics [2] :: STRING, 27, 40)) IN (
SELECT
pair_address
FROM
{{ ref('sushi__dim_kashi_pairs') }}
)
{% if is_incremental() %}
AND _inserted_timestamp :: DATE >= (
SELECT
MAX(_inserted_timestamp) :: DATE - 2
FROM
{{ this }}
)
{% endif %}
),
total AS (
SELECT
*
FROM
borrow
UNION ALL
SELECT
*
FROM
add_coll_same_txn
UNION ALL
SELECT
*
FROM
remove_coll_same_txn
UNION ALL
SELECT
*
FROM
repay
UNION ALL
SELECT
*
FROM
add_coll_in_separate_txn
UNION ALL
SELECT
*
FROM
remove_coll_in_separate_txn
),
prices AS (
SELECT
symbol,
DATE_TRUNC(
'hour',
recorded_at
) AS HOUR,
AVG(price) AS price
FROM
{{ source(
'prices',
'prices_v2'
) }} A
JOIN {{ ref('sushi__dim_kashi_pairs') }}
b
ON A.symbol = b.asset_symbol
WHERE
1 = 1
{% if is_incremental() %}
AND HOUR :: DATE IN (
SELECT
DISTINCT block_timestamp :: DATE
FROM
(select * from borrow
UNION
select * from repay
UNION
select * from add_coll_in_separate_txn
UNION
select * from remove_coll_in_separate_txn)
)
{% else %}
AND hour :: DATE >= '2021-09-01'
AND HOUR :: DATE >= '2021-09-01'
{% endif %}
group by 1,2
GROUP BY
1,
2
),
collateral_prices AS (
select
symbol,
date_trunc('hour',recorded_at) as hour,
avg(price) as collateral_price
from
{{ source('prices','prices_v2') }} a
join {{ ref('sushi__dim_kashi_pairs') }} b
on a.symbol = b.collateral_symbol
WHERE
1 = 1
{% if is_incremental() %}
AND hour :: DATE IN (
SELECT
DISTINCT block_timestamp :: DATE
FROM
borrow
labels AS (
SELECT
*
FROM
{{ ref('sushi__dim_kashi_pairs') }}
),
labled_wo_prices AS (
SELECT
A.block_timestamp,
A.block_number,
A.tx_hash,
A.action,
A.origin_from_address,
A.origin_to_address,
A.origin_function_signature,
A.borrower2 AS borrower,
A.borrower_is_a_contract,
A.lending_pool_address,
b.pair_name AS lending_pool,
A.asset,
CASE
WHEN action IN (
'add collateral',
'Remove collateral'
) THEN b.collateral_symbol
ELSE b.asset_symbol
END AS symbol,
CASE
WHEN b.collateral_decimals IS NULL THEN A.amount
WHEN b.asset_decimals IS NULL THEN A.amount
WHEN b.collateral_decimals IS NOT NULL
AND action = 'add collateral' THEN (A.amount / pow(10, b.collateral_decimals))
WHEN b.collateral_decimals IS NOT NULL
AND action = 'Remove collateral' THEN (A.amount / pow(10, b.collateral_decimals))
WHEN b.asset_decimals IS NOT NULL
AND action = 'Borrow' THEN (A.amount / pow(10, b.asset_decimals))
WHEN b.asset_decimals IS NOT NULL
AND action = 'Repay' THEN (A.amount / pow(10, b.asset_decimals))
END AS amount,
A._log_id,
_inserted_timestamp
FROM
total A
LEFT JOIN labels b
ON A.lending_pool_address = b.pair_address
)
{% else %}
AND hour :: DATE >= '2021-09-01'
{% endif %}
group by 1,2
),
labels as (
select *
from {{ ref('sushi__dim_kashi_pairs') }}
),
Labled_WO_prices as (
select
a.block_timestamp,
a.block_number,
a.tx_hash,
a.action,
a.origin_from_address,
a.origin_to_address,
a.origin_function_signature,
a.asset,
a.Borrower2 as Borrower,
a.Borrower_is_a_contract,
a.lending_pool_address,
a.event_index,
b.asset_decimals,
case when b.asset_decimals is null then a.amount else (a.amount/pow(10,b.asset_decimals)) end as asset_amount,
case when b.collateral_decimals is null then a.collateral_amount else (a.collateral_amount/pow(10,b.collateral_decimals)) end as collateral_amount,
b.pair_name as lending_pool,
b.asset_symbol as symbol,
a._log_id,
substring(b.pair_name,3,charindex('/',b.pair_name)-3) as collateral_symbol,
_inserted_timestamp
from FINAL a
left join labels b
on a.Lending_pool_address = b.pair_address
)
select
a.block_timestamp,
a.block_number,
a.tx_hash,
a.action,
a.origin_from_address,
a.origin_to_address,
a.origin_function_signature,
a.asset,
a.Borrower,
a.Borrower_is_a_contract,
a.lending_pool_address,
a.event_index,
a.lending_pool,
a.asset,
a.symbol,
a.asset_amount,
(a.asset_amount* c.price) as asset_amount_USD ,
a.collateral_address,
a.collateral_symbol,
a.collateral_amount,
(a.collateral_amount* d.collateral_price) as collateral_amount_USD,
a._log_id,
_inserted_timestamp
from Labled_WO_prices a
LEFT JOIN prices c
ON a.symbol = c.symbol
AND DATE_TRUNC(
SELECT
A.block_timestamp,
A.block_number,
A.tx_hash,
A.action,
A.origin_from_address,
A.origin_to_address,
A.origin_function_signature,
A.borrower,
A.borrower_is_a_contract,
A.lending_pool_address,
A.lending_pool,
A.asset,
A.symbol,
A.amount,
CASE
WHEN action = 'add collateral' THEN (
A.amount * C.price
)
WHEN action = 'Remove collateral' THEN (
A.amount * C.price
)
ELSE (
A.amount * C.price
)
END AS amount_USD,
A._log_id,
_inserted_timestamp
FROM
labled_wo_prices A
LEFT JOIN prices C
ON A.symbol = C.symbol
AND DATE_TRUNC(
'hour',
a.block_timestamp
) = c.hour
LEFT JOIN collateral_prices d
ON a.symbol = d.symbol
AND DATE_TRUNC(
'hour',
a.block_timestamp
) = d.hour
A.block_timestamp
) = C.hour

View File

@ -42,7 +42,7 @@ models:
- name: ACTION
description: '{{ doc("borrow_action") }}'
- name: ASSET_AMOUNT
- name: AMOUNT
description: '{{ doc("borrow_amount") }}'
tests:
- not_null
@ -51,7 +51,7 @@ models:
- NUMBER
- FLOAT
- name: ASSET_AMOUNT_USD
- name: AMOUNT_USD
description: '{{ doc("borrow_amount_usd") }}'
tests:
- dbt_expectations.expect_column_values_to_be_in_type_list:
@ -83,14 +83,6 @@ models:
- dbt_expectations.expect_column_values_to_match_regex:
regex: 0[xX][0-9a-fA-F]+
- name: EVENT_INDEX
description: '{{ doc("bsc_event_index") }}'
tests:
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: _LOG_ID
description: '{{ doc("bsc_log_id_events") }}'
tests: