testnet and more curation

This commit is contained in:
Eric Laurello 2023-05-30 15:37:09 -04:00
parent dba95088c6
commit 46a4d34ed1
24 changed files with 784 additions and 6 deletions

View File

@ -5,7 +5,7 @@ on:
workflow_dispatch:
schedule:
# “Hourly”
- cron: '3 * * * *'
- cron: '3,33 * * * *'
env:
USE_VARS: "${{ vars.USE_VARS }}"

View File

@ -9,7 +9,8 @@
(
SELECT
ethereum.streamline.udf_json_rpc_call(
'http://35.158.109.187:26657',{},
--'http://35.158.109.187:26657'
'http://3.76.200.142:26657',{},
[ { 'id': 1, 'jsonrpc': '2.0', 'method': 'abci_info' } ]
) DATA
)
@ -51,7 +52,7 @@ INSERT INTO
852655
)
ORDER BY
1 DESC
1
),
calls AS (
SELECT
@ -93,7 +94,7 @@ INSERT INTO
results AS (
SELECT
ethereum.streamline.udf_json_rpc_call(
'http://35.158.109.187:26657',{},
'http://3.76.200.142:26657',{},
calls
) DATA
FROM

View File

@ -58,7 +58,7 @@ INSERT INTO
WHERE
block_number > 9839243
ORDER BY
1 DESC
1
LIMIT
5000
)
@ -69,7 +69,7 @@ INSERT INTO
results AS (
SELECT
ethereum.streamline.udf_json_rpc_call(
'http://35.158.109.187:26657',{},
'http://3.76.200.142:26657',{},
calls
) DATA
FROM

View File

@ -0,0 +1,5 @@
{% docs memo %}
The memo the user input into the transaction body
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs rewards_recipient %}
The wallet address of the that received the rewards.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs vote_option %}
How the delegator voted on the proposal.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs vote_weight %}
Denotes what portion of a user's stake is put toward the voting option. If "1", 100% of the voter's voting power is put toward the selected option.
{% enddocs %}

View File

@ -0,0 +1,5 @@
{% docs voter %}
Address of the delegator that voted on the proposal.
{% enddocs %}

View File

@ -0,0 +1,16 @@
{{ config(
materialized = 'view',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }} }
) }}
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
depositor,
proposal_id,
amount,
currency
FROM
{{ ref('silver__governance_proposal_deposits') }}

View File

@ -0,0 +1,38 @@
version: 2
models:
- name: core__fact_governance_proposal_deposits
description: Records of all proposal deposits that have occurred on Osmosis, dating back to the genesis block. At least 500 OSMO must be deposited for a proposal to be voted on. The columns blockchain, chain_id, and tx_status will be deprecating soon.
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_SUCCEEDED
description: "{{ doc('tx_succeeded') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: DEPOSITOR
description: "{{ doc('depositor') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSAL_ID
description: "{{ doc('proposal_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: AMOUNT
description: "{{ doc('amount') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: CURRENCY
description: "{{ doc('currency') }}"
tests:
- dbt_expectations.expect_column_to_exist

View File

@ -0,0 +1,22 @@
{{ config(
materialized = 'view',
meta={
'database_tags':{
'table': {
'PURPOSE': 'GOVERNANCE'
}
}
}
) }}
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
proposer,
proposal_id,
proposal_type,
proposal_title,
proposal_description
FROM {{ ref('silver__governance_submit_proposal') }}

View File

@ -0,0 +1,41 @@
version: 2
models:
- name: core__fact_governance_submit_proposal
description: Records of all proposal submissions on Osmosis, dating back to the Genesis block. The columns blockchain, chain_id, and tx_status will be deprecating soon.
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_SUCCEEDED
description: "{{ doc('tx_succeeded') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSER
description: "{{ doc('proposer') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSAL_ID
description: "{{ doc('proposal_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSAL_TYPE
description: "{{ doc('proposal_type') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSAL_TITLE
description: "{{ doc('proposal_title') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSAL_DESCRIPTION
description: "{{ doc('proposal_description') }}"
tests:
- dbt_expectations.expect_column_to_exist

View File

@ -0,0 +1,17 @@
{{ config(
materialized = 'view',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'GOVERNANCE' }}}
) }}
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
voter,
proposal_id,
vote_option,
vote_weight,
memo
FROM
{{ ref('silver__governance_votes') }}

View File

@ -0,0 +1,37 @@
version: 2
models:
- name: core__fact_governance_votes
description: Records of all votes on proposals.
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_SUCCEEDED
description: "{{ doc('tx_succeeded') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: VOTER
description: "{{ doc('voter') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: PROPOSAL_ID
description: "{{ doc('proposal_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: VOTE_OPTION
description: "{{ doc('vote_option') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: MEMO
description: "{{ doc('memo') }}"
tests:
- dbt_expectations.expect_column_to_exist

View File

@ -0,0 +1,18 @@
{{ config(
materialized = 'view',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'PRICE' }} }
) }}
SELECT
A.block_id,
A.block_timestamp,
A.tx_id,
A.tx_succeeded,
A.msg_group,
A.msg_sub_group,
A.tx_sender,
A.voter,
A.amount,
A.currency
FROM
{{ ref('silver__oracle_votes') }} A

View File

@ -0,0 +1,35 @@
version: 2
models:
- name: core__fact_oracle_votes
description: Records of all oracle votes from validators.
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_SUCCEEDED
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- BOOLEAN
- name: VOTER
description: "{{ doc('voter') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: AMOUNT
description: "{{ doc('amount') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: CURRENCY
description: "{{ doc('currency') }}"
tests:
- dbt_expectations.expect_column_to_exist

View File

@ -0,0 +1,19 @@
{{ config(
materialized = 'view',
meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }} }
) }}
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
tx_caller_address,
action,
delegator_address,
rewards_recipient,
validator_address,
amount,
currency
FROM
{{ ref('silver__staking_rewards') }}

View File

@ -0,0 +1,49 @@
version: 2
models:
- name: core__fact_staking_rewards
description: Records of all claimed rewards earned from staking that have occurred on Osmosis, dating back to the genesis block. These actions include claim and withdrawal_rewards. The columns blockchain, chain_id, and tx_status will be deprecating soon.
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_SUCCEEDED
description: "{{ doc('tx_succeeded') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: TX_CALLER_ADDRESS
description: "{{ doc('tx_caller_address') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: ACTION
description: "{{ doc('action') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: DELEGATOR_ADDRESS
description: "{{ doc('delegator_address') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: REWARDS_RECIPIENT
description: "{{ doc('delegator_address') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: AMOUNT
description: "{{ doc('amount') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: CURRENCY
description: "{{ doc('currency') }}"
tests:
- dbt_expectations.expect_column_to_exist
- name: VALIDATOR_ADDRESS
description: "{{ doc('validator_address') }}"
tests:
- dbt_expectations.expect_column_to_exist

View File

@ -0,0 +1,134 @@
{{ config(
materialized = 'incremental',
unique_key = "_unique_key",
incremental_strategy = 'merge',
cluster_by = ['block_timestamp::DATE'],
) }}
WITH base_ma AS (
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
msg_index,
msg_type,
attribute_key,
attribute_value,
_inserted_timestamp
FROM
{{ ref('silver__msg_attributes') }}
{% if is_incremental() %}
WHERE
_inserted_timestamp >= (
SELECT
MAX(
_inserted_timestamp
)
FROM
{{ this }}
)
{% endif %}
),
tx_mg_msg AS (
SELECT
tx_id,
block_timestamp,
msg_group,
msg_sub_group
FROM
base_ma
WHERE
msg_type = 'message'
AND attribute_key = 'action'
AND attribute_value = '/cosmos.gov.v1beta1.MsgVote'
),
vote_msgs AS (
SELECT
A.block_id,
A.block_timestamp,
A.tx_id,
tx_succeeded,
A.msg_group,
A.msg_sub_group,
COALESCE(
C.key,
A.attribute_key
) AS attribute_key,
COALESCE(
C.value,
A.attribute_value
) AS attribute_value,
_inserted_timestamp
FROM
base_ma A
JOIN tx_mg_msg b
ON A.tx_id = b.tx_id
AND A.msg_group = b.msg_group
AND A.msg_sub_group = b.msg_sub_group
AND A.block_timestamp = b.block_timestamp
LEFT JOIN LATERAL FLATTEN (
input => TRY_PARSE_JSON(attribute_value),
outer => TRUE
) C
WHERE
msg_type || attribute_key IN (
'proposal_voteproposal_id',
'proposal_voteoption',
'messagesender'
)
),
agg AS (
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
_inserted_timestamp,
OBJECT_AGG(
attribute_key :: STRING,
attribute_value :: variant
) AS j,
j :proposal_id :: INT AS proposal_id,
j :sender :: STRING AS voter,
j :option :: STRING AS vote_option,
j :weight :: FLOAT AS vote_weight
FROM
vote_msgs
GROUP BY
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
_inserted_timestamp
)
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
proposal_id,
voter,
vote_option,
vote_weight,
NULL AS memo,
_inserted_timestamp,
concat_ws(
'-',
tx_id,
msg_group,
msg_sub_group,
voter
) AS _unique_key
FROM
agg

View File

@ -0,0 +1,75 @@
version: 2
models:
- name: silver__governance_votes
description: Records of all votes on proposals.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_ID
- MSG_GROUP
- MSG_SUB_GROUP
- VOTER
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: TX_SUCCEEDED
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- BOOLEAN
- name: VOTER
description: "{{ doc('voter') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: PROPOSAL_ID
description: "{{ doc('proposal_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: VOTE_OPTION
description: "{{ doc('vote_option') }}"
tests:
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('inserted_timestamp') }}"
tests:
- not_null
- name: VOTE_WEIGHT
description: The percentage of one's delegation put towards a voting option. 1 = 100%, etc.
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: MEMO
description: "{{ doc('memo') }}"

View File

@ -0,0 +1,185 @@
{{ config(
materialized = 'incremental',
unique_key = "_unique_key",
incremental_strategy = 'merge',
cluster_by = ['block_timestamp::DATE'],
) }}
WITH base_ma AS (
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
msg_index,
msg_type,
attribute_key,
attribute_value,
_inserted_timestamp
FROM
{{ ref('silver__msg_attributes') }}
{% if is_incremental() %}
WHERE
_inserted_timestamp >= (
SELECT
MAX(
_inserted_timestamp
)
FROM
{{ this }}
)
{% endif %}
),
tx_mg_msg AS (
SELECT
tx_id,
block_timestamp,
msg_group,
msg_sub_group
FROM
base_ma
WHERE
msg_type = 'message'
AND attribute_key = 'action'
AND attribute_value = '/seiprotocol.seichain.oracle.MsgAggregateExchangeRateVote'
),
vote_msgs AS (
SELECT
A.block_id,
A.block_timestamp,
A.tx_id,
tx_succeeded,
A.msg_group,
A.msg_sub_group,
COALESCE(
C.key,
A.attribute_key
) AS attribute_key,
COALESCE(
C.value,
A.attribute_value
) AS attribute_value,
_inserted_timestamp
FROM
base_ma A
LEFT JOIN tx_mg_msg b
ON A.tx_id = b.tx_id
AND A.msg_group = b.msg_group
AND A.msg_sub_group = b.msg_sub_group
AND A.block_timestamp = b.block_timestamp
LEFT JOIN LATERAL FLATTEN (
input => TRY_PARSE_JSON(attribute_value),
outer => TRUE
) C
WHERE
(
b.tx_id IS NOT NULL
OR (
A.msg_type = 'tx'
AND A.attribute_key = 'acc_seq'
)
)
AND msg_type = 'aggregate_vote'
),
agg AS (
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
_inserted_timestamp,
OBJECT_AGG(
attribute_key :: STRING,
attribute_value :: variant
) AS j,
j :voter :: STRING AS voter,
j :exchange_rates :: STRING AS exchange_rates
FROM
vote_msgs
GROUP BY
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
_inserted_timestamp
),
votes_split AS (
SELECT
block_id,
block_timestamp,
tx_id,
tx_succeeded,
msg_group,
msg_sub_group,
b.value AS split_rates
FROM
agg,
LATERAL SPLIT_TO_TABLE(
exchange_rates,
','
) b
),
tx_sender AS (
SELECT
tx_id,
SPLIT_PART(
attribute_value,
'/',
0
) AS tx_sender
FROM
base_ma
WHERE
attribute_key = 'acc_seq' qualify(ROW_NUMBER() over(PARTITION BY tx_id
ORDER BY
msg_index)) = 1
)
SELECT
A.block_id,
A.block_timestamp,
A.tx_id,
A.tx_succeeded,
A.msg_group,
A.msg_sub_group,
b.tx_sender,
A.voter,
SPLIT_PART(
TRIM(
REGEXP_REPLACE(
split_rates,
'[^[:digit:].]',
' '
)
),
' ',
0
) :: DECIMAL(
38,
18
) AS amount,
RIGHT(split_rates, LENGTH(split_rates) - LENGTH(SPLIT_PART(TRIM(REGEXP_REPLACE(split_rates, '[^[:digit:].]', ' ')), ' ', 0))) :: STRING AS currency,
split_rates AS raw_exchange_rates,
A._inserted_timestamp,
concat_ws(
'-',
A.tx_id,
A.msg_group,
A.msg_sub_group,
voter
) AS _unique_key
FROM
agg A
JOIN tx_sender b
ON A.tx_id = b.tx_id
JOIN votes_split C
ON A.tx_id = C.tx_id
AND A.msg_group = C.msg_group
AND A.msg_sub_group = C.msg_sub_group

View File

@ -0,0 +1,65 @@
version: 2
models:
- name: silver__oracle_votes
description: Records of all oracle votes from validators.
tests:
- dbt_utils.unique_combination_of_columns:
combination_of_columns:
- TX_ID
- MSG_GROUP
- MSG_SUB_GROUP
- CURRENCY
columns:
- name: BLOCK_ID
description: "{{ doc('block_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: BLOCK_TIMESTAMP
description: "{{ doc('block_timestamp') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- TIMESTAMP_NTZ
- name: TX_ID
description: "{{ doc('tx_id') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: TX_SUCCEEDED
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- BOOLEAN
- name: VOTER
description: "{{ doc('voter') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- STRING
- VARCHAR
- name: AMOUNT
description: "{{ doc('amount') }}"
tests:
- not_null
- dbt_expectations.expect_column_values_to_be_in_type_list:
column_type_list:
- NUMBER
- FLOAT
- name: CURRENCY
description: "{{ doc('currency') }}"
tests:
- not_null
- name: _INSERTED_TIMESTAMP
description: "{{ doc('inserted_timestamp') }}"
tests:
- not_null

View File

@ -210,6 +210,7 @@ SELECT
A.amount,
A.currency,
A.validator_address,
'withdraw_rewards' AS action,
A._inserted_timestamp,
concat_ws(
'-',