flip/crosschain-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/crosschain-models.git synced 2026-02-06 15:36:46 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

An 1740/new crosschain prices (#4)

Browse Source 
* integrate coin market cap asset ohlc api

* revert, accidentally moved this file

* add sample profile
This commit is contained in:
desmond-hui 2022-07-22 08:58:02 -07:00 committed by GitHub
parent d0a76edaa9
commit c79ec93d98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 217 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README.md
View File

@ -1 +1,30 @@
# crosschain-models
## Profile Set Up
#### Use the following within profiles.yml
----
```yml
crosschain:
target: dev
outputs:
dev:
type: snowflake
account: <ACCOUNT>
role: <ROLE>
user: <USERNAME>
password: <PASSWORD>
region: <REGION>
database: CROSSCHAIN_DEV
warehouse: <WAREHOUSE>
schema: silver
threads: 4
client_session_keep_alive: False
query_tag: <TAG>
```
### Resources:
- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
- Find [dbt events](https://events.getdbt.com) near you
- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

1
dbt_project.yml
View File

@ -32,6 +32,7 @@ on-run-start:
- '{{ sp_create_bulk_get_coin_market_cap_asset_metadata() }}'
- '{{ sp_create_bulk_get_coin_market_cap_prices() }}'
- '{{ sp_create_bulk_get_coin_gecko_prices() }}'
- '{{ sp_create_bulk_get_coin_market_cap_hourly_ohlc() }}'
- '{{create_tasks()}}'

2
macros/create_udfs.sql
View File

@ -1,10 +1,12 @@
{% macro create_udfs() %}
{% set sql %}
CREATE schema if NOT EXISTS streamline;
{{ udf_bulk_fill_cmc_historical_price_gaps() }};
{{ udf_bulk_get_coin_gecko_asset_metadata() }};
{{ udf_bulk_get_coin_market_cap_asset_metadata() }};
{{ udf_bulk_get_coin_market_cap_prices() }};
{{ udf_bulk_get_coin_gecko_prices() }};
{{ udf_bulk_get_coin_market_cap_hourly_ohlc() }};
{% endset %}
{% do run_query(sql) %}
{% endmacro %}

2
macros/streamline/fill_cmc_historical_price_gaps/sp_create_bulk_fill_cmc_historical_price_gaps.sql
View File

@ -13,7 +13,7 @@ $$
SELECT
COUNT(1)
FROM
silver.legacy_prices_gaps
streamline.legacy_prices_gaps
);
if (
row_cnt > 0

7
macros/streamline/get_coin_market_cap_hourly_ohlc/run_sp_bulk_get_coin_market_cap_hourly_ohlc.sql Normal file
View File

@ -0,0 +1,7 @@
{% macro run_sp_bulk_get_coin_market_cap_hourly_ohlc() %}
{% set sql %}
call streamline.sp_bulk_get_coin_market_cap_hourly_ohlc();
{% endset %}
{% do run_query(sql) %}
{% endmacro %}

30
macros/streamline/get_coin_market_cap_hourly_ohlc/sp_create_get_coin_market_cap_hourly_ohlc.sql Normal file
View File

@ -0,0 +1,30 @@
{% macro sp_create_bulk_get_coin_market_cap_hourly_ohlc() %}
{% set sql %}
CREATE OR REPLACE PROCEDURE streamline.sp_bulk_get_coin_market_cap_hourly_ohlc()
RETURNS variant
LANGUAGE SQL
AS
$$
DECLARE
RESULT VARCHAR;
row_cnt INTEGER;
BEGIN
row_cnt:= (
SELECT
COUNT(1)
FROM
silver.all_unknown_coin_market_cap_asset_ohlc_hourly
);
if (
row_cnt > 0
) THEN RESULT:= (
SELECT
streamline.udf_bulk_get_coin_market_cap_hourly_ohlc()
);
ELSE RESULT:= NULL;
END if;
RETURN RESULT;
END;
$${% endset %}
{% do run_query(sql) %}
{% endmacro %}

8
macros/streamline/get_coin_market_cap_hourly_ohlc/udf_get_coin_market_cap_hourly_ohlc.sql Normal file
View File

@ -0,0 +1,8 @@
{% macro udf_bulk_get_coin_market_cap_hourly_ohlc() %}
CREATE
OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_get_coin_market_cap_hourly_ohlc() returns text api_integration = aws_crosschain_api_dev AS {% if target.name == "prod" -%}
'https://q2il6n5mmg.execute-api.us-east-1.amazonaws.com/prod/bulk_get_coin_market_cap_hourly_ohlc'
{% else %}
'https://ubuxgfotp2.execute-api.us-east-1.amazonaws.com/dev/bulk_get_coin_market_cap_hourly_ohlc'
{%- endif %}
{% endmacro %}

50
models/sources.yml
View File

@ -20,6 +20,52 @@ sources:
schema: core
tables:
- name: fact_transactions
- name: crosschain_external
schema: bronze
loader: S3
tables:
- name: asset_metadata_api
description: "all coins supported by provider"
external:
location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_METADATA_API"
file_format: "( type = json, strip_outer_array = TRUE )"
auto_refresh: true
partitions:
- name: provider
data_type: string
expression: split_part(METADATA$FILENAME,'/',3)
columns:
- name: id
data_type: string
description: ""
- name: symbol
data_type: string
description: ""
- name: name
data_type: string
description: ""
- name: asset_ohlc_coin_market_cap_api
description: "all coins supported by provider"
external:
location: "@crosschain.bronze.analytics_external_tables/{{target.database}}/ASSET_OHLC_API/coinmarketcap"
file_format: "( type = json, strip_outer_array = TRUE )"
auto_refresh: true
partitions:
- name: _inserted_date
data_type: string
expression: substr((split_part(METADATA$FILENAME,'/',4)),16,10)
columns:
- name: id
data_type: number
description: ""
- name: api_start_time
data_type: number
- name: api_end_time
data_type: number
- name: metadata
data_type: string
description: ""
- name: data
data_type: variant
description: ""

90
models/streamline/streamline__all_unknown_coin_market_cap_asset_ohlc_hourly.sql Normal file
View File

@ -0,0 +1,90 @@
{{ config(
materialized = 'view',
) }}
WITH hours AS (
SELECT
HOUR
FROM
{% if target.name == 'prod' %}
{{ source(
'legacy_db',
'hours'
) }}
{% else %}
{{ source(
'legacy_dev_db',
'hours'
) }}
{% endif %}
WHERE
HOUR >= '2022-07-20'
AND HOUR < DATE_TRUNC(
'hour',
CURRENT_TIMESTAMP
) -- the hour should always be less than current time because it must be "completed" before ohlcv is available
),
cmc_active_assets AS (
SELECT
id::number as id
FROM
{{ source(
'crosschain_external',
'asset_metadata_api'
) }}
WHERE
provider = 'coinmarketcap'
AND VALUE :status :: STRING = 'active'
),
base AS (
SELECT
DATE_PART('epoch', DATEADD('minute', -1, HOUR)) AS start_time,
DATE_PART('epoch', DATEADD('hour', 1, HOUR)) AS end_time,
id
FROM
cmc_active_assets
CROSS JOIN hours
EXCEPT
SELECT
api_start_time,
api_end_time,
id
FROM
{{ source(
'crosschain_external',
'asset_ohlc_coin_market_cap_api'
) }}
),
base_params AS (
SELECT
start_time,
end_time,
id,
1 AS cnt,
SUM(cnt) over (
PARTITION BY start_time,
end_time
ORDER BY
id
) AS csum,
CEIL(
csum / 1000
) AS group_cnt
FROM
base
)
SELECT
start_time,
end_time,
group_cnt,
LISTAGG(
id,
','
) AS asset_ids
FROM
base_params
GROUP BY
1,
2,
3

1
models/silver/silver__legacy_prices_gaps.sql → models/streamline/streamline__legacy_prices_gaps.sql
View File

@ -1,6 +1,5 @@
{{ config(
materialized = 'view',
post_hook = 'call silver.sp_bulk_fill_cmc_historical_price_gaps()'
) }}
WITH cte_date (date_rec) AS (