flip/neutron-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/neutron-models.git synced 2026-02-06 10:56:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

updates from dry run

Browse Source
This commit is contained in:
Mike Stepanovic 2025-06-05 13:58:07 -06:00
parent 414792c651
commit a77219e924
10 changed files with 48 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/dbt_run_core.yml vendored
View File

@ -41,4 +41,4 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt run -m <ibc_chain>,tag:core
dbt run -m neutron,tag:core

2
.github/workflows/dbt_run_noncore.yml vendored
View File

@ -41,4 +41,4 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt run -m <ibc_chain>,tag:noncore
dbt run -m neutron,tag:noncore

2
.github/workflows/dbt_test_recency.yml vendored
View File

@ -43,7 +43,7 @@ jobs:
dbt deps
- name: Run DBT Jobs
run: |
dbt source freshness && dbt test -m <ibc_chain>,tag:test_recency
dbt source freshness && dbt test -m neutron,tag:test_recency
continue-on-error: true

4
README.md
View File

@ -1,11 +1,11 @@
## Repo Set Up
1. Create a new repository from the [ibc-models-template](https://github.com/FlipsideCrypto/ibc-models-template)
2. Update all references to `<ibc_chain>` to the new chain name, in lowercase by using find and replace
2. Update all references to `neutron` to the new chain name, in lowercase by using find and replace
3. Update the fsc-ibc package version in `packages.yml` to the latest version
4. Set up the rest of the dbt project, where applicable, including but not limited to:
- `dbt_project.yml` (enable/disable packages, vars, etc.)
- `.github/workflows` (update tags, etc.)
- `overview.md` (update `<ibc_chain>`, table references, docs etc.)
- `overview.md` (update `neutron`, table references, docs etc.)
- `sources.yml` (update schemas, tables etc.)
- `requirements.txt` (update dependencies)
- other files where applicable

50
dbt_project.yml
View File

@ -1,12 +1,12 @@
# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: "<ibc_chain>_models" # replace with the name of the chain
name: "neutron_models" # replace with the name of the chain
version: "1.0.0"
config-version: 2
# This setting configures which "profile" dbt uses for this project.
profile: "<ibc_chain>" # replace with the name of the chain
profile: "neutron" # replace with the name of the chain
# These configurations specify where dbt should look for different types of files.
# The `source-paths` config, for example, states that models in this project can be
@ -26,8 +26,8 @@ clean-targets: # directories to be removed by `dbt clean`
- "dbt_packages"
tests:
<ibc_chain>_models: # replace with the name of the chain
+store_failures: true # all tests
#neutron_models: # replace with the name of the chain
#+store_failures: true # all tests
fsc_ibc:
+store_failures: true
@ -41,7 +41,7 @@ on-run-end:
dispatch:
- macro_namespace: dbt
search_order:
- <ibc_chain>-models
- neutron-models
- dbt_snowflake_query_tags
- dbt
@ -53,14 +53,12 @@ query-comment:
# Full documentation: https://docs.getdbt.com/docs/configuring-models
models:
<ibc_chain>_models: # replace with the name of the chain
+copy_grants: true
+persist_docs:
relation: true
columns: true
+on_schema_change: "append_new_columns"
livequery_base:
+materialized: ephemeral
#neutron_models: # replace with the name of the chain
# +copy_grants: true
# +persist_docs:
# relation: true
# columns: true
# +on_schema_change: "append_new_columns"
fsc_ibc:
+enabled: false # disable fsc_ibc package by default
+copy_grants: true
@ -70,22 +68,10 @@ models:
+on_schema_change: "append_new_columns"
main_package:
+enabled: false # disable main_package by default, enabled other packages as needed
admin:
+enabled: true
core:
+enabled: true # enable subpackages, as needed
github_actions:
+enabled: true
labels:
+enabled: true
observability:
+enabled: true
prices:
+enabled: true
utils:
+enabled: true
scores_package:
+enabled: false
+enabled: false
vars:
"dbt_date:time_zone": GMT
@ -105,19 +91,19 @@ vars:
config:
# The keys correspond to dbt profiles and are case sensitive
dev:
API_INTEGRATION: AWS_<IBC_CHAIN>_API_STG_V2
EXTERNAL_FUNCTION_URI: <insert_uri>.execute-api.us-east-1.amazonaws.com/stg/
API_INTEGRATION: AWS_NEUTRON_API_STG_V2
EXTERNAL_FUNCTION_URI: zjdy6lie8f.execute-api.us-east-1.amazonaws.com/stg/
ROLES:
- AWS_LAMBDA_<IBC_CHAIN>_API # replace with the name of the chain
- AWS_LAMBDA_NEUTRON_API # replace with the name of the chain
- INTERNAL_DEV
prod:
API_INTEGRATION: AWS_<IBC_CHAIN>_API_PROD_V2
API_INTEGRATION: AWS_NEUTRON_API_PROD_V2
EXTERNAL_FUNCTION_URI: <insert_uri>.execute-api.us-east-1.amazonaws.com/prod/
ROLES:
- AWS_LAMBDA_<IBC_CHAIN>_API # replace with the name of the chain
- AWS_LAMBDA_NEUTRON_API # replace with the name of the chain
- INTERNAL_DEV
- DBT_CLOUD_<IBC_CHAIN> # replace with the name of the chain
- DBT_CLOUD_NEUTRON # replace with the name of the chain
#### STREAMLINE 2.0 END ####

6
makefile
View File

@ -8,13 +8,16 @@ deploy_livequery:
@set -e; \
dbt run-operation fsc_ibc.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
dbt run -m livequery_models.deploy.core.live --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
dbt run-operation fsc_ibc.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
deploy_chain_phase_1:
@set -e; \
echo "### DEPLOYING CORE LIVEQUERY DEPENDENCIES ###"; \
dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
dbt run-operation fsc_ibc.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
dbt run-operation fsc_utils.create_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
echo "### STARTING PHASE 1 ###"; \
if [ "$(DBT_TARGET)" != "prod" ]; then \
dbt run -m "fsc_ibc,tag:phase_1" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \
dbt test -m "fsc_ibc,tag:chainhead"; \
@ -29,6 +32,7 @@ deploy_chain_phase_1:
deploy_chain_phase_2:
@set -e; \
echo "### STARTING PHASE 2 ###"; \
if [ "$(DBT_TARGET)" != "prod" ]; then \
dbt run -m "fsc_ibc,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
dbt run -m "fsc_ibc,tag:streamlines,tag:realtime" "fsc_ibc,tag:streamline,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
@ -41,6 +45,7 @@ deploy_chain_phase_2:
deploy_chain_phase_3:
@set -e; \
echo "### STARTING PHASE 3 ###"; \
if [ "$(DBT_TARGET)" != "prod" ]; then \
dbt run -m "fsc_ibc,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
dbt run -m "fsc_ibc,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
@ -55,6 +60,7 @@ deploy_chain_phase_3:
deploy_chain_phase_4:
@set -e; \
echo "### STARTING PHASE 4 ###"; \
if [ "$(DBT_TARGET)" != "prod" ]; then \
dbt run -m "fsc_ibc,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \
dbt run -m "fsc_ibc,tag:phase_4" --full-refresh -t $(DBT_TARGET); \

30
models/__overview__.md
View File

@ -1,9 +1,9 @@
{% docs __overview__ %}
# Welcome to the Flipside Crypto <ibc_chain> Models Documentation!
# Welcome to the Flipside Crypto neutron Models Documentation!
## **What does this documentation cover?**
The documentation included here details the design of the <ibc_chain> tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/<ibc_chain>-models)
The documentation included here details the design of the neutron tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/neutron-models)
## **How do I use these docs?**
The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information.
@ -16,23 +16,23 @@ There is more information on how to use dbt docs in the last section of this doc
**Click on the links below to jump to the documentation for each schema.**
### Core Tables (<ibc_chain>.core)
### Core Tables (neutron.core)
**Dimension Tables:**
- [dim_labels](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.core__dim_labels)
- [dim_labels](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__dim_labels)
**Fact Tables:**
- [fact_blocks](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.core__fact_blocks)
- [fact_msgs](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.core__fact_msgs)
- [fact_msg_attributes](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.core__fact_msg_attributes)
- [fact_transactions](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.core__fact_transactions)
- [fact_transfers](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.core__fact_transfers)
- [fact_blocks](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_blocks)
- [fact_msgs](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_msgs)
- [fact_msg_attributes](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_msg_attributes)
- [fact_transactions](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_transactions)
- [fact_transfers](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_transfers)
### Price Tables (<ibc_chain>.price)
- [dim_asset_metadata](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.price__dim_asset_metadata)
- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.price__fact_prices_ohlc_hourly)
- [ez_asset_metadata](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.price__ez_asset_metadata)
- [ez_prices_hourly](https://flipsidecrypto.github.io/<ibc_chain>/#!/model/model.<ibc_chain>_models.price__ez_prices_hourly)
### Price Tables (neutron.price)
- [dim_asset_metadata](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__dim_asset_metadata)
- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__fact_prices_ohlc_hourly)
- [ez_asset_metadata](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__ez_asset_metadata)
- [ez_prices_hourly](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__ez_prices_hourly)
## **Helpful User-Defined Functions (UDFs)**
@ -76,7 +76,7 @@ Note that you can also right-click on models to interactively filter and explore
- [Flipside](https://flipsidecrypto.xyz/)
- [Data Studio](https://flipsidecrypto.xyz/studio)
- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials)
- [Github](https://github.com/FlipsideCrypto/<ibc_chain>-models)
- [Github](https://github.com/FlipsideCrypto/neutron-models)
- [What is dbt?](https://docs.getdbt.com/docs/introduction)
{% enddocs %}

3
models/sources.yml
View File

@ -13,7 +13,7 @@ sources:
tables:
- name: blocks
- name: transactions
- name: tx_counts
- name: tx_count
- name: complete_streamline
database: "{{ target.database }}"
schema: streamline
@ -32,6 +32,7 @@ sources:
- name: complete_provider_prices
- name: complete_token_asset_metadata
- name: complete_token_prices
- name: hourly_prices_coin_gecko
- name: crosschain_public
database: crosschain
schema: bronze_public

2
packages.yml
View File

@ -1,3 +1,3 @@
packages:
- git: https://github.com/FlipsideCrypto/fsc-ibc.git
revision: <insert-version>
revision: v1.0.0

2
python/dbt_test_alert.py
View File

@ -82,7 +82,7 @@ def create_message(**kwargs):
"type": "button",
"text": "View Warnings",
"style": "primary",
"url": "https://github.com/FlipsideCrypto/<ibc_chain>-models/actions",
"url": "https://github.com/FlipsideCrypto/neutron-models/actions",
"confirm": {
"title": f"{kwargs['warn_count']} Warnings",
"text": "\n".join(kwargs["messages"]["warn"]) if len(kwargs["messages"]["warn"]) > 0 else "None :)",