diff --git a/.github/workflows/dbt_run_core.yml b/.github/workflows/dbt_run_core.yml index 5ade8e6..b547da7 100644 --- a/.github/workflows/dbt_run_core.yml +++ b/.github/workflows/dbt_run_core.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m ,tag:core \ No newline at end of file + dbt run -m neutron,tag:core \ No newline at end of file diff --git a/.github/workflows/dbt_run_noncore.yml b/.github/workflows/dbt_run_noncore.yml index 251cee5..f3789c9 100644 --- a/.github/workflows/dbt_run_noncore.yml +++ b/.github/workflows/dbt_run_noncore.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m ,tag:noncore \ No newline at end of file + dbt run -m neutron,tag:noncore \ No newline at end of file diff --git a/.github/workflows/dbt_test_recency.yml b/.github/workflows/dbt_test_recency.yml index 3b9cb42..b73c800 100644 --- a/.github/workflows/dbt_test_recency.yml +++ b/.github/workflows/dbt_test_recency.yml @@ -43,7 +43,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt source freshness && dbt test -m ,tag:test_recency + dbt source freshness && dbt test -m neutron,tag:test_recency continue-on-error: true diff --git a/README.md b/README.md index dade39c..2d04bb1 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ ## Repo Set Up 1. Create a new repository from the [ibc-models-template](https://github.com/FlipsideCrypto/ibc-models-template) -2. Update all references to `` to the new chain name, in lowercase by using find and replace +2. Update all references to `neutron` to the new chain name, in lowercase by using find and replace 3. Update the fsc-ibc package version in `packages.yml` to the latest version 4. Set up the rest of the dbt project, where applicable, including but not limited to: - `dbt_project.yml` (enable/disable packages, vars, etc.) - `.github/workflows` (update tags, etc.) - - `overview.md` (update ``, table references, docs etc.) + - `overview.md` (update `neutron`, table references, docs etc.) - `sources.yml` (update schemas, tables etc.) - `requirements.txt` (update dependencies) - other files where applicable diff --git a/dbt_project.yml b/dbt_project.yml index 28f15f7..62d2fac 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,12 +1,12 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "_models" # replace with the name of the chain +name: "neutron_models" # replace with the name of the chain version: "1.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: "" # replace with the name of the chain +profile: "neutron" # replace with the name of the chain # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be @@ -26,8 +26,8 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_packages" tests: - _models: # replace with the name of the chain - +store_failures: true # all tests + #neutron_models: # replace with the name of the chain + #+store_failures: true # all tests fsc_ibc: +store_failures: true @@ -41,7 +41,7 @@ on-run-end: dispatch: - macro_namespace: dbt search_order: - - -models + - neutron-models - dbt_snowflake_query_tags - dbt @@ -53,14 +53,12 @@ query-comment: # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - _models: # replace with the name of the chain - +copy_grants: true - +persist_docs: - relation: true - columns: true - +on_schema_change: "append_new_columns" - livequery_base: - +materialized: ephemeral + #neutron_models: # replace with the name of the chain + # +copy_grants: true + # +persist_docs: + # relation: true + # columns: true + # +on_schema_change: "append_new_columns" fsc_ibc: +enabled: false # disable fsc_ibc package by default +copy_grants: true @@ -70,22 +68,10 @@ models: +on_schema_change: "append_new_columns" main_package: +enabled: false # disable main_package by default, enabled other packages as needed - admin: - +enabled: true core: +enabled: true # enable subpackages, as needed github_actions: - +enabled: true - labels: - +enabled: true - observability: - +enabled: true - prices: - +enabled: true - utils: - +enabled: true - scores_package: - +enabled: false + +enabled: false vars: "dbt_date:time_zone": GMT @@ -105,19 +91,19 @@ vars: config: # The keys correspond to dbt profiles and are case sensitive dev: - API_INTEGRATION: AWS__API_STG_V2 - EXTERNAL_FUNCTION_URI: .execute-api.us-east-1.amazonaws.com/stg/ + API_INTEGRATION: AWS_NEUTRON_API_STG_V2 + EXTERNAL_FUNCTION_URI: zjdy6lie8f.execute-api.us-east-1.amazonaws.com/stg/ ROLES: - - AWS_LAMBDA__API # replace with the name of the chain + - AWS_LAMBDA_NEUTRON_API # replace with the name of the chain - INTERNAL_DEV prod: - API_INTEGRATION: AWS__API_PROD_V2 + API_INTEGRATION: AWS_NEUTRON_API_PROD_V2 EXTERNAL_FUNCTION_URI: .execute-api.us-east-1.amazonaws.com/prod/ ROLES: - - AWS_LAMBDA__API # replace with the name of the chain + - AWS_LAMBDA_NEUTRON_API # replace with the name of the chain - INTERNAL_DEV - - DBT_CLOUD_ # replace with the name of the chain + - DBT_CLOUD_NEUTRON # replace with the name of the chain #### STREAMLINE 2.0 END #### diff --git a/makefile b/makefile index a55d252..e7fa408 100644 --- a/makefile +++ b/makefile @@ -8,13 +8,16 @@ deploy_livequery: @set -e; \ dbt run-operation fsc_ibc.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run -m livequery_models.deploy.core.live --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ dbt run-operation fsc_ibc.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) deploy_chain_phase_1: @set -e; \ + echo "### DEPLOYING CORE LIVEQUERY DEPENDENCIES ###"; \ dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ dbt run-operation fsc_ibc.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ dbt run-operation fsc_utils.create_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + echo "### STARTING PHASE 1 ###"; \ if [ "$(DBT_TARGET)" != "prod" ]; then \ dbt run -m "fsc_ibc,tag:phase_1" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ dbt test -m "fsc_ibc,tag:chainhead"; \ @@ -29,6 +32,7 @@ deploy_chain_phase_1: deploy_chain_phase_2: @set -e; \ + echo "### STARTING PHASE 2 ###"; \ if [ "$(DBT_TARGET)" != "prod" ]; then \ dbt run -m "fsc_ibc,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ dbt run -m "fsc_ibc,tag:streamlines,tag:realtime" "fsc_ibc,tag:streamline,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ @@ -41,6 +45,7 @@ deploy_chain_phase_2: deploy_chain_phase_3: @set -e; \ + echo "### STARTING PHASE 3 ###"; \ if [ "$(DBT_TARGET)" != "prod" ]; then \ dbt run -m "fsc_ibc,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ dbt run -m "fsc_ibc,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ @@ -55,6 +60,7 @@ deploy_chain_phase_3: deploy_chain_phase_4: @set -e; \ + echo "### STARTING PHASE 4 ###"; \ if [ "$(DBT_TARGET)" != "prod" ]; then \ dbt run -m "fsc_ibc,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ dbt run -m "fsc_ibc,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ diff --git a/models/__overview__.md b/models/__overview__.md index 7a944a8..308b217 100644 --- a/models/__overview__.md +++ b/models/__overview__.md @@ -1,9 +1,9 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto Models Documentation! +# Welcome to the Flipside Crypto neutron Models Documentation! ## **What does this documentation cover?** -The documentation included here details the design of the tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/-models) +The documentation included here details the design of the neutron tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/neutron-models) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -16,23 +16,23 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (.core) +### Core Tables (neutron.core) **Dimension Tables:** -- [dim_labels](https://flipsidecrypto.github.io//#!/model/model._models.core__dim_labels) +- [dim_labels](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__dim_labels) **Fact Tables:** -- [fact_blocks](https://flipsidecrypto.github.io//#!/model/model._models.core__fact_blocks) -- [fact_msgs](https://flipsidecrypto.github.io//#!/model/model._models.core__fact_msgs) -- [fact_msg_attributes](https://flipsidecrypto.github.io//#!/model/model._models.core__fact_msg_attributes) -- [fact_transactions](https://flipsidecrypto.github.io//#!/model/model._models.core__fact_transactions) -- [fact_transfers](https://flipsidecrypto.github.io//#!/model/model._models.core__fact_transfers) +- [fact_blocks](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_blocks) +- [fact_msgs](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_msgs) +- [fact_msg_attributes](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_msg_attributes) +- [fact_transactions](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_transactions) +- [fact_transfers](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.core__fact_transfers) -### Price Tables (.price) -- [dim_asset_metadata](https://flipsidecrypto.github.io//#!/model/model._models.price__dim_asset_metadata) -- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io//#!/model/model._models.price__fact_prices_ohlc_hourly) -- [ez_asset_metadata](https://flipsidecrypto.github.io//#!/model/model._models.price__ez_asset_metadata) -- [ez_prices_hourly](https://flipsidecrypto.github.io//#!/model/model._models.price__ez_prices_hourly) +### Price Tables (neutron.price) +- [dim_asset_metadata](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__dim_asset_metadata) +- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__fact_prices_ohlc_hourly) +- [ez_asset_metadata](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__ez_asset_metadata) +- [ez_prices_hourly](https://flipsidecrypto.github.io/neutron/#!/model/model.neutron_models.price__ez_prices_hourly) ## **Helpful User-Defined Functions (UDFs)** @@ -76,7 +76,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Data Studio](https://flipsidecrypto.xyz/studio) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/-models) +- [Github](https://github.com/FlipsideCrypto/neutron-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 43ea33d..515cf76 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -13,7 +13,7 @@ sources: tables: - name: blocks - name: transactions - - name: tx_counts + - name: tx_count - name: complete_streamline database: "{{ target.database }}" schema: streamline @@ -32,6 +32,7 @@ sources: - name: complete_provider_prices - name: complete_token_asset_metadata - name: complete_token_prices + - name: hourly_prices_coin_gecko - name: crosschain_public database: crosschain schema: bronze_public diff --git a/packages.yml b/packages.yml index edcac5f..0f1173b 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-ibc.git - revision: + revision: v1.0.0 diff --git a/python/dbt_test_alert.py b/python/dbt_test_alert.py index e5cc247..e50d97e 100644 --- a/python/dbt_test_alert.py +++ b/python/dbt_test_alert.py @@ -82,7 +82,7 @@ def create_message(**kwargs): "type": "button", "text": "View Warnings", "style": "primary", - "url": "https://github.com/FlipsideCrypto/-models/actions", + "url": "https://github.com/FlipsideCrypto/neutron-models/actions", "confirm": { "title": f"{kwargs['warn_count']} Warnings", "text": "\n".join(kwargs["messages"]["warn"]) if len(kwargs["messages"]["warn"]) > 0 else "None :)",