diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2769f0f --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +target/ +dbt_modules/ +# newer versions of dbt use this directory instead of dbt_modules for test dependencies +dbt_packages/ +logs/ + +.venv/ +.python-version +dbt-env/ +venv/ + +# Visual Studio Code files +*/.vscode +*.code-workspace +.history/ +**/.DS_Store +.vscode/ +.env +.DS_Store +.user.yml \ No newline at end of file diff --git a/README.md b/README.md index 1528017..873ac68 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,10 @@ - -# Please find and replace all instances of `xyz` with your project name. - ## Profile Set Up #### Use the following within profiles.yml ---- ```yml -xyz: +eclipse: target: dev outputs: dev: @@ -17,7 +14,7 @@ xyz: user: password: region: - database: xyz_DEV + database: ECLIPSE_DEV warehouse: schema: silver threads: 4 @@ -73,5 +70,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s ``` select * -from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table')); +from table(eclipse.information_schema.tag_references('eclipse.core.fact_blocks', 'table')); ``` \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index b29efc1..095daab 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,14 +1,14 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "xyz_models" +name: "eclipse_models" version: "1.0.0" config-version: 2 require-dbt-version: ">=1.8.0" # This setting configures which "profile" dbt uses for this project. -profile: "xyz" +profile: "eclipse" # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be @@ -38,6 +38,10 @@ models: relation: true columns: true +on_schema_change: "append_new_columns" + livequery_models: + deploy: + core: + materialized: ephemeral vars: "dbt_date:time_zone": GMT @@ -56,7 +60,7 @@ on-run-end: dispatch: - macro_namespace: dbt search_order: - - xyz-models + - eclipse-models - dbt_snowflake_query_tags - dbt diff --git a/macros/create_sps.sql b/macros/create_sps.sql index 3d71edb..25017df 100644 --- a/macros/create_sps.sql +++ b/macros/create_sps.sql @@ -1,12 +1,12 @@ {% macro create_sps() %} - {% if target.database == 'xyz' %} + {% if target.database == 'ECLIPSE' %} CREATE SCHEMA IF NOT EXISTS _internal; {{ sp_create_prod_clone('_internal') }}; {% endif %} {% endmacro %} {% macro enable_search_optimization(schema_name, table_name, condition = '') %} - {% if target.database == 'xyz' %} + {% if target.database == 'ECLIPSE' %} ALTER TABLE {{ schema_name }}.{{ table_name }} ADD SEARCH OPTIMIZATION {{ condition }} {% endif %} {% endmacro %} \ No newline at end of file diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 56fe0ff..dc69c7b 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,2 +1,10 @@ {% macro create_udfs() %} + {% if var("UPDATE_UDFS_AND_SPS") %} + {% set sql %} + {% if target.database != "ECLIPSE_COMMUNITY_DEV" %} + {{ create_udf_bulk_rest_api_v2() }}; + {% endif %} + {% endset %} + {% do run_query(sql) %} + {% endif %} {% endmacro %} diff --git a/macros/run_sp_create_prod_clone.sql b/macros/run_sp_create_prod_clone.sql index 1868683..97389ce 100644 --- a/macros/run_sp_create_prod_clone.sql +++ b/macros/run_sp_create_prod_clone.sql @@ -1,6 +1,6 @@ {% macro run_sp_create_prod_clone() %} {% set clone_query %} -call xyz._internal.create_prod_clone('xyz', 'xyz_dev', 'internal_dev'); +call eclipse._internal.create_prod_clone('eclipse', 'eclipse_dev', 'internal_dev'); {% endset %} {% do run_query(clone_query) %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql index ad4d359..47c7ce4 100644 --- a/macros/sp_create_prod_clone.sql +++ b/macros/sp_create_prod_clone.sql @@ -10,7 +10,7 @@ $$ try { snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */ - snowflake.execute({sqlText: `GRANT USAGE ON DATABASE ${DESTINATION_DB_NAME} TO AWS_LAMBDA_XYZ_API`}); + snowflake.execute({sqlText: `GRANT USAGE ON DATABASE ${DESTINATION_DB_NAME} TO AWS_LAMBDA_ECLIPSE_API`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); @@ -23,7 +23,7 @@ $$ snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}); - snowflake.execute({sqlText: `GRANT USAGE ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE AWS_LAMBDA_XYZ_API;`}); + snowflake.execute({sqlText: `GRANT USAGE ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE AWS_LAMBDA_ECLIPSE_API;`}); snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}) diff --git a/macros/tags/add_database_or_schema_tags.sql b/macros/tags/add_database_or_schema_tags.sql index 184e0bd..b8e931e 100644 --- a/macros/tags/add_database_or_schema_tags.sql +++ b/macros/tags/add_database_or_schema_tags.sql @@ -1,10 +1,6 @@ {% macro add_database_or_schema_tags() %} {{ set_database_tag_value( 'BLOCKCHAIN_NAME', - 'xyz' - ) }} - {{ set_database_tag_value( - 'BLOCKCHAIN_TYPE', - 'IBC' + 'ECLIPSE' ) }} {% endmacro %} diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 588ffbd..8f22081 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -1,10 +1,10 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto xyz Models Documentation +# Welcome to the Flipside Crypto Eclipse Models Documentation ## **What does this documentation cover?** -The documentation included here details the design of the xyz - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/xyz-models/) +The documentation included here details the design of the Eclipse + tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/eclipse-models/) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -17,7 +17,7 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (`xyz`.`CORE`.``) +### Core Tables (`ECLIPSE`.`CORE`.``) **Dimension Tables:** @@ -33,8 +33,7 @@ There is more information on how to use dbt docs in the last section of this doc ## **Data Model Overview** -The xyz - models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +The Eclipse models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -68,7 +67,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/xyz-models) +- [Github](https://github.com/FlipsideCrypto/eclipse-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index e31d5cc..ace16e1 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -2,7 +2,7 @@ version: 2 sources: - name: crosschain - database: "{{ 'crosschain' if target.database == 'xyz' else 'crosschain_dev' }}" + database: "{{ 'crosschain' if target.database == 'ECLIPSE' else 'crosschain_dev' }}" schema: core tables: - name: dim_date_hours diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..1a8ba22 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,16 @@ +packages: + - package: calogica/dbt_expectations + version: 0.8.5 + - git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: eb33ac727af26ebc8a8cc9711d4a6ebc3790a107 + - package: get-select/dbt_snowflake_query_tags + version: 2.5.0 + - package: dbt-labs/dbt_external_tables + version: 0.8.2 + - package: dbt-labs/dbt_utils + version: 1.0.0 + - package: calogica/dbt_date + version: 0.7.2 + - git: https://github.com/FlipsideCrypto/livequery-models.git + revision: b024188be4e9c6bc00ed77797ebdc92d351d620e +sha1_hash: 4d94cfdedbff54a9267b86ee2d14f7aa324282ab diff --git a/profiles.yml b/profiles.yml index 0661f98..decf241 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,4 +1,4 @@ -xyz: +eclipse: target: dev outputs: dev: