Initial setup 1 (#2)

* add gitignore

* replace xyz with eclipse

---------

Co-authored-by: desmond-hui <desmond@flipsidecryto.com>
This commit is contained in:
desmond-hui 2024-08-28 07:31:44 -07:00 committed by GitHub
parent f0ed03342e
commit b0b9bfb514
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 68 additions and 28 deletions

20
.gitignore vendored Normal file
View File

@ -0,0 +1,20 @@
target/
dbt_modules/
# newer versions of dbt use this directory instead of dbt_modules for test dependencies
dbt_packages/
logs/
.venv/
.python-version
dbt-env/
venv/
# Visual Studio Code files
*/.vscode
*.code-workspace
.history/
**/.DS_Store
.vscode/
.env
.DS_Store
.user.yml

View File

@ -1,13 +1,10 @@
# Please find and replace all instances of `xyz` with your project name.
## Profile Set Up
#### Use the following within profiles.yml
----
```yml
xyz:
eclipse:
target: dev
outputs:
dev:
@ -17,7 +14,7 @@ xyz:
user: <USERNAME>
password: <PASSWORD>
region: <REGION>
database: xyz_DEV
database: ECLIPSE_DEV
warehouse: <WAREHOUSE>
schema: silver
threads: 4
@ -73,5 +70,5 @@ dbt run --var '{"UPDATE_SNOWFLAKE_TAGS":True}' -s models/core/core__fact_swaps.s
```
select *
from table(xyz.information_schema.tag_references('xyz.core.fact_blocks', 'table'));
from table(eclipse.information_schema.tag_references('eclipse.core.fact_blocks', 'table'));
```

View File

@ -1,14 +1,14 @@
# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
name: "xyz_models"
name: "eclipse_models"
version: "1.0.0"
config-version: 2
require-dbt-version: ">=1.8.0"
# This setting configures which "profile" dbt uses for this project.
profile: "xyz"
profile: "eclipse"
# These configurations specify where dbt should look for different types of files.
# The `model-paths` config, for example, states that models in this project can be
@ -38,6 +38,10 @@ models:
relation: true
columns: true
+on_schema_change: "append_new_columns"
livequery_models:
deploy:
core:
materialized: ephemeral
vars:
"dbt_date:time_zone": GMT
@ -56,7 +60,7 @@ on-run-end:
dispatch:
- macro_namespace: dbt
search_order:
- xyz-models
- eclipse-models
- dbt_snowflake_query_tags
- dbt

View File

@ -1,12 +1,12 @@
{% macro create_sps() %}
{% if target.database == 'xyz' %}
{% if target.database == 'ECLIPSE' %}
CREATE SCHEMA IF NOT EXISTS _internal;
{{ sp_create_prod_clone('_internal') }};
{% endif %}
{% endmacro %}
{% macro enable_search_optimization(schema_name, table_name, condition = '') %}
{% if target.database == 'xyz' %}
{% if target.database == 'ECLIPSE' %}
ALTER TABLE {{ schema_name }}.{{ table_name }} ADD SEARCH OPTIMIZATION {{ condition }}
{% endif %}
{% endmacro %}

View File

@ -1,2 +1,10 @@
{% macro create_udfs() %}
{% if var("UPDATE_UDFS_AND_SPS") %}
{% set sql %}
{% if target.database != "ECLIPSE_COMMUNITY_DEV" %}
{{ create_udf_bulk_rest_api_v2() }};
{% endif %}
{% endset %}
{% do run_query(sql) %}
{% endif %}
{% endmacro %}

View File

@ -1,6 +1,6 @@
{% macro run_sp_create_prod_clone() %}
{% set clone_query %}
call xyz._internal.create_prod_clone('xyz', 'xyz_dev', 'internal_dev');
call eclipse._internal.create_prod_clone('eclipse', 'eclipse_dev', 'internal_dev');
{% endset %}
{% do run_query(clone_query) %}

View File

@ -10,7 +10,7 @@ $$
try {
snowflake.execute({sqlText: `CREATE OR REPLACE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`});
snowflake.execute({sqlText: `DROP SCHEMA IF EXISTS ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */
snowflake.execute({sqlText: `GRANT USAGE ON DATABASE ${DESTINATION_DB_NAME} TO AWS_LAMBDA_XYZ_API`});
snowflake.execute({sqlText: `GRANT USAGE ON DATABASE ${DESTINATION_DB_NAME} TO AWS_LAMBDA_ECLIPSE_API`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL SCHEMAS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON ALL FUNCTIONS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
@ -23,7 +23,7 @@ $$
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE VIEWS IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUTURE TABLES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`});
snowflake.execute({sqlText: `GRANT USAGE ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE AWS_LAMBDA_XYZ_API;`});
snowflake.execute({sqlText: `GRANT USAGE ON ALL STAGES IN DATABASE ${DESTINATION_DB_NAME} TO ROLE AWS_LAMBDA_ECLIPSE_API;`});
snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`})

View File

@ -1,10 +1,6 @@
{% macro add_database_or_schema_tags() %}
{{ set_database_tag_value(
'BLOCKCHAIN_NAME',
'xyz'
) }}
{{ set_database_tag_value(
'BLOCKCHAIN_TYPE',
'IBC'
'ECLIPSE'
) }}
{% endmacro %}

View File

@ -1,10 +1,10 @@
{% docs __overview__ %}
# Welcome to the Flipside Crypto xyz Models Documentation
# Welcome to the Flipside Crypto Eclipse Models Documentation
## **What does this documentation cover?**
The documentation included here details the design of the xyz
tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/xyz-models/)
The documentation included here details the design of the Eclipse
tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/eclipse-models/)
## **How do I use these docs?**
The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information.
@ -17,7 +17,7 @@ There is more information on how to use dbt docs in the last section of this doc
**Click on the links below to jump to the documentation for each schema.**
### Core Tables (`xyz`.`CORE`.`<table_name>`)
### Core Tables (`ECLIPSE`.`CORE`.`<table_name>`)
**Dimension Tables:**
@ -33,8 +33,7 @@ There is more information on how to use dbt docs in the last section of this doc
## **Data Model Overview**
The xyz
models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).**
The Eclipse models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).**
- Bronze: Data is loaded in from the source as a view
- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here
@ -68,7 +67,7 @@ Note that you can also right-click on models to interactively filter and explore
- [Flipside](https://flipsidecrypto.xyz/)
- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover)
- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials)
- [Github](https://github.com/FlipsideCrypto/xyz-models)
- [Github](https://github.com/FlipsideCrypto/eclipse-models)
- [What is dbt?](https://docs.getdbt.com/docs/introduction)
{% enddocs %}

View File

@ -2,7 +2,7 @@ version: 2
sources:
- name: crosschain
database: "{{ 'crosschain' if target.database == 'xyz' else 'crosschain_dev' }}"
database: "{{ 'crosschain' if target.database == 'ECLIPSE' else 'crosschain_dev' }}"
schema: core
tables:
- name: dim_date_hours

16
package-lock.yml Normal file
View File

@ -0,0 +1,16 @@
packages:
- package: calogica/dbt_expectations
version: 0.8.5
- git: https://github.com/FlipsideCrypto/fsc-utils.git
revision: eb33ac727af26ebc8a8cc9711d4a6ebc3790a107
- package: get-select/dbt_snowflake_query_tags
version: 2.5.0
- package: dbt-labs/dbt_external_tables
version: 0.8.2
- package: dbt-labs/dbt_utils
version: 1.0.0
- package: calogica/dbt_date
version: 0.7.2
- git: https://github.com/FlipsideCrypto/livequery-models.git
revision: b024188be4e9c6bc00ed77797ebdc92d351d620e
sha1_hash: 4d94cfdedbff54a9267b86ee2d14f7aa324282ab

View File

@ -1,4 +1,4 @@
xyz:
eclipse:
target: dev
outputs:
dev: