From 70c25ac007f35399e900630c17a0eca77074cb8c Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 10:50:07 -0800 Subject: [PATCH 1/7] allow each repo to sepcify execution env, dbt ver, dbt profiles --- .github/workflows/dbt_run_template.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index cd03cf0..609e13b 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -9,24 +9,22 @@ on: required: true env: - DBT_PROFILES_DIR: ./ + DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" ACCOUNT: "${{ secrets.ACCOUNT }}" ROLE: "${{ secrets.ROLE }}" USER: "${{ secrets.USER }}" PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ secrets.REGION }}" - DATABASE_DEV: "${{ secrets.DATABASE_DEV }}" - DATABASE_PROD: "${{ secrets.DATABASE_PROD }}" - WAREHOUSE_DEV: "${{ secrets.WAREHOUSE_DEV }}" - WAREHOUSE_PROD: "${{ secrets.WAREHOUSE_PROD }}" + DATABASE: "${{ secrets.DATABASE }}" + WAREHOUSE: "${{ secrets.WAREHOUSE }}" SCHEMA: "${{ secrets.SCHEMA }}" jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: workflow + name: "${{ secrets.GITHUB_ENVIRONMENT }}" strategy: matrix: command: ${{fromJson(inputs.dbt_command)}} @@ -40,7 +38,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake cli_passthrough requests click + pip3 install dbt-snowflake==${{ secrets.DBT_VERSION }} cli_passthrough requests click dbt deps - name: Run DBT Jobs run: | From b53ae157036428530185a635d6878edc633c8323 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 13:19:26 -0800 Subject: [PATCH 2/7] env should be input param --- .github/workflows/dbt_run_template.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index 609e13b..8ab22e0 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -7,6 +7,9 @@ on: type: string description: 'dbt commands to run' required: true + environment: + type: string + description: 'github environment to get configuration values' env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" @@ -24,7 +27,7 @@ jobs: run_dbt_jobs: runs-on: ubuntu-latest environment: - name: "${{ secrets.GITHUB_ENVIRONMENT }}" + name: "${{ inputs.environment }}" strategy: matrix: command: ${{fromJson(inputs.dbt_command)}} From 90033bf7eccfd15aec56f6f5ae01900f2ae90475 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 17:52:12 -0800 Subject: [PATCH 3/7] use warehouse env as default if non provided --- .github/workflows/dbt_run_template.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index 8ab22e0..3b04b8e 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -10,6 +10,10 @@ on: environment: type: string description: 'github environment to get configuration values' + warehouse: + type: string + description: 'dbt warehouse' + default: "${{ secrets.WAREHOUSE }}" env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" @@ -20,7 +24,7 @@ env: PASSWORD: "${{ secrets.PASSWORD }}" REGION: "${{ secrets.REGION }}" DATABASE: "${{ secrets.DATABASE }}" - WAREHOUSE: "${{ secrets.WAREHOUSE }}" + WAREHOUSE: "${{ inputs.warehouse }}" SCHEMA: "${{ secrets.SCHEMA }}" jobs: From 9a46f0cfaecde1097de3a321be4e15c420f6f4d9 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 5 Jan 2023 17:56:21 -0800 Subject: [PATCH 4/7] doesnt like secrets for default value --- .github/workflows/dbt_run_template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index 3b04b8e..a250362 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -13,7 +13,6 @@ on: warehouse: type: string description: 'dbt warehouse' - default: "${{ secrets.WAREHOUSE }}" env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" From d90c166d2ea9fde2200db762369f3995fa6ea545 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Thu, 12 Jan 2023 15:12:24 -0800 Subject: [PATCH 5/7] make required --- .github/workflows/dbt_run_template.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dbt_run_template.yml b/.github/workflows/dbt_run_template.yml index a250362..faa122d 100644 --- a/.github/workflows/dbt_run_template.yml +++ b/.github/workflows/dbt_run_template.yml @@ -10,9 +10,11 @@ on: environment: type: string description: 'github environment to get configuration values' + required: true warehouse: type: string description: 'dbt warehouse' + required: true env: DBT_PROFILES_DIR: "${{ secrets.DBT_PROFILES_DIR }}" From b473c6c0c7a1f371d7de0e170c8706d32617173c Mon Sep 17 00:00:00 2001 From: austin Date: Wed, 1 Feb 2023 15:11:12 -0500 Subject: [PATCH 6/7] sample overview --- documentation/__overview__.md | 88 +++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 documentation/__overview__.md diff --git a/documentation/__overview__.md b/documentation/__overview__.md new file mode 100644 index 0000000..5a7733d --- /dev/null +++ b/documentation/__overview__.md @@ -0,0 +1,88 @@ +{% docs __overview__ %} + +# Welcome to the Flipside Crypto Models Documentation + +## **What does this documentation cover?** +The documentation included here details the design of the tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/earn) For more information on how these models are built, please see [the github repository.]( ) + +## **How do I use these docs?** +The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. + +If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. + +There is more information on how to use dbt docs in the last section of this document. + +## **Quick Links to Table Documentation** + +**Click on the links below to jump to the documentation for each schema.** + +### Core Tables (.core) + +**Dimension Tables:** +- []() +- []() + +**Fact Tables:** +- []() +- []() + +**Convenience Tables:** +- []() +- []() + +### Tables (.) + + +## **Helpful User-Defined Functions (UDFs)** + +UDFs are custom functions built by the Flipside team that can be used in your queries to make your life easier. + + + +Helpful UDFs for working with EVM data: +```sql +-- Convert a hex encoded value to an integer with ethereum.public.udf_hex_to_int(FIELD::string) +select '0xFC3C88'::string as hex_value, ethereum.public.udf_hex_to_int('0xFC3C88') as int_value +``` + +## **Data Model Overview** + +The models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** + +- Bronze: Data is loaded in from the source as a view +- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here +- Gold (or core): Final views and tables that are available publicly + +The dimension tables are sourced from a variety of on-chain and off-chain sources. + +Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. + +## **Using dbt docs** +### Navigation + +You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project. + +### Database Tab + +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database. + +### Graph Exploration + +You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models. + +On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. + +Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). + +Note that you can also right-click on models to interactively filter and explore the graph. + + +### **More information** +- [Flipside](https://flipsidecrypto.xyz/earn) +- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) +- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) +- [Github]( ) +- [Query Editor Shortcuts](https://docs.flipsidecrypto.com/velocity/query-editor-shortcuts) +- [What is dbt?](https://docs.getdbt.com/docs/introduction) + +{% enddocs %} From 70d5aa3a19b135fe434abcb3888d6b0e4841a7b3 Mon Sep 17 00:00:00 2001 From: forgash_ Date: Thu, 9 Feb 2023 10:31:56 -0700 Subject: [PATCH 7/7] update urls in docs --- documentation/__overview__.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/__overview__.md b/documentation/__overview__.md index 5a7733d..caf2b70 100644 --- a/documentation/__overview__.md +++ b/documentation/__overview__.md @@ -78,11 +78,10 @@ Note that you can also right-click on models to interactively filter and explore ### **More information** -- [Flipside](https://flipsidecrypto.xyz/earn) +- [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) - [Github]( ) -- [Query Editor Shortcuts](https://docs.flipsidecrypto.com/velocity/query-editor-shortcuts) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %}