Update/package-overview-docs (#293)

* package and overview docs

* deploy new workflows and makefile

* livequery_models reinstated

* update sources

* comments

* docs

* docs

* docs

.github/workflows/dbt_deploy_new_workflows.yml

@@ -15,5 +15,5 @@ jobs:
     with:
       command_name: Deploy New Github Actions
       command: |
-        make deploy_new_github_action DBT_TARGET=prod
+        make deploy_new_gha_tasks DBT_TARGET=prod
     secrets: inherit

dbt_project.yml

@@ -57,7 +57,7 @@ models:
       relation: true
       columns: true
     +on_schema_change: "append_new_columns"
-  livequery_base:
+  livequery_models:
     +materialized: ephemeral
   fsc_evm:
     +enabled: false # disable fsc_evm package by default

makefile

@@ -4,28 +4,44 @@ cleanup_time:
 	@set -e; \
 	rm -f package-lock.yml && dbt clean && dbt deps
 
-deploy_github_actions:
+deploy_gha_workflows_table:
 	@set -e; \
-	dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \
-	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
-	dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+	echo "Collecting workflow names..." ; \
+	WORKFLOW_VALUES="" ; \
+	for file in $$(find .github/workflows -name "*.yml" -type f); do \
+		filename=$$(basename "$$file" .yml) ; \
+		if [ -z "$$WORKFLOW_VALUES" ]; then \
+			WORKFLOW_VALUES="('$$filename')" ; \
+		else \
+			WORKFLOW_VALUES="$$WORKFLOW_VALUES,('$$filename')" ; \
+		fi ; \
+	done ; \
+	echo "Found workflows: $$WORKFLOW_VALUES" ; \
+	dbt run-operation create_workflow_table --args "{\"workflow_values\": \"$$WORKFLOW_VALUES\"}" -t $(DBT_TARGET)
 
-deploy_new_github_action:
+deploy_gha_tasks:
 	@set -e; \
-	dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET); \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \
 	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
-	dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
+
+deploy_new_gha_tasks:
+	@set -e; \
+	make deploy_gha_workflows_table DBT_TARGET=$(DBT_TARGET); \
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \
+	dbt run-operation fsc_evm.create_gha_tasks --vars '{"RESUME_GHA_TASKS":True}' -t $(DBT_TARGET)
 
 deploy_livequery:
 	@set -e; \
 	dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
-	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
 	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET)
 
 deploy_chain_phase_1:
 	@set -e; \
 	read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \
-	dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
+	dbt run -m livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
 	dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
 	dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \
 	dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \
@@ -61,10 +77,8 @@ deploy_chain_phase_2:
 	else \
 		dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
 		dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
-		make deploy_github_actions DBT_TARGET=$(DBT_TARGET); \
+		make deploy_gha_tasks DBT_TARGET=$(DBT_TARGET); \
 	fi; \
-	echo "# tasks set to SUSPEND by default"; \
-	echo "# kick dbt_alter_gha_task or dbt_alter_all_gha_tasks workflow to RESUME tasks, where applicable"; \
 	echo "# wait ~10 minutes"; \
 	echo "# run deploy_chain_phase_3"
 
@@ -79,8 +93,6 @@ deploy_chain_phase_3:
 		dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \
 		dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \
 	fi; \
-	echo "# tasks set to SUSPEND by default"; \
-	echo "# kick dbt_alter_gha_task or dbt_alter_all_gha_tasks workflow to RESUME tasks, where applicable"; \
 	echo "# wait ~10 minutes"; \
 	echo "# run deploy_chain_phase_4"
 
@@ -93,7 +105,5 @@ deploy_chain_phase_4:
 		dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \
 		dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \
 	fi; \
-	echo "# tasks set to SUSPEND by default"; \
-	echo "# kick dbt_alter_gha_task or dbt_alter_all_gha_tasks workflow to RESUME tasks, where applicable"
 
-.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery deploy_chain_phase_4
+.PHONY: cleanup_time deploy_gha_workflows_table deploy_gha_tasks deploy_new_gha_tasks deploy_livequery deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_chain_phase_4

models/overview.md

@@ -0,0 +1,123 @@
+{% docs __overview__ %}
+
+# Welcome to the Flipside Crypto Base Models Documentation!
+
+## **What does this documentation cover?**
+The documentation included here details the design of the Base tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/base-models)
+
+## **How do I use these docs?**
+The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information.
+
+If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them.
+
+There is more information on how to use dbt docs in the last section of this document.
+
+## **Quick Links to Table Documentation**
+
+**Click on the links below to jump to the documentation for each schema.**
+
+### Core Tables
+
+**Dimensional Tables**
+- [dim_contracts](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__dim_contracts)
+- [dim_contract_abis](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__dim_contract_abis)
+- [dim_labels](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__dim_labels)
+
+**Fact Tables:**
+- [fact_blocks](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__fact_blocks)
+- [fact_transactions](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__fact_transactions)
+- [fact_event_logs](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__fact_event_logs)
+- [fact_traces](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__fact_traces)
+
+**Convenience Tables:**
+- [ez_decoded_event_logs](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__ez_decoded_event_logs)
+- [ez_native_transfers](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__ez_native_transfers)
+- [ez_token_transfers](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.core__ez_token_transfers)
+
+### Price Tables (base.price)
+- [dim_asset_metadata](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.price__dim_asset_metadata)
+- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly)
+- [ez_asset_metadata](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.price__ez_asset_metadata)
+- [ez_prices_hourly](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.price__ez_prices_hourly)
+
+### DeFi Tables (base.defi)
+- [ez_bridge_activity](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_bridge_activity)
+- [ez_dex_swaps](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_dex_swaps)
+- [dim_dex_liquidity_pools](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__dim_dex_liquidity_pools)
+- [ez_lending_borrows](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_lending_borrows) 
+- [ez_lending_deposits](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_lending_deposits)
+- [ez_lending_flashloans](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_lending_flashloans)
+- [ez_lending_liquidations](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_lending_liquidations)
+- [ez_lending_repayments](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_lending_repayments)
+- [ez_lending_withdraws](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.defi__ez_lending_withdraws)
+
+### Flipside Partner Tables (base.partner_name)
+
+**Vertex Tables:**
+- [dim_products](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__dim_products)
+- [ez_clearing_house_events](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_clearing_house_events)
+- [ez_liquidations](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_liquidations)
+- [ez_perp_trades](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_perp_trades)
+- [ez_spot_trades](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_spot_trades)
+- [ez_account_stats](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_account_stats)
+- [ez_edge_trades](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_edge_trades)
+- [ez_market_depth_stats](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_market_depth_stats)
+- [ez_market_stats](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_market_stats)
+- [ez_money_markets](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.vertex__ez_money_markets)
+
+### NFT Tables (base.nft)
+- [ez_nft_transfers](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.nft__ez_nft_transfers)
+- [ez_nft_sales](https://flipsidecrypto.github.io/base-models/#!/model/model.base_models.nft__ez_nft_sales)
+
+### Stats Tables (base.stats)
+- [ez_core_metrics_hourly](https://flipsidecrypto.github.io/base-models/#!/model/model.fsc_evm.stats__ez_core_metrics_hourly)
+
+## **Helpful User-Defined Functions (UDFs)**
+
+UDFs are custom functions built by the Flipside team that can be used in your queries to make your life easier. 
+
+Please visit [LiveQuery Functions Overview](https://flipsidecrypto.github.io/livequery-models/#!/overview) for a full list of helpful UDFs.
+
+## **Data Model Overview**
+
+The Base models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).**
+
+- Bronze: Data is loaded in from the source as a view
+- Silver: All necessary parsing, filtering, de-duping, and other transformations are done here
+- Gold (or core): Final views and tables that are available publicly
+
+The dimension tables are sourced from a variety of on-chain and off-chain sources.
+
+Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data.
+
+NOTE: Base is currently operating in it's Testnet phase. Flipside will provide Mainnet data tables once Base Mainnet is deployed. 
+
+## **Using dbt docs**
+### Navigation
+
+You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project.
+
+### Database Tab
+
+This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database.
+
+### Graph Exploration
+
+You can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.
+
+On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring.
+
+Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).
+
+Note that you can also right-click on models to interactively filter and explore the graph.
+
+
+### **More information**
+- [Flipside](https://flipsidecrypto.xyz/)
+- [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover)
+- [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials)
+- [Github](https://github.com/FlipsideCrypto/base-models)
+- [What is dbt?](https://docs.getdbt.com/docs/introduction)
+
+
+{% enddocs %}

models/sources.yml

@@ -62,6 +62,7 @@ sources:
     schema: silver
     tables:
       - name: verified_abis
+      - name: complete_event_abis
   - name: fsc_evm_admin
     database: >-
       {{ 'FSC_EVM_DEV' if '_DEV' in target.database.upper() else 'FSC_EVM' }}
@@ -74,3 +75,4 @@ sources:
     schema: silver
     tables:
       - name: logs
+      - name: decoded_logs

packages.yml

@@ -1,3 +1,3 @@
 packages:
   - git: https://github.com/FlipsideCrypto/fsc-evm.git
-    revision: v4.0.0-beta.24
+    revision: v4.0.0-beta.25