diff --git a/.github/workflows/dbt_run_overflowed_traces.yml b/.github/workflows/dbt_run_overflowed_traces.yml deleted file mode 100644 index 5510953..0000000 --- a/.github/workflows/dbt_run_overflowed_traces.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: dbt_run_overflowed_traces -run-name: dbt_run_overflowed_traces - -on: - workflow_dispatch: - branches: - - "main" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/fsc-evm/.github/workflows/dbt_run_template.yml@pre-release/v4-beta - with: - command_name: Run Overflow Models - command: | - dbt run -m "fsc_evm,tag:overflow,tag:traces" --vars '{"MAIN_CORE_TRACES_OVERFLOW_ENABLED":True}' - secrets: inherit \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 22a53ed..db6694b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -59,7 +59,7 @@ models: relation: true columns: true +on_schema_change: "append_new_columns" - livequery_models: + livequery_base: +materialized: ephemeral fsc_evm: +enabled: false # disable fsc_evm package by default @@ -70,16 +70,32 @@ models: +on_schema_change: "append_new_columns" main_package: +enabled: false # disable main_package by default, enabled other packages as needed + admin: + +enabled: true core: +enabled: true # enable subpackages, as needed github_actions: +enabled: true labels: +enabled: true + observability: + +enabled: true prices: +enabled: true utils: +enabled: true + decoder_package: + +enabled: false + abis: + +enabled: true + decoded_logs: + +enabled: true + curated_package: + +enabled: false + stats: + +enabled: true + scores_package: + +enabled: true vars: "dbt_date:time_zone": GMT @@ -100,15 +116,15 @@ vars: config: # The keys correspond to dbt profiles and are case sensitive dev: - API_INTEGRATION: - EXTERNAL_FUNCTION_URI: + API_INTEGRATION: AWS__API_STG_V2 + EXTERNAL_FUNCTION_URI: .execute-api.us-east-1.amazonaws.com/stg/ ROLES: - AWS_LAMBDA__API # replace with the name of the chain - INTERNAL_DEV prod: - API_INTEGRATION: - EXTERNAL_FUNCTION_URI: + API_INTEGRATION: AWS__API_PROD_V2 + EXTERNAL_FUNCTION_URI: .execute-api.us-east-1.amazonaws.com/prod/ ROLES: - AWS_LAMBDA__API # replace with the name of the chain - INTERNAL_DEV diff --git a/makefile b/makefile index 69c29ae..b9308d4 100644 --- a/makefile +++ b/makefile @@ -1,41 +1,99 @@ DBT_TARGET ?= dev cleanup_time: + @set -e; \ rm -f package-lock.yml && dbt clean && dbt deps deploy_github_actions: - dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) + @set -e; \ + dbt run -s livequery_base.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) deploy_new_github_action: - dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET) + @set -e; \ + dbt run-operation fsc_evm.drop_github_actions_schema -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET); \ dbt run-operation fsc_evm.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET) deploy_livequery: - dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) + @set -e; \ + dbt run-operation fsc_evm.drop_livequery_schemas --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) deploy_chain_phase_1: - dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET) - dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET) - dbt run -m "fsc_evm,tag:phase_1" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET) - # kick chainhead workflow - # wait ~10 minutes - # run deploy_chain_phase_2 + @set -e; \ + read -p "Exclude receipts_by_hash? [y/n] " receipts_by_hash; \ + dbt run -m livequery_base.deploy.core --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.livequery_grants --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS": true}' -t $(DBT_TARGET); \ + dbt run-operation fsc_evm.call_sample_rpc_node -t $(DBT_TARGET); \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + if [ "$$receipts_by_hash" = "n" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True, "MAIN_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + fi; \ + else \ + if [ "$$receipts_by_hash" = "n" ]; then \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_1" --exclude "fsc_evm,tag:receipts_by_hash" --full-refresh --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "GLOBAL_STREAMLINE_FR_ENABLED": true}' -t $(DBT_TARGET); \ + dbt test -m "fsc_evm,tag:chainhead"; \ + dbt run -m "fsc_evm,tag:streamline,tag:core,tag:complete" "fsc_evm,tag:streamline,tag:core,tag:realtime" --exclude "fsc_evm,tag:receipts_by_hash" "fsc_evm,tag:confirm_blocks" --vars '{"MAIN_SL_NEW_BUILD_ENABLED": true, "STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + fi; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_2" deploy_chain_phase_2: - dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) - make deploy_github_actions -t $(DBT_TARGET) - # tasks set to SUSPEND by default - # kick alter_gha_task workflow to RESUME individual tasks, as needed + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:streamline,tag:abis,tag:realtime" "fsc_evm,tag:streamline,tag:abis,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + make deploy_github_actions DBT_TARGET=$(DBT_TARGET); \ + fi; \ + echo "# tasks set to SUSPEND by default"; \ + echo "# kick dbt_alter_gha_task or dbt_alter_all_gha_tasks workflow to RESUME tasks, where applicable"; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_3" deploy_chain_phase_3: - dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_BRONZE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true}' -t $(DBT_TARGET) - # kick alter_gha_task workflow to RESUME individual tasks, as needed + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_2" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true, "GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True, "DECODER_SL_TESTING_LIMIT": 500}' -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_2" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_3" --full-refresh --vars '{"GLOBAL_STREAMLINE_FR_ENABLED": true, "GLOBAL_SILVER_FR_ENABLED": true, "GLOBAL_GOLD_FR_ENABLED": true, "GLOBAL_NEW_BUILD_ENABLED": true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:silver,tag:abis" "fsc_evm,tag:streamline,tag:decoded_logs,tag:realtime" "fsc_evm,tag:streamline,tag:decoded_logs,tag:complete" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET); \ + fi; \ + echo "# tasks set to SUSPEND by default"; \ + echo "# kick dbt_alter_gha_task or dbt_alter_all_gha_tasks workflow to RESUME tasks, where applicable"; \ + echo "# wait ~10 minutes"; \ + echo "# run deploy_chain_phase_4" -.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery \ No newline at end of file +deploy_chain_phase_4: + @set -e; \ + if [ "$(DBT_TARGET)" != "prod" ]; then \ + dbt run -m "fsc_evm,tag:phase_3" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":true}' -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + else \ + dbt run -m "fsc_evm,tag:phase_3" -t $(DBT_TARGET); \ + dbt run -m "fsc_evm,tag:phase_4" --full-refresh -t $(DBT_TARGET); \ + fi; \ + echo "# tasks set to SUSPEND by default"; \ + echo "# kick dbt_alter_gha_task or dbt_alter_all_gha_tasks workflow to RESUME tasks, where applicable" + +.PHONY: deploy_github_actions cleanup_time deploy_new_github_action deploy_chain_phase_1 deploy_chain_phase_2 deploy_chain_phase_3 deploy_livequery deploy_chain_phase_4 \ No newline at end of file diff --git a/models/__overview__.md b/models/__overview__.md index b2b78ec..94661dc 100644 --- a/models/__overview__.md +++ b/models/__overview__.md @@ -3,7 +3,7 @@ # Welcome to the Flipside Crypto Models Documentation! ## **What does this documentation cover?** -The documentation included here details the design of the Core tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/-models) +The documentation included here details the design of the tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/-models) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -19,19 +19,29 @@ There is more information on how to use dbt docs in the last section of this doc ### Core Tables (.core) **Dimension Tables:** -- [dim_labels](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__dim_labels) +- [dim_labels](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__dim_labels) +- [dim_contracts](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__dim_contracts) +- [dim_contract_abis](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__dim_contract_abis) **Fact Tables:** -- [fact_blocks](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_blocks) -- [fact_event_logs](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_event_logs) -- [fact_transactions](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_transactions) -- [fact_traces](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_traces) +- [fact_blocks](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__fact_blocks) +- [fact_event_logs](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__fact_event_logs) +- [fact_transactions](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__fact_transactions) +- [fact_traces](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__fact_traces) +**Convenience Tables:** +- [ez_decoded_event_logs](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__ez_decoded_event_logs) +- [ez_native_transfers](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__ez_native_transfers) +- [ez_token_transfers](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.core__ez_token_transfers) + ### Price Tables (.price) -- [dim_asset_metadata](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__dim_asset_metadata) -- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) -- [ez_asset_metadata](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__ez_asset_metadata) -- [ez_prices_hourly](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__ez_prices_hourly) +- [dim_asset_metadata](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.price__dim_asset_metadata) +- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) +- [ez_asset_metadata](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.price__ez_asset_metadata) +- [ez_prices_hourly](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.price__ez_prices_hourly) + +### NFT Tables (.nft) +- [ez_nft_transfers](https://flipsidecrypto.github.io/mantle-models/#!/model/model.fsc_evm.nft__ez_nft_transfers) ## **Helpful User-Defined Functions (UDFs)** diff --git a/models/sources.yml b/models/sources.yml index 1b7ec4f..28a3a30 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -8,7 +8,8 @@ sources: - name: workflows - name: bronze_streamline database: streamline - schema: "{{ target.database }}" + schema: | + {{ target.database.upper() | replace('_DEV', '') ~ '_DEV' if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else target.database.upper() | replace('_DEV', '') }} tables: - name: blocks - name: transactions