diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 1e46ed5..34ff646 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,3 +1,2 @@ workflow_name,workflow_schedule -dbt_run_streamline_chainhead,"" -dbt_run_scheduled_main,"" \ No newline at end of file +dbt_run_streamline_history,"21 */2 * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index c4ba09d..fa57d1a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,12 +1,12 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "_models" # replace with the name of the chain +name: "bob_models" # replace with the name of the chain version: "1.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: "" # replace with the name of the chain +profile: "bob" # replace with the name of the chain # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be @@ -26,7 +26,7 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_packages" tests: - _models: # replace with the name of the chain + bob_models: # replace with the name of the chain +store_failures: true # all tests fsc_evm: +store_failures: true @@ -41,7 +41,7 @@ on-run-end: dispatch: - macro_namespace: dbt search_order: - - -models + - bob_models - dbt_snowflake_query_tags - dbt @@ -53,7 +53,7 @@ query-comment: # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - _models: # replace with the name of the chain + bob_models: # replace with the name of the chain +copy_grants: true +persist_docs: relation: true @@ -72,14 +72,10 @@ models: +enabled: false # disable main_package by default, enabled other packages as needed core: +enabled: true # enable subpackages, as needed - github_actions: - +enabled: true - labels: + utils: +enabled: true prices: +enabled: true - utils: - +enabled: true vars: "dbt_date:time_zone": GMT @@ -103,24 +99,47 @@ vars: config: # The keys correspond to dbt profiles and are case sensitive dev: - API_INTEGRATION: - EXTERNAL_FUNCTION_URI: + API_INTEGRATION: AWS_BOB_API_STG_V2 + EXTERNAL_FUNCTION_URI: w8mcqakc59.execute-api.us-east-1.amazonaws.com/stg/ ROLES: - - AWS_LAMBDA__API # replace with the name of the chain + - AWS_LAMBDA_BOB_API - INTERNAL_DEV prod: - API_INTEGRATION: - EXTERNAL_FUNCTION_URI: + API_INTEGRATION: AWS_BOB_API_PROD_V2 + EXTERNAL_FUNCTION_URI: dibb1bdcd1.execute-api.us-east-1.amazonaws.com/prod/ ROLES: - - AWS_LAMBDA__API # replace with the name of the chain + - AWS_LAMBDA_BOB_API - INTERNAL_DEV - - DBT_CLOUD_ # replace with the name of the chain + - DBT_CLOUD_BOB #### STREAMLINE 2.0 END #### #### FSC_EVM BEGIN #### + GLOBAL_PROD_DB_NAME: "bob" + GLOBAL_NODE_SECRET_PATH: "Vault/prod/bob/drpc/mainnet" + GLOBAL_BLOCKS_PER_HOUR: 1800 + # Please visit https://github.com/FlipsideCrypto/fsc-evm/wiki for more information on required and optional variables + BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True + RECEIPTS_REALTIME_NEW_BUILD: True + TRACES_REALTIME_NEW_BUILD: True + + BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True + BLOCKS_TRANSACTIONS_HISTORY_PRODUCER_BATCH_SIZE: 10000 + BLOCKS_TRANSACTIONS_HISTORY_WORKER_BATCH_SIZE: 2000 + BLOCKS_TRANSACTIONS_HISTORY_SQL_LIMIT: 500000 + + RECEIPTS_HISTORY_NEW_BUILD: True + RECEIPTS_HISTORY_PRODUCER_BATCH_SIZE: 10000 + RECEIPTS_HISTORY_WORKER_BATCH_SIZE: 2000 + RECEIPTS_HISTORY_SQL_LIMIT: 500000 + + TRACES_HISTORY_NEW_BUILD: True + TRACES_HISTORY_PRODUCER_BATCH_SIZE: 2000 + TRACES_HISTORY_WORKER_BATCH_SIZE: 2000 + TRACES_HISTORY_SQL_LIMIT: 500000 + #### FSC_EVM END #### \ No newline at end of file diff --git a/makefile b/makefile index 99710c1..8f09980 100644 --- a/makefile +++ b/makefile @@ -11,15 +11,15 @@ cleanup_time: deploy_streamline_tables: rm -f package-lock.yml && dbt clean && dbt deps ifeq ($(findstring dev,$(DBT_TARGET)),dev) - dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:bronze_core" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET) else - dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:bronze_core" -t $(DBT_TARGET) endif - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET) deploy_streamline_requests: rm -f package-lock.yml && dbt clean && dbt deps - dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) + dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_complete_receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) deploy_github_actions: dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET) @@ -41,4 +41,8 @@ else dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET) endif -.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action \ No newline at end of file +kickoff_backfill: + rm -f package-lock.yml && dbt clean && dbt deps + dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_history_receipts" "fsc_evm,tag:streamline_core_complete_receipts" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET) + +.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time deploy_new_github_action kickoff_backfill \ No newline at end of file diff --git a/models/__overview__.md b/models/__overview__.md index 430f616..5666c5a 100644 --- a/models/__overview__.md +++ b/models/__overview__.md @@ -1,9 +1,9 @@ {% docs __overview__ %} -# Welcome to the Flipside Crypto Core Models Documentation! +# Welcome to the Flipside Crypto BoB Models Documentation! ## **What does this documentation cover?** -The documentation included here details the design of the Core tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/-models) +The documentation included here details the design of the Core tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/bob-models) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -16,22 +16,22 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (.core) +### Core Tables (bob.core) **Dimension Tables:** -- [dim_labels](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__dim_labels) +- [dim_labels](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.core__dim_labels) **Fact Tables:** -- [fact_blocks](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_blocks) -- [fact_event_logs](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_event_logs) -- [fact_transactions](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_transactions) -- [fact_traces](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_traces) +- [fact_blocks](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.core__fact_blocks) +- [fact_event_logs](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.core__fact_event_logs) +- [fact_transactions](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.core__fact_transactions) +- [fact_traces](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.core__fact_traces) -### Price Tables (.price) -- [dim_asset_metadata](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__dim_asset_metadata) -- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) -- [ez_asset_metadata](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__ez_asset_metadata) -- [ez_prices_hourly](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__ez_prices_hourly) +### Price Tables (bob.price) +- [dim_asset_metadata](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.price__dim_asset_metadata) +- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) +- [ez_asset_metadata](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.price__ez_asset_metadata) +- [ez_prices_hourly](https://flipsidecrypto.github.io/bob-models/#!/model/model.fsc_evm.price__ez_prices_hourly) ## **Helpful User-Defined Functions (UDFs)** @@ -75,7 +75,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Data Studio](https://flipsidecrypto.xyz/studio) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/-models) +- [Github](https://github.com/FlipsideCrypto/bob-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/packages.yml b/packages.yml index 52e2031..d424484 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: + revision: v3.22.1 \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index 229f691..9549dca 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,4 +1,4 @@ -: +bob: target: prod outputs: dev: