diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml deleted file mode 100644 index 8f00313..0000000 --- a/.github/workflows/dbt_alter_gha_task.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: dbt_alter_gha_task -run-name: dbt_alter_gha_task - -on: - workflow_dispatch: - branches: - - "main" - inputs: - workflow_name: - type: string - description: Name of the workflow to perform the action on, no .yml extension - required: true - task_action: - type: choice - description: Action to perform - required: true - options: - - SUSPEND - - RESUME - default: SUSPEND - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 - with: - workflow_name: | - ${{ inputs.workflow_name }} - task_action: | - ${{ inputs.task_action }} - environment: workflow_prod - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_operation_reorg.yml b/.github/workflows/dbt_run_operation_reorg.yml deleted file mode 100644 index 8f53096..0000000 --- a/.github/workflows/dbt_run_operation_reorg.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: dbt_run_operation_reorg -run-name: dbt_run_operation_reorg - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: List reorg models - id: list_models - run: | - reorg_model_list=$(dbt list --select "_models,tag:reorg" --resource-type model --output name | grep '__' | awk -F'.' '{print $NF}' | tr '\n' ',' | sed 's/,$//') - echo "model_list=$reorg_model_list" >> $GITHUB_OUTPUT - - - name: Execute block_reorg macro - run: | - dbt run-operation fsc_utils.block_reorg --args "{reorg_model_list: '${{ steps.list_models.outputs.model_list }}', hours: '12'}" && awk '/SQL status/ {print; next} /DELETE FROM/{getline; print} /\/\* {/ {print}' logs/dbt.log \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_abis.yml b/.github/workflows/dbt_run_scheduled_abis.yml index 61a044c..7046f1e 100644 --- a/.github/workflows/dbt_run_scheduled_abis.yml +++ b/.github/workflows/dbt_run_scheduled_abis.yml @@ -42,4 +42,8 @@ jobs: - name: Run ABI Models run: | - dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" \ No newline at end of file + dbt run -m "fsc_evm,tag:silver_abis" "fsc_evm,tag:gold_abis" + + - name: Kick off decoded logs history, if there are new ABIs from users + run: | + dbt run-operation fsc_evm.run_decoded_logs_history \ No newline at end of file diff --git a/.github/workflows/dbt_run_heal_models.yml b/.github/workflows/dbt_run_scheduled_decoded_logs.yml similarity index 71% rename from .github/workflows/dbt_run_heal_models.yml rename to .github/workflows/dbt_run_scheduled_decoded_logs.yml index 834347b..9b5c274 100644 --- a/.github/workflows/dbt_run_heal_models.yml +++ b/.github/workflows/dbt_run_scheduled_decoded_logs.yml @@ -1,11 +1,11 @@ -name: dbt_run_heal_models -run-name: dbt_run_heal_models +name: dbt_run_scheduled_decoded_logs +run-name: dbt_run_scheduled_decoded_logs on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -24,7 +24,7 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -40,6 +40,6 @@ jobs: pip install -r requirements.txt dbt deps - - name: Run Heal Models + - name: Update the silver and gold tables with the latest decoded logs run: | - dbt run -m "_models,tag:heal" --vars '{"HEAL_MODEL":True}' \ No newline at end of file + dbt run -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:silver_decoded_logs" "fsc_evm,tag:gold_decoded_logs" \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml index e96788e..f40a4c0 100644 --- a/.github/workflows/dbt_run_scheduled_main.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -42,8 +42,8 @@ jobs: - name: Run Main Models run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" - - - name: Run Streamline Models + dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_confirm_blocks" "fsc_evm,tag:bronze_abis" "fsc_evm,tag:nft_core" "fsc_evm,tag:bronze_core" --exclude "fsc_evm,tag:ez_prices_model" + + - name: Send new logs for decoding run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:bronze_decoded_logs" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index ed7781e..a82bab9 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -42,7 +42,7 @@ jobs: - name: Run Chainhead Models run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_complete_confirm_blocks" "fsc_evm,tag:streamline_core_realtime_confirm_blocks" - name: Run Chainhead Tests run: | diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_streamline_decoded_logs_history.yml similarity index 66% rename from .github/workflows/dbt_run_scheduled_curated.yml rename to .github/workflows/dbt_run_streamline_decoded_logs_history.yml index d430226..baec4d0 100644 --- a/.github/workflows/dbt_run_scheduled_curated.yml +++ b/.github/workflows/dbt_run_streamline_decoded_logs_history.yml @@ -1,11 +1,11 @@ -name: dbt_run_scheduled_curated -run-name: dbt_run_scheduled_curated +name: dbt_run_streamline_decoded_logs_history +run-name: dbt_run_streamline_decoded_logs_history on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -24,7 +24,7 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: @@ -40,6 +40,10 @@ jobs: pip install -r requirements.txt dbt deps - - name: Run Curated Models + - name: Update complete table run: | - dbt run -m "_models,tag:curated" "fsc_evm,tag:curated" \ No newline at end of file + dbt run -m "fsc_evm,tag:streamline_decoded_logs_complete" + + - name: Decode historical logs + run: | + dbt run-operation fsc_evm.decoded_logs_history --vars '{"STREAMLINE_INVOKE_STREAMS":True}' \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder.yml b/.github/workflows/dbt_run_streamline_decoder.yml deleted file mode 100644 index ee57d05..0000000 --- a/.github/workflows/dbt_run_streamline_decoder.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_streamline_decoder -run-name: dbt_run_streamline_decoder - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "fsc_evm,tag:decoded_logs" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml deleted file mode 100644 index 1163979..0000000 --- a/.github/workflows/dbt_run_streamline_decoder_history.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: dbt_run_streamline_decoder_history -run-name: dbt_run_streamline_decoder_history - -on: - workflow_dispatch: - branches: - - "main" - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m "fsc_evm,tag:streamline_decoded_logs_complete" "fsc_evm,tag:streamline_decoded_logs_history" \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index 2aeea57..935cef2 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -42,8 +42,8 @@ jobs: - name: Build Recent Testing Views run: | - dbt run -m "fsc_evm,tag:recent_test" + dbt run -m "fsc_evm,tag:recent_test" --exclude "fsc_evm,tag:ez_prices_model" - name: Run Recent Tests run: | - dbt test -m "fsc_evm,tag:recent_test" \ No newline at end of file + dbt test -m "fsc_evm,tag:recent_test" --exclude "fsc_evm,tag:ez_prices_model" \ No newline at end of file diff --git a/README.md b/README.md index c53e43d..73e2842 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ ## Repo Set Up 1. Create a new repository from the [evm-models-template](https://github.com/FlipsideCrypto/evm-models-template) -2. Update all references to `` to the new chain name, in lowercase by using find and replace +2. Update all references to `ronin` to the new chain name, in lowercase by using find and replace 3. Update the fsc-evm package version in `packages.yml` to the latest version 4. Set up the rest of the dbt project, where applicable, including but not limited to: - `dbt_project.yml` (enable/disable packages, vars, etc.) - `.github/workflows` (update tags, etc.) - `github_actions__workflows.csv` (update schedule, workflows, etc.) - - `overview.md` (update ``, table references, docs etc.) + - `overview.md` (update `ronin`, table references, docs etc.) - `sources.yml` (update schemas, tables etc.) - `requirements.txt` (update dependencies) - other files where applicable diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 1e46ed5..506fd20 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,3 +1,9 @@ workflow_name,workflow_schedule -dbt_run_streamline_chainhead,"" -dbt_run_scheduled_main,"" \ No newline at end of file +dbt_run_streamline_chainhead,"19,49 * * * *" +dbt_run_dev_refresh,"33 3 * * 1" +dbt_run_scheduled_main,"29 * * * *" +dbt_run_scheduled_decoded_logs,"37 * * * *" +dbt_test_intraday,"9 */4 * * *" +dbt_test_daily,"34 7 * * *" +dbt_run_scheduled_abis,"33 23 * * *" +dbt_run_scheduled_decoded_logs_history,"31 22 * * 6" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index c4ba09d..f0436f1 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,12 +1,12 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: "_models" # replace with the name of the chain +name: "ronin_models" # replace with the name of the chain version: "1.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: "" # replace with the name of the chain +profile: "ronin" # replace with the name of the chain # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be @@ -26,7 +26,7 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_packages" tests: - _models: # replace with the name of the chain + ronin_models: # replace with the name of the chain +store_failures: true # all tests fsc_evm: +store_failures: true @@ -41,7 +41,7 @@ on-run-end: dispatch: - macro_namespace: dbt search_order: - - -models + - ronin-models - dbt_snowflake_query_tags - dbt @@ -53,7 +53,7 @@ query-comment: # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - _models: # replace with the name of the chain + ronin_models: # replace with the name of the chain +copy_grants: true +persist_docs: relation: true @@ -69,17 +69,21 @@ models: columns: true +on_schema_change: "append_new_columns" main_package: - +enabled: false # disable main_package by default, enabled other packages as needed + +enabled: false core: - +enabled: true # enable subpackages, as needed - github_actions: +enabled: true - labels: + github_actions: +enabled: true prices: +enabled: true utils: +enabled: true + decoder_package: + +enabled: false + abis: + +enabled: true + decoded_logs: + +enabled: true vars: "dbt_date:time_zone": GMT @@ -106,16 +110,16 @@ vars: API_INTEGRATION: EXTERNAL_FUNCTION_URI: ROLES: - - AWS_LAMBDA__API # replace with the name of the chain + - AWS_LAMBDA_RONIN_API # replace with the name of the chain - INTERNAL_DEV - prod: - API_INTEGRATION: - EXTERNAL_FUNCTION_URI: - ROLES: - - AWS_LAMBDA__API # replace with the name of the chain - - INTERNAL_DEV - - DBT_CLOUD_ # replace with the name of the chain + # prod: + # API_INTEGRATION: + # EXTERNAL_FUNCTION_URI: + # ROLES: + # - AWS_LAMBDA_RONIN_API # replace with the name of the chain + # - INTERNAL_DEV + # - DBT_CLOUD_RONIN # replace with the name of the chain #### STREAMLINE 2.0 END #### @@ -123,4 +127,20 @@ vars: # Please visit https://github.com/FlipsideCrypto/fsc-evm/wiki for more information on required and optional variables + ### GLOBAL VARIABLES BEGIN ### + + ## REQUIRED + GLOBAL_PROD_DB_NAME: "ronin" + GLOBAL_NODE_SECRET_PATH: "Vault/prod/ronin/tatum/mainnet" + GLOBAL_BLOCKS_PER_HOUR: 1200 + GLOBAL_WRAPPED_ASSET_ADDRESS: '' + + ### GLOBAL VARIABLES END ### + + TRACES_FULL_RELOAD_START_BLOCK: 39200000 + + ## ABIS + # BLOCK_EXPLORER_NAME: "RoninChain" + # BLOCK_EXPLORER_ABI_URL: "https://explorer-kintsugi.roninchain.com/v2/2020/contract/""/abi" + #### FSC_EVM END #### \ No newline at end of file diff --git a/models/__overview__.md b/models/__overview__.md index 430f616..4d09d57 100644 --- a/models/__overview__.md +++ b/models/__overview__.md @@ -3,7 +3,7 @@ # Welcome to the Flipside Crypto Core Models Documentation! ## **What does this documentation cover?** -The documentation included here details the design of the Core tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/-models) +The documentation included here details the design of the Core tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/FlipsideCrypto/ronin-models) ## **How do I use these docs?** The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. @@ -16,22 +16,22 @@ There is more information on how to use dbt docs in the last section of this doc **Click on the links below to jump to the documentation for each schema.** -### Core Tables (.core) +### Core Tables (ronin.core) **Dimension Tables:** -- [dim_labels](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__dim_labels) +- [dim_labels](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__dim_labels) **Fact Tables:** -- [fact_blocks](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_blocks) -- [fact_event_logs](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_event_logs) -- [fact_transactions](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_transactions) -- [fact_traces](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.core__fact_traces) +- [fact_blocks](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_blocks) +- [fact_event_logs](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_event_logs) +- [fact_transactions](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_transactions) +- [fact_traces](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.core__fact_traces) -### Price Tables (.price) -- [dim_asset_metadata](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__dim_asset_metadata) -- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) -- [ez_asset_metadata](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__ez_asset_metadata) -- [ez_prices_hourly](https://flipsidecrypto.github.io/-models/#!/model/model.fsc_evm.price__ez_prices_hourly) +### Price Tables (ronin.price) +- [dim_asset_metadata](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__dim_asset_metadata) +- [fact_prices_ohlc_hourly](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__fact_prices_ohlc_hourly) +- [ez_asset_metadata](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__ez_asset_metadata) +- [ez_prices_hourly](https://flipsidecrypto.github.io/ronin-models/#!/model/model.fsc_evm.price__ez_prices_hourly) ## **Helpful User-Defined Functions (UDFs)** @@ -75,7 +75,7 @@ Note that you can also right-click on models to interactively filter and explore - [Flipside](https://flipsidecrypto.xyz/) - [Data Studio](https://flipsidecrypto.xyz/studio) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) -- [Github](https://github.com/FlipsideCrypto/-models) +- [Github](https://github.com/FlipsideCrypto/ronin-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) {% enddocs %} \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index 229f691..f39384b 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,4 +1,4 @@ -: +ronin: target: prod outputs: dev: