Merge pull request #29 from FlipsideCrypto/AN-5278/core-fsc-evm-upgrade

AN-5278/core-fsc-evm-upgrade
2026-02-06 15:56:48 +00:00 · 2024-10-30 09:12:20 -06:00 · 2024-10-30 09:12:20 -06:00 · 07f55040db
commit 07f55040db
parent 6df469908f 3a979561c0
7 changed files with 117 additions and 68 deletions
--- a/.github/workflows/dbt_run_scheduled_main.yml
+++ b/.github/workflows/dbt_run_scheduled_main.yml
@ -0,0 +1,44 @@
+name: dbt_run_scheduled_main
+run-name: dbt_run_scheduled_main
+
+on:
+  workflow_dispatch:
+    branches:
+      - "main"
+
+env:
+  DBT_PROFILES_DIR: ./
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment:
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m "core_models,tag:core" "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels"
--- a/.github/workflows/dbt_run_streamline_chainhead.yml
+++ b/.github/workflows/dbt_run_streamline_chainhead.yml
@ -41,4 +41,8 @@ jobs:
          dbt deps
      - name: Run DBT Jobs
        run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime"
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash"
+
+      - name: Test Chainhead
+        run: |
+          dbt test -m "fsc_evm,tag:chainhead"
--- a/.github/workflows/dbt_run_streamline_history.yml
+++ b/.github/workflows/dbt_run_streamline_history.yml
@ -41,4 +41,4 @@ jobs:
          dbt deps
      - name: Run DBT Jobs
        run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history"
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_history_receipts_by_hash"
--- a/.github/workflows/dbt_run_scheduled_core.yml
+++ b/.github/workflows/dbt_run_scheduled_core.yml
@ -1,11 +1,11 @@
-name: dbt_run_scheduled_core
-run-name: dbt_run_scheduled_core
+name: dbt_test_intraday
+run-name: dbt_test_intraday

 on:
  workflow_dispatch:
    branches:
      - "main"
-
+    
 env:
  DBT_PROFILES_DIR: ./

@ -21,11 +21,13 @@ env:
 concurrency:
  group: ${{ github.workflow }}

+
+  
 jobs:
  run_dbt_jobs:
    runs-on: ubuntu-latest
-    environment:
-      name: workflow_prod
+    environment: 
+      name: workflow_test

    steps:
      - uses: actions/checkout@v3
@ -41,4 +43,4 @@ jobs:
          dbt deps
      - name: Run DBT Jobs
        run: |
-          dbt run -m "fsc_evm,tag:core" "core_models,tag:core"
+          dbt test -m "fsc_evm,tag:recent_test"
--- a/data/github_actions__workflows.csv
+++ b/data/github_actions__workflows.csv
@ -1,4 +1,5 @@
 workflow_name,workflow_schedule
 dbt_run_streamline_chainhead,"23,53 * * * *"
-dbt_run_scheduled_core,"1,31 * * * *"
+dbt_run_scheduled_main,"1,31 * * * *"
+dbt_test_intraday,"39 */4 * * *"
 dbt_run_streamline_history,"22 * * * *"
--- a/dbt_project.yml
+++ b/dbt_project.yml
@ -17,6 +17,7 @@ test-paths: ["tests"]
 seed-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
+docs-paths: ["dbt_packages/fsc_evm/doc_descriptions", "models/doc_descriptions"]

 target-path: "target" # directory which will store compiled SQL files
 clean-targets: # directories to be removed by `dbt clean`
@ -25,7 +26,10 @@ clean-targets: # directories to be removed by `dbt clean`
  - "dbt_packages"

 tests:
-  +store_failures: true # all tests
+  core_models:
+    +store_failures: true # all tests
+  fsc_evm:
+    +store_failures: true

 on-run-start:
  - "{{ fsc_evm.create_sps() }}"
@ -49,14 +53,28 @@ query-comment:
 # Full documentation: https://docs.getdbt.com/docs/configuring-models

 models:
-  +copy_grants: true
-  +persist_docs:
-    relation: true
-    columns: true
-  +on_schema_change: "append_new_columns"
-  
+  core_models:
+    +copy_grants: true
+    +persist_docs:
+      relation: true
+      columns: true
+    +on_schema_change: "append_new_columns"
  livequery_models:
    +materialized: ephemeral
+  fsc_evm:
+    +enabled: false
+    main_package:
+      +enabled: false
+      core:
+        +enabled: true
+      github_actions:
+        +enabled: true
+      labels:
+        +enabled: true
+      prices:
+        +enabled: true
+      utils:
+        +enabled: true

 vars:
  "dbt_date:time_zone": GMT
@ -98,46 +116,34 @@ vars:

 #### FSC_EVM BEGIN ####

-  ### GLOBAL VARIABLES
+  ### GLOBAL VARIABLES BEGIN ###
+  ## REQUIRED
  GLOBAL_PROD_DB_NAME: 'core'
  GLOBAL_NODE_SECRET_PATH: 'Vault/prod/core/ankr/mainnet'
  GLOBAL_BLOCKS_PER_HOUR: 1200

-  ### OPTIONAL
-  # GLOBAL_NODE_URL: '{Service}/{Authentication}' is default
-  # GLOBAL_PROD_NETWORK: 'mainnet' ### default is 'mainnet' (move comment to wiki)
-  GLOBAL_USES_RECEIPTS_BY_HASH: True ### default is False (move comment to wiki)
-  GLOBAL_USES_BASE_FEE: False ### default is True (move comment to wiki)
-  GLOBAL_AVG_TXS_PER_BLOCK: 20 ### only used if USES_RECEIPTS_BY_HASH is True (move comment to wiki)
-  GLOBAL_BLOCKS_TOTAL_TO_BACKFILL: 19000000 ### high estimate for total blocks to backfill (move comment to wiki)
-  GLOBAL_START_UP_BLOCK: 18343780 ### default is 0 (move comment to wiki)
-  GLOBAL_MAX_SEQUENCE_NUMBER: 50000000 ### default is 100000000 blocks. (move comment to wiki)
+  ## OPTIONAL
+  GLOBAL_USES_RECEIPTS_BY_HASH: True
+  GLOBAL_USES_BASE_FEE: False
+  GLOBAL_AVG_TXS_PER_BLOCK: 20
+  GLOBAL_BLOCKS_TOTAL_TO_BACKFILL: 19000000
+  GLOBAL_START_UP_BLOCK: 18343780
+  GLOBAL_MAX_SEQUENCE_NUMBER: 50000000

-  ### CORE MODEL PARAMETERS BEGIN ###
+  ### GLOBAL VARIABLES END ###

-  TRACES_FULL_RELOAD_START_BLOCK: 20000000 ### default is 0 (move comment to wiki)
+  ### MAIN_PACKAGE VARIABLES BEGIN ###

-  ### PRICES ###
-  PRICES_SYMBOLS: 'CORE' ### REQUIRED 
-  PRICES_PLATFORMS: ['Core', 'core'] ### REQUIRED
-  # PRICES_BLOCKCHAINS: ### OPTIONAL (defaults to 'GLOBAL_PROD_DB_NAME')
-  # PRICES_TOKEN_ADDRESSES: ### OPTIONAL
-  
-  ### LABELS ###
-  # LABELS_BLOCKCHAINS: ### OPTIONAL (defaults to 'GLOBAL_PROD_DB_NAME')
+  ### CORE ###
+  ## REQUIRED
+  TRACES_FULL_RELOAD_START_BLOCK: 20000000

-  ### CORE MODEL PARAMETERS END ###
-
-#### FSC_EVM END ####
-
-#### STREAMLINE PARAMETERS BEGIN - SEE abc.xyz FOR ALL AVAILABLE PARAMETERS ####
-
-  ### OPTIONAL 
+  ## OPTIONAL 
  BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True
  RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True
  TRACES_REALTIME_NEW_BUILD: True
  CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True
-  
+
  BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True

  RECEIPTS_BY_HASH_HISTORY_NEW_BUILD: True
@ -153,19 +159,21 @@ vars:
  TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC, block_number DESC'

  CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True
+
+  ### PRICES ###
+  ## REQUIRED
+  PRICES_NATIVE_SYMBOLS: 'CORE'
+  PRICES_PROVIDER_PLATFORMS: ['Core', 'core']
  
-#### STREAMLINE PARAMETERS END ####
+  ### LABELS ###

-# MOVE TO WIKI AND DOCUMENT THOROUGHLY, THEN REMOVE FROM HERE
-  # BLOCKS_TRANSACTIONS_REALTIME_SQL_LIMIT: 4800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki)
-  # BLOCKS_TRANSACTIONS_REALTIME_PRODUCER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki)
-  # BLOCKS_TRANSACTIONS_REALTIME_WORKER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR / lambdas in model config (1 for all models, 2 for traces and block receipts) (move comment to wiki)
-  # BLOCKS_TRANSACTIONS_REALTIME_ORDER_BY_CLAUSE: 'ORDER BY partition_key ASC' ### desc is default for realtime and asc is default for history (move comment to wiki)
-  # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 ### meant for testing purposes only (move comment to wiki)
-  # BLOCKS_TRANSACTIONS_REALTIME_QUANTUM_STATE: 'streamline' ### default is 'streamline' (move comment to wiki)
+  ### MAIN_PACKAGE VARIABLES END ###

-# document sample Variable Name Format 
-  # <model_name>_<model_type>_<parameter_name>
-  # <parameter_name> If only supplied, global parameter. Required for all variables.
-  # <model_type> could be realtime, history, complete, etc., if applicable
-  # <model_name> is the name of the model, if applicable
+  ### CURATED_PACKAGE VARIABLES BEGIN ###
+
+  ### STATS ###
+  # STATS_TOKEN_ADDRESS: '' ### REQUIRED
+
+  ### CURATED_PACKAGE VARIABLES END ###
+
+#### FSC_EVM END ####
--- a/packages.yml
+++ b/packages.yml
@ -1,13 +1,3 @@
 packages:
-  - package: calogica/dbt_expectations
-    version: 0.8.2
-  - package: dbt-labs/dbt_external_tables
-    version: 0.8.2
-  - package: dbt-labs/dbt_utils
-    version: 1.0.0
-  - git: https://github.com/FlipsideCrypto/fsc-utils.git
-    revision: v1.31.0
-  - package: get-select/dbt_snowflake_query_tags
-    version: [">=2.0.0", "<3.0.0"]
  - git: https://github.com/FlipsideCrypto/fsc-evm.git
-    revision: v2.2.1
+    revision: v3.0.0