From 831840e8c49321a6937dc87978edc18fd2f05816 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 11 Oct 2024 15:33:50 -0600 Subject: [PATCH 01/11] package and dbt project updates --- dbt_project.yml | 18 +++++++++++------- packages.yml | 12 +----------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 17eca09..ee29b64 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -25,7 +25,8 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_packages" tests: - +store_failures: true # all tests + core_models: + +store_failures: true # all tests on-run-start: - "{{ fsc_evm.create_sps() }}" @@ -49,14 +50,16 @@ query-comment: # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - +copy_grants: true - +persist_docs: - relation: true - columns: true - +on_schema_change: "append_new_columns" - + core_models: + +copy_grants: true + +persist_docs: + relation: true + columns: true + +on_schema_change: "append_new_columns" livequery_models: +materialized: ephemeral + fsc_evm: + +enabled: true vars: "dbt_date:time_zone": GMT @@ -102,6 +105,7 @@ vars: GLOBAL_PROD_DB_NAME: 'core' GLOBAL_NODE_SECRET_PATH: 'Vault/prod/core/ankr/mainnet' GLOBAL_BLOCKS_PER_HOUR: 1200 + GLOBAL_STREAMLINE_MIGRATION: False ### OPTIONAL # GLOBAL_NODE_URL: '{Service}/{Authentication}' is default diff --git a/packages.yml b/packages.yml index 92868ee..4826865 100644 --- a/packages.yml +++ b/packages.yml @@ -1,13 +1,3 @@ packages: - - package: calogica/dbt_expectations - version: 0.8.2 - - package: dbt-labs/dbt_external_tables - version: 0.8.2 - - package: dbt-labs/dbt_utils - version: 1.0.0 - - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.31.0 - - package: get-select/dbt_snowflake_query_tags - version: [">=2.0.0", "<3.0.0"] - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v2.0.0 \ No newline at end of file + revision: "AN-5278/sl-other-macros" \ No newline at end of file From e3afb35d49c5afd800fed21d98ade2abc8cbe517 Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 11 Oct 2024 16:48:37 -0600 Subject: [PATCH 02/11] project vars --- dbt_project.yml | 58 +++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index ee29b64..327430b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -101,13 +101,14 @@ vars: #### FSC_EVM BEGIN #### - ### GLOBAL VARIABLES + ### GLOBAL VARIABLES BEGIN ### + ## REQUIRED GLOBAL_PROD_DB_NAME: 'core' GLOBAL_NODE_SECRET_PATH: 'Vault/prod/core/ankr/mainnet' GLOBAL_BLOCKS_PER_HOUR: 1200 - GLOBAL_STREAMLINE_MIGRATION: False + GLOBAL_USES_STREAMLINE_V1: False - ### OPTIONAL + ## OPTIONAL # GLOBAL_NODE_URL: '{Service}/{Authentication}' is default # GLOBAL_PROD_NETWORK: 'mainnet' ### default is 'mainnet' (move comment to wiki) GLOBAL_USES_RECEIPTS_BY_HASH: True ### default is False (move comment to wiki) @@ -117,40 +118,55 @@ vars: GLOBAL_START_UP_BLOCK: 18343780 ### default is 0 (move comment to wiki) GLOBAL_MAX_SEQUENCE_NUMBER: 50000000 ### default is 100000000 blocks. (move comment to wiki) - ### CORE MODEL PARAMETERS BEGIN ### + ### GLOBAL VARIABLES END ### + ### CORE_PACKAGE VARIABLES BEGIN ### + + ### CORE ### + ## REQUIRED TRACES_FULL_RELOAD_START_BLOCK: 20000000 ### default is 0 (move comment to wiki) ### PRICES ### - PRICES_SYMBOLS: 'CORE' ### REQUIRED - PRICES_PLATFORMS: ['Core', 'core'] ### REQUIRED - # PRICES_BLOCKCHAINS: ### OPTIONAL (defaults to 'GLOBAL_PROD_DB_NAME') - # PRICES_TOKEN_ADDRESSES: ### OPTIONAL + ## REQUIRED + PRICES_SYMBOLS: 'CORE' + PRICES_PLATFORMS: ['Core', 'core'] + + ## OPTIONAL + # PRICES_BLOCKCHAINS: ### (defaults to 'GLOBAL_PROD_DB_NAME') + # PRICES_TOKEN_ADDRESSES: ### LABELS ### - # LABELS_BLOCKCHAINS: ### OPTIONAL (defaults to 'GLOBAL_PROD_DB_NAME') + ## OPTIONAL + # LABELS_BLOCKCHAINS: ### (defaults to 'GLOBAL_PROD_DB_NAME') - ### CORE MODEL PARAMETERS END ### + ### CORE_PACKAGE VARIABLES END ### -#### FSC_EVM END #### + ### CURATED_PACKAGE VARIABLES BEGIN ### -#### STREAMLINE PARAMETERS BEGIN - SEE abc.xyz FOR ALL AVAILABLE PARAMETERS #### + ### STATS ### + # STATS_TOKEN_ADDRESS: '' ### REQUIRED - ### OPTIONAL + ### CURATED_PACKAGE VARIABLES END ### + + ### STREAMLINE VARIABLES BEGIN - SEE abc.xyz FOR ALL AVAILABLE VARIABLES ### + + ## OPTIONAL BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) TRACES_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) -#### STREAMLINE PARAMETERS END #### + ### STREAMLINE VARIABLES END ### -# MOVE TO WIKI AND DOCUMENT THOROUGHLY, THEN REMOVE FROM HERE - # BLOCKS_TRANSACTIONS_REALTIME_SQL_LIMIT: 4800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_PRODUCER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_WORKER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR / lambdas in model config (1 for all models, 2 for traces and block receipts) (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_ORDER_BY_CLAUSE: 'ORDER BY partition_key ASC' ### desc is default for realtime and asc is default for history (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 ### meant for testing purposes only (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_QUANTUM_STATE: 'streamline' ### default is 'streamline' (move comment to wiki) + # MOVE TO WIKI AND DOCUMENT THOROUGHLY, THEN REMOVE FROM HERE + # BLOCKS_TRANSACTIONS_REALTIME_SQL_LIMIT: 4800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) + # BLOCKS_TRANSACTIONS_REALTIME_PRODUCER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) + # BLOCKS_TRANSACTIONS_REALTIME_WORKER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR / lambdas in model config (1 for all models, 2 for traces and block receipts) (move comment to wiki) + # BLOCKS_TRANSACTIONS_REALTIME_ORDER_BY_CLAUSE: 'ORDER BY partition_key ASC' ### desc is default for realtime and asc is default for history (move comment to wiki) + # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 ### meant for testing purposes only (move comment to wiki) + # BLOCKS_TRANSACTIONS_REALTIME_QUANTUM_STATE: 'streamline' ### default is 'streamline' (move comment to wiki) + +#### FSC_EVM END #### # document sample Variable Name Format # __ From 16744dfea384e8eba27217a5fbd9ed6559ed6c6d Mon Sep 17 00:00:00 2001 From: drethereum Date: Thu, 17 Oct 2024 16:45:13 -0600 Subject: [PATCH 03/11] dbt project vars --- dbt_project.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 327430b..6ef4524 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -59,7 +59,11 @@ models: livequery_models: +materialized: ephemeral fsc_evm: - +enabled: true + +enabled: false + core_package: + +enabled: true + github_actions_package: + +enabled: true vars: "dbt_date:time_zone": GMT From 57399760275c979a6c788a392b7dd6e4578b4dad Mon Sep 17 00:00:00 2001 From: drethereum Date: Fri, 25 Oct 2024 11:36:08 -0600 Subject: [PATCH 04/11] prices vars --- dbt_project.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 6ef4524..9d9b224 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -132,16 +132,10 @@ vars: ### PRICES ### ## REQUIRED - PRICES_SYMBOLS: 'CORE' - PRICES_PLATFORMS: ['Core', 'core'] - - ## OPTIONAL - # PRICES_BLOCKCHAINS: ### (defaults to 'GLOBAL_PROD_DB_NAME') - # PRICES_TOKEN_ADDRESSES: + PRICES_NATIVE_SYMBOLS: 'CORE' + PRICES_PROVIDER_PLATFORMS: ['Core', 'core'] ### LABELS ### - ## OPTIONAL - # LABELS_BLOCKCHAINS: ### (defaults to 'GLOBAL_PROD_DB_NAME') ### CORE_PACKAGE VARIABLES END ### From 532d098c1ed98ff5c35581b52b6b5c4c065237f1 Mon Sep 17 00:00:00 2001 From: drethereum Date: Mon, 28 Oct 2024 10:46:17 -0600 Subject: [PATCH 05/11] cleaned up dbt project.yml --- .github/workflows/dbt_run_scheduled_core.yml | 2 +- .../dbt_run_streamline_chainhead.yml | 2 +- dbt_project.yml | 35 ++++--------------- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_run_scheduled_core.yml index 946dc29..03250fc 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_run_scheduled_core.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "fsc_evm,tag:core" + dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 94ea7dc..a6671e0 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 9d9b224..6cc131e 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -110,11 +110,8 @@ vars: GLOBAL_PROD_DB_NAME: 'core' GLOBAL_NODE_SECRET_PATH: 'Vault/prod/core/ankr/mainnet' GLOBAL_BLOCKS_PER_HOUR: 1200 - GLOBAL_USES_STREAMLINE_V1: False ## OPTIONAL - # GLOBAL_NODE_URL: '{Service}/{Authentication}' is default - # GLOBAL_PROD_NETWORK: 'mainnet' ### default is 'mainnet' (move comment to wiki) GLOBAL_USES_RECEIPTS_BY_HASH: True ### default is False (move comment to wiki) GLOBAL_USES_BASE_FEE: False ### default is True (move comment to wiki) GLOBAL_AVG_TXS_PER_BLOCK: 20 ### only used if USES_RECEIPTS_BY_HASH is True (move comment to wiki) @@ -130,6 +127,12 @@ vars: ## REQUIRED TRACES_FULL_RELOAD_START_BLOCK: 20000000 ### default is 0 (move comment to wiki) + ## OPTIONAL + BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) + RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) + TRACES_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) + CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) + ### PRICES ### ## REQUIRED PRICES_NATIVE_SYMBOLS: 'CORE' @@ -146,28 +149,4 @@ vars: ### CURATED_PACKAGE VARIABLES END ### - ### STREAMLINE VARIABLES BEGIN - SEE abc.xyz FOR ALL AVAILABLE VARIABLES ### - - ## OPTIONAL - BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - TRACES_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - - ### STREAMLINE VARIABLES END ### - - # MOVE TO WIKI AND DOCUMENT THOROUGHLY, THEN REMOVE FROM HERE - # BLOCKS_TRANSACTIONS_REALTIME_SQL_LIMIT: 4800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_PRODUCER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_WORKER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR / lambdas in model config (1 for all models, 2 for traces and block receipts) (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_ORDER_BY_CLAUSE: 'ORDER BY partition_key ASC' ### desc is default for realtime and asc is default for history (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 ### meant for testing purposes only (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_QUANTUM_STATE: 'streamline' ### default is 'streamline' (move comment to wiki) - -#### FSC_EVM END #### - -# document sample Variable Name Format - # __ - # If only supplied, global parameter. Required for all variables. - # could be realtime, history, complete, etc., if applicable - # is the name of the model, if applicable \ No newline at end of file +#### FSC_EVM END #### \ No newline at end of file From 1abd790c0a831c10788f009d3bdb4c05add9309d Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 29 Oct 2024 09:36:35 -0600 Subject: [PATCH 06/11] test limit vars --- dbt_project.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/dbt_project.yml b/dbt_project.yml index 6cc131e..ca04417 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -133,6 +133,18 @@ vars: TRACES_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) + # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 + # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 3 + + # RECEIPTS_BY_HASH_REALTIME_TESTING_LIMIT: 3 + # RECEIPTS_BY_HASH_HISTORY_TESTING_LIMIT: 3 + + # TRACES_REALTIME_TESTING_LIMIT: 3 + # TRACES_HISTORY_TESTING_LIMIT: 3 + + # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 + # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 3 + ### PRICES ### ## REQUIRED PRICES_NATIVE_SYMBOLS: 'CORE' From 567343b39772304d5a480fd242017a17708d3f67 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 29 Oct 2024 13:31:45 -0600 Subject: [PATCH 07/11] updated workflows and dbt_project.yml --- .github/workflows/dbt_run_scheduled_core.yml | 2 +- .../dbt_run_streamline_chainhead.yml | 6 ++- .github/workflows/dbt_test_intraday.yml | 49 +++++++++++++++++++ data/github_actions__workflows.csv | 4 +- dbt_project.yml | 39 +++++++++------ 5 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/dbt_test_intraday.yml diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_run_scheduled_core.yml index 03250fc..6b2dbb3 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_run_scheduled_core.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" + dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index a6671e0..f9d8b75 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -41,4 +41,8 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" + + - name: Test Chainhead + run: | + dbt test -m "fsc_evm,tag:chainhead" \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml new file mode 100644 index 0000000..81f42f8 --- /dev/null +++ b/.github/workflows/dbt_test_intraday.yml @@ -0,0 +1,49 @@ +name: dbt_test_intraday +run-name: dbt_test_intraday + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + + + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_test + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt test -m "fsc_evm,tag:recent_test" + + + diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index ac4a5db..9218768 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,3 +1,5 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"23,53 * * * *" -dbt_run_scheduled_core,"1,31 * * * *" \ No newline at end of file +dbt_run_scheduled_core,"1,31 * * * *" +dbt_test_intraday,"39 */4 * * *" +dbt_run_streamline_history,"22 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index ca04417..ab46849 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -17,6 +17,7 @@ test-paths: ["tests"] seed-paths: ["data"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] +docs-paths: ["dbt_packages/fsc_evm/doc_descriptions", "models/doc_descriptions"] target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` @@ -60,10 +61,18 @@ models: +materialized: ephemeral fsc_evm: +enabled: false - core_package: - +enabled: true - github_actions_package: - +enabled: true + main_package: + +enabled: false + core: + +enabled: true + github_actions: + +enabled: true + labels: + +enabled: true + prices: + +enabled: true + utils: + +enabled: true vars: "dbt_date:time_zone": GMT @@ -112,12 +121,12 @@ vars: GLOBAL_BLOCKS_PER_HOUR: 1200 ## OPTIONAL - GLOBAL_USES_RECEIPTS_BY_HASH: True ### default is False (move comment to wiki) - GLOBAL_USES_BASE_FEE: False ### default is True (move comment to wiki) - GLOBAL_AVG_TXS_PER_BLOCK: 20 ### only used if USES_RECEIPTS_BY_HASH is True (move comment to wiki) - GLOBAL_BLOCKS_TOTAL_TO_BACKFILL: 19000000 ### high estimate for total blocks to backfill (move comment to wiki) - GLOBAL_START_UP_BLOCK: 18343780 ### default is 0 (move comment to wiki) - GLOBAL_MAX_SEQUENCE_NUMBER: 50000000 ### default is 100000000 blocks. (move comment to wiki) + GLOBAL_USES_RECEIPTS_BY_HASH: True + GLOBAL_USES_BASE_FEE: False + GLOBAL_AVG_TXS_PER_BLOCK: 20 + GLOBAL_BLOCKS_TOTAL_TO_BACKFILL: 19000000 + GLOBAL_START_UP_BLOCK: 18343780 + GLOBAL_MAX_SEQUENCE_NUMBER: 50000000 ### GLOBAL VARIABLES END ### @@ -125,13 +134,13 @@ vars: ### CORE ### ## REQUIRED - TRACES_FULL_RELOAD_START_BLOCK: 20000000 ### default is 0 (move comment to wiki) + TRACES_FULL_RELOAD_START_BLOCK: 20000000 ## OPTIONAL - BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - TRACES_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) - CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True ### default is False (move comment to wiki) + BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True + RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True + TRACES_REALTIME_NEW_BUILD: True + CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 3 From cf37a589ca998024a91cfcc0831dcab69200a737 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 29 Oct 2024 14:43:19 -0600 Subject: [PATCH 08/11] workflow cmds --- .github/workflows/dbt_run_streamline_history.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index a55dcf3..867b3e5 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_history_receipts_by_hash" \ No newline at end of file From fc53f8190123df26c91d4ef9a858077c8334ee80 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 29 Oct 2024 14:57:54 -0600 Subject: [PATCH 09/11] updates to workflows --- ...bt_run_scheduled_core.yml => dbt_run_scheduled_main.yml} | 6 +++--- .github/workflows/dbt_test_intraday.yml | 5 +---- data/github_actions__workflows.csv | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) rename .github/workflows/{dbt_run_scheduled_core.yml => dbt_run_scheduled_main.yml} (75%) diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_run_scheduled_main.yml similarity index 75% rename from .github/workflows/dbt_run_scheduled_core.yml rename to .github/workflows/dbt_run_scheduled_main.yml index 6b2dbb3..78e7267 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -1,5 +1,5 @@ -name: dbt_run_scheduled_core -run-name: dbt_run_scheduled_core +name: dbt_run_scheduled_main +run-name: dbt_run_scheduled_main on: workflow_dispatch: @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" + dbt run -m "core_models,tag:core" "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" \ No newline at end of file diff --git a/.github/workflows/dbt_test_intraday.yml b/.github/workflows/dbt_test_intraday.yml index 81f42f8..fbb78c4 100644 --- a/.github/workflows/dbt_test_intraday.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -43,7 +43,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test -m "fsc_evm,tag:recent_test" - - - + dbt test -m "fsc_evm,tag:recent_test" \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 9218768..a24f5f1 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,5 +1,5 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"23,53 * * * *" -dbt_run_scheduled_core,"1,31 * * * *" +dbt_run_scheduled_main,"1,31 * * * *" dbt_test_intraday,"39 */4 * * *" dbt_run_streamline_history,"22 * * * *" \ No newline at end of file From ce56880e9a582bf894de38daf99c77301374f356 Mon Sep 17 00:00:00 2001 From: drethereum Date: Tue, 29 Oct 2024 16:24:15 -0600 Subject: [PATCH 10/11] package version --- packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages.yml b/packages.yml index 13c7577..ddaf129 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: "AN-5278/sl-other-macros" + revision: v3.0.0 From 3a979561c0fc8aa192934e50c590dfc9ac422c0d Mon Sep 17 00:00:00 2001 From: drethereum Date: Wed, 30 Oct 2024 08:53:27 -0600 Subject: [PATCH 11/11] store failures and remove comments --- dbt_project.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index c093901..6e08fa0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -28,6 +28,8 @@ clean-targets: # directories to be removed by `dbt clean` tests: core_models: +store_failures: true # all tests + fsc_evm: + +store_failures: true on-run-start: - "{{ fsc_evm.create_sps() }}" @@ -158,18 +160,6 @@ vars: CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True - # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 - # BLOCKS_TRANSACTIONS_HISTORY_TESTING_LIMIT: 3 - - # RECEIPTS_BY_HASH_REALTIME_TESTING_LIMIT: 3 - # RECEIPTS_BY_HASH_HISTORY_TESTING_LIMIT: 3 - - # TRACES_REALTIME_TESTING_LIMIT: 3 - # TRACES_HISTORY_TESTING_LIMIT: 3 - - # CONFIRM_BLOCKS_REALTIME_TESTING_LIMIT: 3 - # CONFIRM_BLOCKS_HISTORY_TESTING_LIMIT: 3 - ### PRICES ### ## REQUIRED PRICES_NATIVE_SYMBOLS: 'CORE'