diff --git a/.github/workflows/dbt_run_scheduled_main.yml b/.github/workflows/dbt_run_scheduled_main.yml new file mode 100644 index 0000000..78e7267 --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_main.yml @@ -0,0 +1,44 @@ +name: dbt_run_scheduled_main +run-name: dbt_run_scheduled_main + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "core_models,tag:core" "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:silver_prices" "fsc_evm,tag:gold_prices" "fsc_evm,tag:silver_labels" "fsc_evm,tag:gold_labels" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_chainhead.yml b/.github/workflows/dbt_run_streamline_chainhead.yml index 94ea7dc..f9d8b75 100644 --- a/.github/workflows/dbt_run_streamline_chainhead.yml +++ b/.github/workflows/dbt_run_streamline_chainhead.yml @@ -41,4 +41,8 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts_by_hash" + + - name: Test Chainhead + run: | + dbt test -m "fsc_evm,tag:chainhead" \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index a55dcf3..867b3e5 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_complete_receipts_by_hash" "fsc_evm,tag:streamline_core_history_receipts_by_hash" \ No newline at end of file diff --git a/.github/workflows/dbt_run_scheduled_core.yml b/.github/workflows/dbt_test_intraday.yml similarity index 81% rename from .github/workflows/dbt_run_scheduled_core.yml rename to .github/workflows/dbt_test_intraday.yml index db47f59..fbb78c4 100644 --- a/.github/workflows/dbt_run_scheduled_core.yml +++ b/.github/workflows/dbt_test_intraday.yml @@ -1,11 +1,11 @@ -name: dbt_run_scheduled_core -run-name: dbt_run_scheduled_core +name: dbt_test_intraday +run-name: dbt_test_intraday on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -21,11 +21,13 @@ env: concurrency: group: ${{ github.workflow }} + + jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: - name: workflow_prod + environment: + name: workflow_test steps: - uses: actions/checkout@v3 @@ -41,4 +43,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m "fsc_evm,tag:core" "core_models,tag:core" + dbt test -m "fsc_evm,tag:recent_test" \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index 96ae381..a24f5f1 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,4 +1,5 @@ workflow_name,workflow_schedule dbt_run_streamline_chainhead,"23,53 * * * *" -dbt_run_scheduled_core,"1,31 * * * *" +dbt_run_scheduled_main,"1,31 * * * *" +dbt_test_intraday,"39 */4 * * *" dbt_run_streamline_history,"22 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 20d46be..6e08fa0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -17,6 +17,7 @@ test-paths: ["tests"] seed-paths: ["data"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] +docs-paths: ["dbt_packages/fsc_evm/doc_descriptions", "models/doc_descriptions"] target-path: "target" # directory which will store compiled SQL files clean-targets: # directories to be removed by `dbt clean` @@ -25,7 +26,10 @@ clean-targets: # directories to be removed by `dbt clean` - "dbt_packages" tests: - +store_failures: true # all tests + core_models: + +store_failures: true # all tests + fsc_evm: + +store_failures: true on-run-start: - "{{ fsc_evm.create_sps() }}" @@ -49,14 +53,28 @@ query-comment: # Full documentation: https://docs.getdbt.com/docs/configuring-models models: - +copy_grants: true - +persist_docs: - relation: true - columns: true - +on_schema_change: "append_new_columns" - + core_models: + +copy_grants: true + +persist_docs: + relation: true + columns: true + +on_schema_change: "append_new_columns" livequery_models: +materialized: ephemeral + fsc_evm: + +enabled: false + main_package: + +enabled: false + core: + +enabled: true + github_actions: + +enabled: true + labels: + +enabled: true + prices: + +enabled: true + utils: + +enabled: true vars: "dbt_date:time_zone": GMT @@ -98,46 +116,34 @@ vars: #### FSC_EVM BEGIN #### - ### GLOBAL VARIABLES + ### GLOBAL VARIABLES BEGIN ### + ## REQUIRED GLOBAL_PROD_DB_NAME: 'core' GLOBAL_NODE_SECRET_PATH: 'Vault/prod/core/ankr/mainnet' GLOBAL_BLOCKS_PER_HOUR: 1200 - ### OPTIONAL - # GLOBAL_NODE_URL: '{Service}/{Authentication}' is default - # GLOBAL_PROD_NETWORK: 'mainnet' ### default is 'mainnet' (move comment to wiki) - GLOBAL_USES_RECEIPTS_BY_HASH: True ### default is False (move comment to wiki) - GLOBAL_USES_BASE_FEE: False ### default is True (move comment to wiki) - GLOBAL_AVG_TXS_PER_BLOCK: 20 ### only used if USES_RECEIPTS_BY_HASH is True (move comment to wiki) - GLOBAL_BLOCKS_TOTAL_TO_BACKFILL: 19000000 ### high estimate for total blocks to backfill (move comment to wiki) - GLOBAL_START_UP_BLOCK: 18343780 ### default is 0 (move comment to wiki) - GLOBAL_MAX_SEQUENCE_NUMBER: 50000000 ### default is 100000000 blocks. (move comment to wiki) + ## OPTIONAL + GLOBAL_USES_RECEIPTS_BY_HASH: True + GLOBAL_USES_BASE_FEE: False + GLOBAL_AVG_TXS_PER_BLOCK: 20 + GLOBAL_BLOCKS_TOTAL_TO_BACKFILL: 19000000 + GLOBAL_START_UP_BLOCK: 18343780 + GLOBAL_MAX_SEQUENCE_NUMBER: 50000000 - ### CORE MODEL PARAMETERS BEGIN ### + ### GLOBAL VARIABLES END ### - TRACES_FULL_RELOAD_START_BLOCK: 20000000 ### default is 0 (move comment to wiki) + ### MAIN_PACKAGE VARIABLES BEGIN ### - ### PRICES ### - PRICES_SYMBOLS: 'CORE' ### REQUIRED - PRICES_PLATFORMS: ['Core', 'core'] ### REQUIRED - # PRICES_BLOCKCHAINS: ### OPTIONAL (defaults to 'GLOBAL_PROD_DB_NAME') - # PRICES_TOKEN_ADDRESSES: ### OPTIONAL - - ### LABELS ### - # LABELS_BLOCKCHAINS: ### OPTIONAL (defaults to 'GLOBAL_PROD_DB_NAME') + ### CORE ### + ## REQUIRED + TRACES_FULL_RELOAD_START_BLOCK: 20000000 - ### CORE MODEL PARAMETERS END ### - -#### FSC_EVM END #### - -#### STREAMLINE PARAMETERS BEGIN - SEE abc.xyz FOR ALL AVAILABLE PARAMETERS #### - - ### OPTIONAL + ## OPTIONAL BLOCKS_TRANSACTIONS_REALTIME_NEW_BUILD: True RECEIPTS_BY_HASH_REALTIME_NEW_BUILD: True TRACES_REALTIME_NEW_BUILD: True CONFIRM_BLOCKS_REALTIME_NEW_BUILD: True - + BLOCKS_TRANSACTIONS_HISTORY_NEW_BUILD: True RECEIPTS_BY_HASH_HISTORY_NEW_BUILD: True @@ -153,19 +159,21 @@ vars: TRACES_HISTORY_ORDER_BY_CLAUSE: 'ORDER BY partition_key DESC, block_number DESC' CONFIRM_BLOCKS_HISTORY_NEW_BUILD: True + + ### PRICES ### + ## REQUIRED + PRICES_NATIVE_SYMBOLS: 'CORE' + PRICES_PROVIDER_PLATFORMS: ['Core', 'core'] -#### STREAMLINE PARAMETERS END #### + ### LABELS ### -# MOVE TO WIKI AND DOCUMENT THOROUGHLY, THEN REMOVE FROM HERE - # BLOCKS_TRANSACTIONS_REALTIME_SQL_LIMIT: 4800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_PRODUCER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_WORKER_BATCH_SIZE: 800 ### default is 2 x BLOCKS_PER_HOUR / lambdas in model config (1 for all models, 2 for traces and block receipts) (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_ORDER_BY_CLAUSE: 'ORDER BY partition_key ASC' ### desc is default for realtime and asc is default for history (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_TESTING_LIMIT: 3 ### meant for testing purposes only (move comment to wiki) - # BLOCKS_TRANSACTIONS_REALTIME_QUANTUM_STATE: 'streamline' ### default is 'streamline' (move comment to wiki) + ### MAIN_PACKAGE VARIABLES END ### -# document sample Variable Name Format - # __ - # If only supplied, global parameter. Required for all variables. - # could be realtime, history, complete, etc., if applicable - # is the name of the model, if applicable \ No newline at end of file + ### CURATED_PACKAGE VARIABLES BEGIN ### + + ### STATS ### + # STATS_TOKEN_ADDRESS: '' ### REQUIRED + + ### CURATED_PACKAGE VARIABLES END ### + +#### FSC_EVM END #### \ No newline at end of file diff --git a/packages.yml b/packages.yml index 6347bd6..ddaf129 100644 --- a/packages.yml +++ b/packages.yml @@ -1,13 +1,3 @@ packages: - - package: calogica/dbt_expectations - version: 0.8.2 - - package: dbt-labs/dbt_external_tables - version: 0.8.2 - - package: dbt-labs/dbt_utils - version: 1.0.0 - - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: v1.31.0 - - package: get-select/dbt_snowflake_query_tags - version: [">=2.0.0", "<3.0.0"] - git: https://github.com/FlipsideCrypto/fsc-evm.git - revision: v2.2.1 \ No newline at end of file + revision: v3.0.0