diff --git a/.github/workflows/dbt_run_abi_refresh.yml b/.github/workflows/dbt_run_abi_refresh.yml index d6d93f4..5fe0e18 100644 --- a/.github/workflows/dbt_run_abi_refresh.yml +++ b/.github/workflows/dbt_run_abi_refresh.yml @@ -41,7 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/abis + dbt run -m tag:abis diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index ea543e1..70ce444 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -41,7 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m models/silver/_observability + dbt run --threads 2 --vars '{"OBSERV_FULL_TEST":True}' -m tag:observability diff --git a/.github/workflows/dbt_run_api_feeder_table.yml b/.github/workflows/dbt_run_incremental_non_realtime.yml similarity index 79% rename from .github/workflows/dbt_run_api_feeder_table.yml rename to .github/workflows/dbt_run_incremental_non_realtime.yml index 44efb67..77dc296 100644 --- a/.github/workflows/dbt_run_api_feeder_table.yml +++ b/.github/workflows/dbt_run_incremental_non_realtime.yml @@ -1,11 +1,11 @@ -name: dbt_run_api_feeder_table -run-name: dbt_run_api_feeder_table +name: dbt_run_scheduled_non_realtime +run-name: dbt_run_scheduled_non_realtime on: workflow_dispatch: schedule: - # Runs "at 7:00 UTC" (see https://crontab.guru) - - cron: '0 7 * * *' + # Runs "At minute 40, every hour" (see https://crontab.guru) + - cron: '40 * * * *' env: DBT_PROFILES_DIR: ./ @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/API_udf \ No newline at end of file + dbt run -m tag:non_realtime \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder.yml b/.github/workflows/dbt_run_streamline_decoder.yml index b67517d..d6df25d 100644 --- a/.github/workflows/dbt_run_streamline_decoder.yml +++ b/.github/workflows/dbt_run_streamline_decoder.yml @@ -41,4 +41,5 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run -m models/silver/core/silver__decoded_logs.sql \ No newline at end of file + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+tag:streamline_decoded_logs_realtime + dbt run -m tag:decoded_logs \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml index 6fa3656..3ded573 100644 --- a/.github/workflows/dbt_run_streamline_decoder_history.yml +++ b/.github/workflows/dbt_run_streamline_decoder_history.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m models/silver/streamline/decoder/streamline__complete_decode_logs.sql models/silver/streamline/decoder/history \ No newline at end of file + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m 1+tag:streamline_decoded_logs_history \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_history_adhoc.yml b/.github/workflows/dbt_run_streamline_history_adhoc.yml new file mode 100644 index 0000000..bd1fc86 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_history_adhoc.yml @@ -0,0 +1,69 @@ +name: dbt_run_streamline_history_adhoc +run-name: dbt_run_streamline_history_adhoc + +on: + workflow_dispatch: + branches: + - "main" + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + - prod_backfill + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: choice + description: 'DBT Run Command' + required: true + options: + - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m 1+tag:streamline_decoded_logs_history + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ inputs.warehouse }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_realtime.yml b/.github/workflows/dbt_run_streamline_realtime.yml index f242366..6bde26d 100644 --- a/.github/workflows/dbt_run_streamline_realtime.yml +++ b/.github/workflows/dbt_run_streamline_realtime.yml @@ -41,4 +41,4 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/core/realtime + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+tag:streamline_core_realtime diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index f87d76c..25963f1 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -41,7 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt test --exclude tag:full_test tag:recent_test models/silver/goerli + dbt test --exclude tag:full_test tag:recent_test tag:base_goerli diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/legacy_workflows/dbt_run_api_feeder_table.yml similarity index 59% rename from .github/workflows/dbt_run_incremental.yml rename to .github/workflows/legacy_workflows/dbt_run_api_feeder_table.yml index 220e58d..4346348 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/legacy_workflows/dbt_run_api_feeder_table.yml @@ -1,11 +1,11 @@ -name: dbt_run_scheduled -run-name: dbt_run_scheduled +name: dbt_run_api_feeder_table +run-name: dbt_run_api_feeder_table on: workflow_dispatch: schedule: - # Runs "At minute 40, every hour" (see https://crontab.guru) - - cron: '40 * * * *' + # Runs "at 7:00 UTC" (see https://crontab.guru) + - cron: '0 7 * * *' env: DBT_PROFILES_DIR: ./ @@ -41,5 +41,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --exclude models/silver/abis models/silver/goerli models/silver/streamline models/bronze/eth_goerli models/silver/_observability models/silver/API_udf models/silver/core/silver__decoded_logs.sql - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m models/silver/streamline/decoder/streamline__complete_decode_logs.sql models/silver/streamline/decoder/streamline__decode_logs_realtime.sql \ No newline at end of file + dbt run -m models/silver/API_udf +# moved to non_realtime job to match ethereum's structure +# relevant_abi_contracts now running in abi_refresh job +# relevant contracts now in non_realtime incremental job (will need to be here when we add contract_reads anyway) \ No newline at end of file diff --git a/models/bronze/api_udf/bronze_api__contract_abis.sql b/models/bronze/api_udf/bronze_api__contract_abis.sql index 2f30df6..676513d 100644 --- a/models/bronze/api_udf/bronze_api__contract_abis.sql +++ b/models/bronze/api_udf/bronze_api__contract_abis.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = "contract_address", - full_refresh = false + full_refresh = false, + tags = ['non_realtime'] ) }} WITH api_keys AS ( diff --git a/models/bronze/api_udf/bronze_api__token_reads.sql b/models/bronze/api_udf/bronze_api__token_reads.sql index 514f4e5..1f24c55 100644 --- a/models/bronze/api_udf/bronze_api__token_reads.sql +++ b/models/bronze/api_udf/bronze_api__token_reads.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = "contract_address", - full_refresh = false + full_refresh = false, + tags = ['non_realtime'] ) }} WITH base AS ( diff --git a/models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql b/models/legacy_models/bronze_goerli/bronze__eth_goerli_contract_logs.sql similarity index 96% rename from models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql rename to models/legacy_models/bronze_goerli/bronze__eth_goerli_contract_logs.sql index 42d26e7..2d9e29b 100644 --- a/models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.sql +++ b/models/legacy_models/bronze_goerli/bronze__eth_goerli_contract_logs.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'batch_id', - full_refresh = False + full_refresh = false, + tags = ['base_goerli'] ) }} WITH request_details AS ( diff --git a/models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.yml b/models/legacy_models/bronze_goerli/bronze__eth_goerli_contract_logs.yml similarity index 100% rename from models/bronze/eth_goerli/bronze__eth_goerli_contract_logs.yml rename to models/legacy_models/bronze_goerli/bronze__eth_goerli_contract_logs.yml diff --git a/models/gold/base_goerli/goerli__fact_blocks.sql b/models/legacy_models/gold_goerli/goerli__fact_blocks.sql similarity index 100% rename from models/gold/base_goerli/goerli__fact_blocks.sql rename to models/legacy_models/gold_goerli/goerli__fact_blocks.sql diff --git a/models/gold/base_goerli/goerli__fact_blocks.yml b/models/legacy_models/gold_goerli/goerli__fact_blocks.yml similarity index 100% rename from models/gold/base_goerli/goerli__fact_blocks.yml rename to models/legacy_models/gold_goerli/goerli__fact_blocks.yml diff --git a/models/gold/base_goerli/goerli__fact_event_logs.sql b/models/legacy_models/gold_goerli/goerli__fact_event_logs.sql similarity index 100% rename from models/gold/base_goerli/goerli__fact_event_logs.sql rename to models/legacy_models/gold_goerli/goerli__fact_event_logs.sql diff --git a/models/gold/base_goerli/goerli__fact_event_logs.yml b/models/legacy_models/gold_goerli/goerli__fact_event_logs.yml similarity index 100% rename from models/gold/base_goerli/goerli__fact_event_logs.yml rename to models/legacy_models/gold_goerli/goerli__fact_event_logs.yml diff --git a/models/gold/base_goerli/goerli__fact_traces.sql b/models/legacy_models/gold_goerli/goerli__fact_traces.sql similarity index 100% rename from models/gold/base_goerli/goerli__fact_traces.sql rename to models/legacy_models/gold_goerli/goerli__fact_traces.sql diff --git a/models/gold/base_goerli/goerli__fact_traces.yml b/models/legacy_models/gold_goerli/goerli__fact_traces.yml similarity index 100% rename from models/gold/base_goerli/goerli__fact_traces.yml rename to models/legacy_models/gold_goerli/goerli__fact_traces.yml diff --git a/models/gold/base_goerli/goerli__fact_transactions.sql b/models/legacy_models/gold_goerli/goerli__fact_transactions.sql similarity index 100% rename from models/gold/base_goerli/goerli__fact_transactions.sql rename to models/legacy_models/gold_goerli/goerli__fact_transactions.sql diff --git a/models/gold/base_goerli/goerli__fact_transactions.yml b/models/legacy_models/gold_goerli/goerli__fact_transactions.yml similarity index 100% rename from models/gold/base_goerli/goerli__fact_transactions.yml rename to models/legacy_models/gold_goerli/goerli__fact_transactions.yml diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.sql b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__blocks_method.sql similarity index 97% rename from models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.sql rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__blocks_method.sql index e3b3936..b7e1e80 100644 --- a/models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.sql +++ b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__blocks_method.sql @@ -2,7 +2,8 @@ materialized = "incremental", unique_key = "block_number", cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["block_number"] + merge_update_columns = ["block_number"], + tags = ['base_goerli'] ) }} WITH meta AS ( diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.yml b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__blocks_method.yml similarity index 100% rename from models/silver/goerli/base_goerli/methods/silver_goerli__blocks_method.yml rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__blocks_method.yml diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.sql b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__receipts_method.sql similarity index 95% rename from models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.sql rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__receipts_method.sql index 47b90fb..c2c0ada 100644 --- a/models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.sql +++ b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__receipts_method.sql @@ -1,7 +1,8 @@ {{ config ( materialized = "incremental", unique_key = "tx_hash", - cluster_by = "ROUND(block_number, -3)" + cluster_by = "ROUND(block_number, -3)", + tags = ['base_goerli'] ) }} WITH meta AS ( diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.yml b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__receipts_method.yml similarity index 100% rename from models/silver/goerli/base_goerli/methods/silver_goerli__receipts_method.yml rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__receipts_method.yml diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.sql b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__traces_method.sql similarity index 96% rename from models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.sql rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__traces_method.sql index b30f711..fc27fc4 100644 --- a/models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.sql +++ b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__traces_method.sql @@ -2,7 +2,8 @@ materialized = "incremental", unique_key = "ID", cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["ID"] + merge_update_columns = ["ID"], + tags = ['base_goerli'] ) }} WITH meta AS ( diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.yml b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__traces_method.yml similarity index 100% rename from models/silver/goerli/base_goerli/methods/silver_goerli__traces_method.yml rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__traces_method.yml diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.sql b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__tx_method.sql similarity index 97% rename from models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.sql rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__tx_method.sql index 4435332..d2015cb 100644 --- a/models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.sql +++ b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__tx_method.sql @@ -2,7 +2,8 @@ materialized = "incremental", unique_key = "tx_hash", cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["tx_hash"] + merge_update_columns = ["tx_hash"], + tags = ['base_goerli'] ) }} WITH meta AS ( diff --git a/models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.yml b/models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__tx_method.yml similarity index 100% rename from models/silver/goerli/base_goerli/methods/silver_goerli__tx_method.yml rename to models/legacy_models/silver_goerli/base_goerli/methods/silver_goerli__tx_method.yml diff --git a/models/silver/goerli/base_goerli/silver_goerli__blocks.sql b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__blocks.sql similarity index 95% rename from models/silver/goerli/base_goerli/silver_goerli__blocks.sql rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__blocks.sql index a834ee4..461244b 100644 --- a/models/silver/goerli/base_goerli/silver_goerli__blocks.sql +++ b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__blocks.sql @@ -2,7 +2,8 @@ materialized = "incremental", unique_key = "block_number", cluster_by = "block_timestamp::date", - merge_update_columns = ["block_number"] + merge_update_columns = ["block_number"], + tags = ['base_goerli'] ) }} SELECT diff --git a/models/silver/goerli/base_goerli/silver_goerli__blocks.yml b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__blocks.yml similarity index 100% rename from models/silver/goerli/base_goerli/silver_goerli__blocks.yml rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__blocks.yml diff --git a/models/silver/goerli/base_goerli/silver_goerli__logs.sql b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__logs.sql similarity index 98% rename from models/silver/goerli/base_goerli/silver_goerli__logs.sql rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__logs.sql index a6caa3a..e0afef7 100644 --- a/models/silver/goerli/base_goerli/silver_goerli__logs.sql +++ b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__logs.sql @@ -1,7 +1,8 @@ {{ config ( materialized = "incremental", unique_key = "_log_id", - cluster_by = "ROUND(block_number, -3)" + cluster_by = "ROUND(block_number, -3)", + tags = ['base_goerli'] ) }} WITH logs_response AS ( diff --git a/models/silver/goerli/base_goerli/silver_goerli__logs.yml b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__logs.yml similarity index 100% rename from models/silver/goerli/base_goerli/silver_goerli__logs.yml rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__logs.yml diff --git a/models/silver/goerli/base_goerli/silver_goerli__traces.sql b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__traces.sql similarity index 98% rename from models/silver/goerli/base_goerli/silver_goerli__traces.sql rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__traces.sql index b623119..91938de 100644 --- a/models/silver/goerli/base_goerli/silver_goerli__traces.sql +++ b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__traces.sql @@ -2,7 +2,8 @@ materialized = "incremental", unique_key = "_call_id", cluster_by = "ROUND(block_number, -3)", - merge_update_columns = ["_call_id"] + merge_update_columns = ["_call_id"], + tags = ['base_goerli'] ) }} WITH new_txs AS ( diff --git a/models/silver/goerli/base_goerli/silver_goerli__traces.yml b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__traces.yml similarity index 100% rename from models/silver/goerli/base_goerli/silver_goerli__traces.yml rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__traces.yml diff --git a/models/silver/goerli/base_goerli/silver_goerli__transactions.sql b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__transactions.sql similarity index 98% rename from models/silver/goerli/base_goerli/silver_goerli__transactions.sql rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__transactions.sql index 9ec7a4a..6ad5529 100644 --- a/models/silver/goerli/base_goerli/silver_goerli__transactions.sql +++ b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__transactions.sql @@ -1,7 +1,8 @@ {{ config ( materialized = "incremental", unique_key = "tx_hash", - cluster_by = "BLOCK_TIMESTAMP::DATE" + cluster_by = "BLOCK_TIMESTAMP::DATE", + tags = ['base_goerli'] ) }} WITH flat_base AS ( diff --git a/models/silver/goerli/base_goerli/silver_goerli__transactions.yml b/models/legacy_models/silver_goerli/base_goerli/silver_goerli__transactions.yml similarity index 100% rename from models/silver/goerli/base_goerli/silver_goerli__transactions.yml rename to models/legacy_models/silver_goerli/base_goerli/silver_goerli__transactions.yml diff --git a/models/silver/goerli/eth_goerli/silver__eth_goerli_block_ranges.sql b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_block_ranges.sql similarity index 96% rename from models/silver/goerli/eth_goerli/silver__eth_goerli_block_ranges.sql rename to models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_block_ranges.sql index 34bcf3d..b46593b 100644 --- a/models/silver/goerli/eth_goerli/silver__eth_goerli_block_ranges.sql +++ b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_block_ranges.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "table", - unique_key = "batch_id" + unique_key = "batch_id", + tags = ['base_goerli'] ) }} WITH inputs AS ( diff --git a/models/silver/goerli/eth_goerli/silver__eth_goerli_contract_reads.sql b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_contract_reads.sql similarity index 88% rename from models/silver/goerli/eth_goerli/silver__eth_goerli_contract_reads.sql rename to models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_contract_reads.sql index 53ad045..9992bfd 100644 --- a/models/silver/goerli/eth_goerli/silver__eth_goerli_contract_reads.sql +++ b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_contract_reads.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "table", - unique_key = "batch_id" + unique_key = "batch_id", + tags = ['base_goerli'] ) }} WITH ranges AS ( diff --git a/models/silver/goerli/eth_goerli/silver__eth_goerli_contracts.sql b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_contracts.sql similarity index 90% rename from models/silver/goerli/eth_goerli/silver__eth_goerli_contracts.sql rename to models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_contracts.sql index c91448e..6123182 100644 --- a/models/silver/goerli/eth_goerli/silver__eth_goerli_contracts.sql +++ b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_contracts.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = "view" + materialized = "view", + tags = ['base_goerli'] ) }} WITH base AS ( diff --git a/models/silver/goerli/eth_goerli/silver__eth_goerli_logs.sql b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_logs.sql similarity index 95% rename from models/silver/goerli/eth_goerli/silver__eth_goerli_logs.sql rename to models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_logs.sql index 972a753..80ced29 100644 --- a/models/silver/goerli/eth_goerli/silver__eth_goerli_logs.sql +++ b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_logs.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = '_log_id', - cluster_by = ['round(block_number,-3)'] + cluster_by = ['round(block_number,-3)'], + tags = ['base_goerli'] ) }} WITH base AS ( diff --git a/models/silver/goerli/eth_goerli/silver__eth_goerli_logs.yml b/models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_logs.yml similarity index 100% rename from models/silver/goerli/eth_goerli/silver__eth_goerli_logs.yml rename to models/legacy_models/silver_goerli/eth_goerli/silver__eth_goerli_logs.yml diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 2e1bd9c..87a8cda 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'test_timestamp', - full_refresh = false + full_refresh = false, + tags = ['observability'] ) }} WITH summary_stats AS ( diff --git a/models/silver/_observability/silver_observability__logs_completeness.sql b/models/silver/_observability/silver_observability__logs_completeness.sql index f305ee4..201fbca 100644 --- a/models/silver/_observability/silver_observability__logs_completeness.sql +++ b/models/silver/_observability/silver_observability__logs_completeness.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'test_timestamp', - full_refresh = false + full_refresh = false, + tags = ['observability'] ) }} WITH summary_stats AS ( diff --git a/models/silver/_observability/silver_observability__receipts_completeness.sql b/models/silver/_observability/silver_observability__receipts_completeness.sql index 8ac50a3..0c31691 100644 --- a/models/silver/_observability/silver_observability__receipts_completeness.sql +++ b/models/silver/_observability/silver_observability__receipts_completeness.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'test_timestamp', - full_refresh = false + full_refresh = false, + tags = ['observability'] ) }} WITH summary_stats AS ( diff --git a/models/silver/_observability/silver_observability__traces_completeness.sql b/models/silver/_observability/silver_observability__traces_completeness.sql index 5c4c597..01ec470 100644 --- a/models/silver/_observability/silver_observability__traces_completeness.sql +++ b/models/silver/_observability/silver_observability__traces_completeness.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'test_timestamp', - full_refresh = false + full_refresh = false, + tags = ['observability'] ) }} WITH summary_stats AS ( diff --git a/models/silver/_observability/silver_observability__transactions_completeness.sql b/models/silver/_observability/silver_observability__transactions_completeness.sql index 013b85a..e96e4b4 100644 --- a/models/silver/_observability/silver_observability__transactions_completeness.sql +++ b/models/silver/_observability/silver_observability__transactions_completeness.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = 'test_timestamp', - full_refresh = false + full_refresh = false, + tags = ['observability'] ) }} WITH summary_stats AS ( diff --git a/models/silver/abis/silver__abis.sql b/models/silver/abis/silver__abis.sql index 1c42481..def87b0 100644 --- a/models/silver/abis/silver__abis.sql +++ b/models/silver/abis/silver__abis.sql @@ -1,7 +1,8 @@ {{ config ( materialized = "incremental", unique_key = "contract_address", - post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)" + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address)", + tags = ['abis'] ) }} WITH override_abis AS ( diff --git a/models/silver/abis/silver__bytecode_abis.sql b/models/silver/abis/silver__bytecode_abis.sql index 4efeae3..186fec7 100644 --- a/models/silver/abis/silver__bytecode_abis.sql +++ b/models/silver/abis/silver__bytecode_abis.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "incremental", - unique_key = "abi_id" + unique_key = "abi_id", + tags = ['abis'] ) }} WITH bytecodes AS ( diff --git a/models/silver/abis/silver__complete_event_abis.sql b/models/silver/abis/silver__complete_event_abis.sql index 2b52236..cf3181b 100644 --- a/models/silver/abis/silver__complete_event_abis.sql +++ b/models/silver/abis/silver__complete_event_abis.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = 'table' + materialized = 'table', + tags = ['abis'] ) }} WITH abi_base AS ( diff --git a/models/silver/abis/silver__override_abis.sql b/models/silver/abis/silver__override_abis.sql index f711ffe..88b8439 100644 --- a/models/silver/abis/silver__override_abis.sql +++ b/models/silver/abis/silver__override_abis.sql @@ -1,5 +1,6 @@ {{ config( - materialized = 'view' + materialized = 'view', + tags = ['abis'] ) }} SELECT diff --git a/models/silver/abis/silver__proxies.sql b/models/silver/abis/silver__proxies.sql index 6ad81c7..f0a8898 100644 --- a/models/silver/abis/silver__proxies.sql +++ b/models/silver/abis/silver__proxies.sql @@ -1,5 +1,6 @@ {{ config ( - materialized = "table" + materialized = "table", + tags = ['abis'] ) }} WITH base AS ( diff --git a/models/silver/API_udf/silver__relevant_abi_contracts.sql b/models/silver/abis/silver__relevant_abi_contracts.sql similarity index 87% rename from models/silver/API_udf/silver__relevant_abi_contracts.sql rename to models/silver/abis/silver__relevant_abi_contracts.sql index 1e866da..3c0b8a5 100644 --- a/models/silver/API_udf/silver__relevant_abi_contracts.sql +++ b/models/silver/abis/silver__relevant_abi_contracts.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'table', - unique_key = "contract_address" + unique_key = "contract_address", + tags = ['abis'] ) }} WITH base AS ( diff --git a/models/silver/API_udf/silver__relevant_abi_contracts.yml b/models/silver/abis/silver__relevant_abi_contracts.yml similarity index 100% rename from models/silver/API_udf/silver__relevant_abi_contracts.yml rename to models/silver/abis/silver__relevant_abi_contracts.yml diff --git a/models/silver/abis/silver__user_verified_abis.sql b/models/silver/abis/silver__user_verified_abis.sql index 3b4882c..f9fe72f 100644 --- a/models/silver/abis/silver__user_verified_abis.sql +++ b/models/silver/abis/silver__user_verified_abis.sql @@ -1,7 +1,8 @@ {{ config ( materialized = "incremental", unique_key = "id", - merge_update_columns = ["id"] + merge_update_columns = ["id"], + tags = ['abis'] ) }} WITH base AS ( diff --git a/models/silver/abis/silver__verified_abis.sql b/models/silver/abis/silver__verified_abis.sql index b47783b..89ba461 100644 --- a/models/silver/abis/silver__verified_abis.sql +++ b/models/silver/abis/silver__verified_abis.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = "contract_address" + unique_key = "contract_address", + tags = ['abis'] ) }} WITH base AS ( diff --git a/models/silver/core/silver__blocks.sql b/models/silver/core/silver__blocks.sql index 575b298..1aa4ad5 100644 --- a/models/silver/core/silver__blocks.sql +++ b/models/silver/core/silver__blocks.sql @@ -3,7 +3,7 @@ materialized = 'incremental', unique_key = "block_number", cluster_by = "block_timestamp::date", - tags = ['core'], + tags = ['core','non_realtime'], full_refresh = false ) }} diff --git a/models/silver/core/silver__confirmed_blocks.sql b/models/silver/core/silver__confirmed_blocks.sql index 3a24502..91c0e69 100644 --- a/models/silver/core/silver__confirmed_blocks.sql +++ b/models/silver/core/silver__confirmed_blocks.sql @@ -4,7 +4,8 @@ incremental_strategy = 'delete+insert', unique_key = "block_number", cluster_by = "round(block_number,-3)", - full_refresh = false + full_refresh = false, + tags = ['non_realtime'] ) }} WITH base AS ( diff --git a/models/silver/contracts/silver__contracts.sql b/models/silver/core/silver__contracts.sql similarity index 97% rename from models/silver/contracts/silver__contracts.sql rename to models/silver/core/silver__contracts.sql index 15f6bd5..d25a11d 100644 --- a/models/silver/contracts/silver__contracts.sql +++ b/models/silver/core/silver__contracts.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = 'contract_address' + unique_key = 'contract_address', + tags = ['non_realtime'] ) }} WITH base_metadata AS ( diff --git a/models/silver/contracts/silver__contracts.yml b/models/silver/core/silver__contracts.yml similarity index 100% rename from models/silver/contracts/silver__contracts.yml rename to models/silver/core/silver__contracts.yml diff --git a/models/silver/contracts/silver__created_contracts.sql b/models/silver/core/silver__created_contracts.sql similarity index 90% rename from models/silver/contracts/silver__created_contracts.sql rename to models/silver/core/silver__created_contracts.sql index 8f12ad6..01178b5 100644 --- a/models/silver/contracts/silver__created_contracts.sql +++ b/models/silver/core/silver__created_contracts.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "incremental", - unique_key = "created_contract_address" + unique_key = "created_contract_address", + tags = ['non_realtime'] ) }} SELECT diff --git a/models/silver/contracts/silver__created_contracts.yml b/models/silver/core/silver__created_contracts.yml similarity index 100% rename from models/silver/contracts/silver__created_contracts.yml rename to models/silver/core/silver__created_contracts.yml diff --git a/models/silver/core/silver__decoded_logs.sql b/models/silver/core/silver__decoded_logs.sql index 4bff3ad..47b570b 100644 --- a/models/silver/core/silver__decoded_logs.sql +++ b/models/silver/core/silver__decoded_logs.sql @@ -4,7 +4,8 @@ unique_key = ['block_number', 'event_index'], cluster_by = "block_timestamp::date", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - full_refresh = false + full_refresh = false, + tags = ['decoded_logs'] ) }} WITH base_data AS ( diff --git a/models/silver/core/silver__logs.sql b/models/silver/core/silver__logs.sql index 135041d..1b2849e 100644 --- a/models/silver/core/silver__logs.sql +++ b/models/silver/core/silver__logs.sql @@ -4,7 +4,7 @@ unique_key = "block_number", cluster_by = "block_timestamp::date, _inserted_timestamp::date", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core'], + tags = ['core','non_realtime'], full_refresh = false ) }} diff --git a/models/silver/core/silver__receipts.sql b/models/silver/core/silver__receipts.sql index 71a42db..f774910 100644 --- a/models/silver/core/silver__receipts.sql +++ b/models/silver/core/silver__receipts.sql @@ -5,7 +5,7 @@ unique_key = "block_number", cluster_by = "ROUND(block_number, -3)", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(tx_hash)", - tags = ['core'], + tags = ['core','non_realtime'], full_refresh = false ) }} diff --git a/models/silver/API_udf/silver__relevant_contracts.sql b/models/silver/core/silver__relevant_contracts.sql similarity index 81% rename from models/silver/API_udf/silver__relevant_contracts.sql rename to models/silver/core/silver__relevant_contracts.sql index f404742..96b9cbc 100644 --- a/models/silver/API_udf/silver__relevant_contracts.sql +++ b/models/silver/core/silver__relevant_contracts.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'table', - unique_key = "contract_address" + unique_key = "contract_address", + tags = ['non_realtime'] ) }} SELECT diff --git a/models/silver/API_udf/silver__relevant_contracts.yml b/models/silver/core/silver__relevant_contracts.yml similarity index 100% rename from models/silver/API_udf/silver__relevant_contracts.yml rename to models/silver/core/silver__relevant_contracts.yml diff --git a/models/silver/core/silver__traces.sql b/models/silver/core/silver__traces.sql index ccd60f8..1a224fc 100644 --- a/models/silver/core/silver__traces.sql +++ b/models/silver/core/silver__traces.sql @@ -5,7 +5,7 @@ unique_key = "block_number", cluster_by = "block_timestamp::date, _inserted_timestamp::date", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core'], + tags = ['core','non_realtime'], full_refresh = false ) }} diff --git a/models/silver/core/silver__transactions.sql b/models/silver/core/silver__transactions.sql index 671a43e..c02d660 100644 --- a/models/silver/core/silver__transactions.sql +++ b/models/silver/core/silver__transactions.sql @@ -5,7 +5,7 @@ unique_key = "block_number", cluster_by = "block_timestamp::date, _inserted_timestamp::date", post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", - tags = ['core'], + tags = ['core','non_realtime'], full_refresh = false ) }} diff --git a/models/silver/core/silver__transfers.sql b/models/silver/core/silver__transfers.sql index 7a248b8..027ac76 100644 --- a/models/silver/core/silver__transfers.sql +++ b/models/silver/core/silver__transfers.sql @@ -1,7 +1,8 @@ {{ config( materialized = 'incremental', unique_key = '_log_id', - cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'] + cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'], + tags = ['non_realtime'] ) }} WITH logs AS ( diff --git a/models/silver/core/silver__tx_count.sql b/models/silver/core/silver__tx_count.sql index 0144d01..e90e919 100644 --- a/models/silver/core/silver__tx_count.sql +++ b/models/silver/core/silver__tx_count.sql @@ -1,6 +1,7 @@ {{ config( materialized = 'incremental', - unique_key = "block_number" + unique_key = "block_number", + tags = ['non_realtime'] ) }} SELECT diff --git a/models/bronze/core/bronze__streamline_FR_blocks.sql b/models/streamline/bronze/core/bronze__streamline_FR_blocks.sql similarity index 100% rename from models/bronze/core/bronze__streamline_FR_blocks.sql rename to models/streamline/bronze/core/bronze__streamline_FR_blocks.sql diff --git a/models/bronze/core/bronze__streamline_FR_confirm_blocks.sql b/models/streamline/bronze/core/bronze__streamline_FR_confirm_blocks.sql similarity index 100% rename from models/bronze/core/bronze__streamline_FR_confirm_blocks.sql rename to models/streamline/bronze/core/bronze__streamline_FR_confirm_blocks.sql diff --git a/models/bronze/core/bronze__streamline_FR_receipts.sql b/models/streamline/bronze/core/bronze__streamline_FR_receipts.sql similarity index 100% rename from models/bronze/core/bronze__streamline_FR_receipts.sql rename to models/streamline/bronze/core/bronze__streamline_FR_receipts.sql diff --git a/models/bronze/core/bronze__streamline_FR_traces.sql b/models/streamline/bronze/core/bronze__streamline_FR_traces.sql similarity index 100% rename from models/bronze/core/bronze__streamline_FR_traces.sql rename to models/streamline/bronze/core/bronze__streamline_FR_traces.sql diff --git a/models/bronze/core/bronze__streamline_FR_transactions.sql b/models/streamline/bronze/core/bronze__streamline_FR_transactions.sql similarity index 100% rename from models/bronze/core/bronze__streamline_FR_transactions.sql rename to models/streamline/bronze/core/bronze__streamline_FR_transactions.sql diff --git a/models/bronze/core/bronze__streamline_blocks.sql b/models/streamline/bronze/core/bronze__streamline_blocks.sql similarity index 100% rename from models/bronze/core/bronze__streamline_blocks.sql rename to models/streamline/bronze/core/bronze__streamline_blocks.sql diff --git a/models/bronze/core/bronze__streamline_confirm_blocks.sql b/models/streamline/bronze/core/bronze__streamline_confirm_blocks.sql similarity index 100% rename from models/bronze/core/bronze__streamline_confirm_blocks.sql rename to models/streamline/bronze/core/bronze__streamline_confirm_blocks.sql diff --git a/models/bronze/core/bronze__streamline_receipts.sql b/models/streamline/bronze/core/bronze__streamline_receipts.sql similarity index 100% rename from models/bronze/core/bronze__streamline_receipts.sql rename to models/streamline/bronze/core/bronze__streamline_receipts.sql diff --git a/models/bronze/core/bronze__streamline_traces.sql b/models/streamline/bronze/core/bronze__streamline_traces.sql similarity index 100% rename from models/bronze/core/bronze__streamline_traces.sql rename to models/streamline/bronze/core/bronze__streamline_traces.sql diff --git a/models/bronze/core/bronze__streamline_transactions.sql b/models/streamline/bronze/core/bronze__streamline_transactions.sql similarity index 100% rename from models/bronze/core/bronze__streamline_transactions.sql rename to models/streamline/bronze/core/bronze__streamline_transactions.sql diff --git a/models/bronze/decoder/bronze__decoded_logs.sql b/models/streamline/bronze/decoder/bronze__decoded_logs.sql similarity index 100% rename from models/bronze/decoder/bronze__decoded_logs.sql rename to models/streamline/bronze/decoder/bronze__decoded_logs.sql diff --git a/models/bronze/decoder/bronze__fr_decoded_logs.sql b/models/streamline/bronze/decoder/bronze__fr_decoded_logs.sql similarity index 100% rename from models/bronze/decoder/bronze__fr_decoded_logs.sql rename to models/streamline/bronze/decoder/bronze__fr_decoded_logs.sql diff --git a/models/silver/streamline/_max_block_by_date.sql b/models/streamline/silver/_max_block_by_date.sql similarity index 100% rename from models/silver/streamline/_max_block_by_date.sql rename to models/streamline/silver/_max_block_by_date.sql diff --git a/models/silver/streamline/_max_block_by_hour.sql b/models/streamline/silver/_max_block_by_hour.sql similarity index 100% rename from models/silver/streamline/_max_block_by_hour.sql rename to models/streamline/silver/_max_block_by_hour.sql diff --git a/models/silver/streamline/core/complete/streamline__complete_confirmed_blocks.sql b/models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql similarity index 100% rename from models/silver/streamline/core/complete/streamline__complete_confirmed_blocks.sql rename to models/streamline/silver/core/complete/streamline__complete_confirmed_blocks.sql diff --git a/models/silver/streamline/core/complete/streamline__complete_debug_traceBlockByNumber.sql b/models/streamline/silver/core/complete/streamline__complete_debug_traceBlockByNumber.sql similarity index 100% rename from models/silver/streamline/core/complete/streamline__complete_debug_traceBlockByNumber.sql rename to models/streamline/silver/core/complete/streamline__complete_debug_traceBlockByNumber.sql diff --git a/models/silver/streamline/core/complete/streamline__complete_qn_getBlockWithReceipts.sql b/models/streamline/silver/core/complete/streamline__complete_qn_getBlockWithReceipts.sql similarity index 100% rename from models/silver/streamline/core/complete/streamline__complete_qn_getBlockWithReceipts.sql rename to models/streamline/silver/core/complete/streamline__complete_qn_getBlockWithReceipts.sql diff --git a/models/silver/streamline/core/realtime/streamline__confirm_blocks_realtime.sql b/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql similarity index 96% rename from models/silver/streamline/core/realtime/streamline__confirm_blocks_realtime.sql rename to models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql index ceed6a3..2394dc6 100644 --- a/models/silver/streamline/core/realtime/streamline__confirm_blocks_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__confirm_blocks_realtime.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'confirm_blocks', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','100000')}}, 'batch_call_limit', {{var('batch_call_limit','10')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_core_realtime'] ) }} with tbl AS ( diff --git a/models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql b/models/streamline/silver/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql similarity index 97% rename from models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql rename to models/streamline/silver/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql index a6630e5..9eae42c 100644 --- a/models/silver/streamline/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__debug_traceBlockByNumber_realtime.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_get_traces(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'debug_traceBlockByNumber', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','50000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_core_realtime'] ) }} WITH blocks AS ( diff --git a/models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql b/models/streamline/silver/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql similarity index 97% rename from models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql rename to models/streamline/silver/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql index be21884..edf80fc 100644 --- a/models/silver/streamline/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql +++ b/models/streamline/silver/core/realtime/streamline__qn_getBlockWithReceipts_realtime.sql @@ -3,7 +3,8 @@ post_hook = if_data_call_function( func = "{{this.schema}}.udf_bulk_json_rpc(object_construct('sql_source', '{{this.identifier}}', 'external_table', 'qn_getBlockWithReceipts', 'sql_limit', {{var('sql_limit','100000')}}, 'producer_batch_size', {{var('producer_batch_size','100000')}}, 'worker_batch_size', {{var('worker_batch_size','100000')}}, 'batch_call_limit', {{var('batch_call_limit','10')}}))", target = "{{this.schema}}.{{this.identifier}}" - ) + ), + tags = ['streamline_core_realtime'] ) }} WITH blocks AS ( diff --git a/models/silver/streamline/core/retry/_missing_receipts.sql b/models/streamline/silver/core/retry/_missing_receipts.sql similarity index 100% rename from models/silver/streamline/core/retry/_missing_receipts.sql rename to models/streamline/silver/core/retry/_missing_receipts.sql diff --git a/models/silver/streamline/core/retry/_missing_traces.sql b/models/streamline/silver/core/retry/_missing_traces.sql similarity index 100% rename from models/silver/streamline/core/retry/_missing_traces.sql rename to models/streamline/silver/core/retry/_missing_traces.sql diff --git a/models/silver/streamline/core/retry/_missing_txs.sql b/models/streamline/silver/core/retry/_missing_txs.sql similarity index 100% rename from models/silver/streamline/core/retry/_missing_txs.sql rename to models/streamline/silver/core/retry/_missing_txs.sql diff --git a/models/silver/streamline/core/retry/_unconfirmed_blocks.sql b/models/streamline/silver/core/retry/_unconfirmed_blocks.sql similarity index 100% rename from models/silver/streamline/core/retry/_unconfirmed_blocks.sql rename to models/streamline/silver/core/retry/_unconfirmed_blocks.sql diff --git a/models/silver/streamline/core/streamline__blocks.sql b/models/streamline/silver/core/streamline__blocks.sql similarity index 100% rename from models/silver/streamline/core/streamline__blocks.sql rename to models/streamline/silver/core/streamline__blocks.sql diff --git a/models/silver/streamline/decoder/streamline__complete_decode_logs.sql b/models/streamline/silver/decoder/complete/streamline__complete_decode_logs.sql similarity index 100% rename from models/silver/streamline/decoder/streamline__complete_decode_logs.sql rename to models/streamline/silver/decoder/complete/streamline__complete_decode_logs.sql diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_000000001_002500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_000000001_002500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_000000001_002500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_000000001_002500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_000000001_002500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_000000001_002500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_002500001_005000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_002500001_005000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_002500001_005000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_002500001_005000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_002500001_005000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_002500001_005000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_005000001_007500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_005000001_007500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_005000001_007500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_005000001_007500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_005000001_007500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_005000001_007500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_010000001_012500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_010000001_012500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_010000001_012500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_010000001_012500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_010000001_012500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_010000001_012500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_012500001_015000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_012500001_015000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_012500001_015000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_012500001_015000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_012500001_015000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_012500001_015000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_015000001_017500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_015000001_017500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_015000001_017500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_015000001_017500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_015000001_017500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_015000001_017500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_017500001_020000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_017500001_020000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_017500001_020000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_017500001_020000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_017500001_020000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_017500001_020000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_020000001_022500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_020000001_022500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_020000001_022500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_020000001_022500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_020000001_022500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_020000001_022500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_022500001_025000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_022500001_025000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_022500001_025000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_022500001_025000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_022500001_025000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_022500001_025000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_025000001_027500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_025000001_027500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_025000001_027500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_025000001_027500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_025000001_027500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_025000001_027500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_027500001_030000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_027500001_030000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_027500001_030000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_027500001_030000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_027500001_030000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_027500001_030000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_030000001_032500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_030000001_032500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_030000001_032500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_030000001_032500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_030000001_032500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_030000001_032500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_032500001_035000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_032500001_035000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_032500001_035000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_032500001_035000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_032500001_035000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_032500001_035000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_035000001_037500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_035000001_037500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_035000001_037500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_035000001_037500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_035000001_037500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_035000001_037500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_037500001_040000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_037500001_040000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_037500001_040000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_037500001_040000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_037500001_040000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_037500001_040000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_040000001_042500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_040000001_042500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_040000001_042500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_040000001_042500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_040000001_042500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_040000001_042500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_042500001_045000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_042500001_045000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_042500001_045000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_042500001_045000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_042500001_045000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_042500001_045000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_045000001_047500000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_045000001_047500000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_045000001_047500000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_045000001_047500000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_045000001_047500000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_045000001_047500000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_047500001_050000000.sql b/models/streamline/silver/decoder/history/streamline__decode_logs_history_047500001_050000000.sql similarity index 84% rename from models/silver/streamline/decoder/history/streamline__decode_logs_history_047500001_050000000.sql rename to models/streamline/silver/decoder/history/streamline__decode_logs_history_047500001_050000000.sql index d59dde6..cb4ad80 100644 --- a/models/silver/streamline/decoder/history/streamline__decode_logs_history_047500001_050000000.sql +++ b/models/streamline/silver/decoder/history/streamline__decode_logs_history_047500001_050000000.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()] + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] ) }} {% set start = this.identifier.split("_") [-2] %} diff --git a/models/silver/streamline/decoder/streamline__decode_logs_realtime.sql b/models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql similarity index 94% rename from models/silver/streamline/decoder/streamline__decode_logs_realtime.sql rename to models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql index f721fc3..16d909d 100644 --- a/models/silver/streamline/decoder/streamline__decode_logs_realtime.sql +++ b/models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql @@ -1,6 +1,7 @@ {{ config ( materialized = "view", - post_hook = [if_data_call_function( func = "{{this.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{this.identifier}}','producer_batch_size', 20000000,'producer_limit_size', 20000000))", target = "{{this.schema}}.{{this.identifier}}" ),"call system$wait(" ~ var("WAIT", 400) ~ ")" ] + post_hook = [if_data_call_function( func = "{{this.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{this.identifier}}','producer_batch_size', 20000000,'producer_limit_size', 20000000))", target = "{{this.schema}}.{{this.identifier}}" ),"call system$wait(" ~ var("WAIT", 400) ~ ")" ], + tags = ['streamline_decoded_logs_realtime'] ) }} WITH look_back AS (