diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 4c79ceb..ab258d5 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -41,4 +41,5 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt run --exclude models/silver/ABIs/silver__relevant_abi_contracts.sql models/silver/abis \ No newline at end of file + dbt run --exclude models/silver/abis/silver__relevant_abi_contracts.sql models/silver/abis models/silver/streamline + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+tag:streamline_decoded_logs_realtime \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_decoder_history.yml b/.github/workflows/dbt_run_streamline_decoder_history.yml new file mode 100644 index 0000000..d2ccb8c --- /dev/null +++ b/.github/workflows/dbt_run_streamline_decoder_history.yml @@ -0,0 +1,45 @@ +name: dbt_run_streamline_decoder_history +run-name: dbt_run_streamline_decoder_history + +on: + workflow_dispatch: + schedule: + # Runs "at 10:10 UTC AM" (see https://crontab.guru) + - cron: '10 10 * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod_backfill + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120}' -m 1+tag:streamline_decoded_logs_history \ No newline at end of file diff --git a/macros/dbt/get_merge_sql.sql b/macros/dbt/get_merge_sql.sql new file mode 100644 index 0000000..8fefc01 --- /dev/null +++ b/macros/dbt/get_merge_sql.sql @@ -0,0 +1,44 @@ +{% macro get_merge_sql( + target, + source, + unique_key, + dest_columns, + incremental_predicates + ) -%} + {% set predicate_override = "" %} + {% if incremental_predicates [0] == "dynamic_range" %} + -- run some queries to dynamically determine the min + max of this 'input_column' in the new data + {% set input_column = incremental_predicates [1] %} + {% set get_limits_query %} + SELECT + MIN( + {{ input_column }} + ) AS lower_limit, + MAX( + {{ input_column }} + ) AS upper_limit + FROM + {{ source }} + + {% endset %} + {% set limits = run_query(get_limits_query) [0] %} + {% set lower_limit, + upper_limit = limits [0], + limits [1] %} + -- use those calculated min + max values to limit 'target' scan, to only the days with new data + {% set predicate_override %} + dbt_internal_dest.{{ input_column }} BETWEEN '{{ lower_limit }}' + AND '{{ upper_limit }}' {% endset %} + {% endif %} + + {% set predicates = [predicate_override] if predicate_override else incremental_predicates %} + -- standard merge from here + {% set merge_sql = dbt.get_merge_sql( + target, + source, + unique_key, + dest_columns, + predicates + ) %} + {{ return(merge_sql) }} +{% endmacro %} diff --git a/macros/dbt/get_tmp_relation_type.sql b/macros/dbt/get_tmp_relation_type.sql new file mode 100644 index 0000000..3bb7438 --- /dev/null +++ b/macros/dbt/get_tmp_relation_type.sql @@ -0,0 +1,8 @@ +{% macro dbt_snowflake_get_tmp_relation_type( + strategy, + unique_key, + language + ) %} + -- always table + {{ return('table') }} +{% endmacro %} diff --git a/models/bronze/decoder/bronze__decoded_logs.sql b/models/bronze/decoder/bronze__decoded_logs.sql new file mode 100644 index 0000000..81c61bf --- /dev/null +++ b/models/bronze/decoder/bronze__decoded_logs.sql @@ -0,0 +1,41 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + + SELECT + last_modified AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", "decoded_logs") }}') + ) A + ) + SELECT + block_number, + id :: STRING AS id, + DATA, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date + FROM + {{ source( + "bronze_streamline", + "decoded_logs" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date + AND s._partition_by_created_date >= DATEADD('day', -2, CURRENT_TIMESTAMP()) \ No newline at end of file diff --git a/models/bronze/decoder/bronze__fr_decoded_logs.sql b/models/bronze/decoder/bronze__fr_decoded_logs.sql new file mode 100644 index 0000000..f40c439 --- /dev/null +++ b/models/bronze/decoder/bronze__fr_decoded_logs.sql @@ -0,0 +1,40 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 6), '_', 1) AS INTEGER) AS _partition_by_block_number, + TO_DATE( + concat_ws('-', SPLIT_PART(file_name, '/', 3), SPLIT_PART(file_name, '/', 4), SPLIT_PART(file_name, '/', 5)) + ) AS _partition_by_created_date + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "decoded_logs") }}' + ) + ) A +) +SELECT + block_number, + id :: STRING AS id, + DATA, + _inserted_timestamp, + s._partition_by_block_number AS _partition_by_block_number, + s._partition_by_created_date AS _partition_by_created_date +FROM + {{ source( + "bronze_streamline", + "decoded_logs" + ) }} + s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date +WHERE + b._partition_by_block_number = s._partition_by_block_number + AND b._partition_by_created_date = s._partition_by_created_date \ No newline at end of file diff --git a/models/silver/streamline/decoder/complete/streamline__complete_decode_logs.sql b/models/silver/streamline/decoder/complete/streamline__complete_decode_logs.sql new file mode 100644 index 0000000..7f470a0 --- /dev/null +++ b/models/silver/streamline/decoder/complete/streamline__complete_decode_logs.sql @@ -0,0 +1,31 @@ +-- depends_on: {{ ref('bronze__decoded_logs') }} +{{ config ( + materialized = "incremental", + unique_key = "_log_id", + cluster_by = "ROUND(block_number, -3)", + incremental_predicates = ["dynamic_range", "block_number"], + merge_update_columns = ["_log_id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(_log_id)" +) }} + +SELECT + block_number, + id AS _log_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__decoded_logs') }} +WHERE + TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( + SELECT + COALESCE(MAX(TO_TIMESTAMP_NTZ(_inserted_timestamp)), '1970-01-01 00:00:00') _inserted_timestamp + FROM + {{ this }}) + {% else %} + {{ ref('bronze__fr_decoded_logs') }} + {% endif %} + + qualify(ROW_NUMBER() over (PARTITION BY id + ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_013840178_014632549.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_013840178_014632549.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_013840178_014632549.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_014632550_015112214.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_014632550_015112214.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_014632550_015112214.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_015112215_015604044.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_015112215_015604044.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_015112215_015604044.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_015604045_015969013.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_015604045_015969013.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_015604045_015969013.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_015969014_016201644.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_015969014_016201644.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_015969014_016201644.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_016201645_016415916.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_016201645_016415916.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_016201645_016415916.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_016415917_016800900.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_016415917_016800900.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_016415917_016800900.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_016800901_017157990.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_016800901_017157990.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_016800901_017157990.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_017157991_017518790.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_017157991_017518790.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_017157991_017518790.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_017518791_017784488.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_017518791_017784488.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_017518791_017784488.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_017784489_018091163.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_017784489_018091163.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_017784489_018091163.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_018091164_018342530.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018091164_018342530.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018091164_018342530.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_018342531_018617404.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018342531_018617404.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018342531_018617404.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_018617405_018921443.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018617405_018921443.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018617405_018921443.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_018921445_019234360.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018921445_019234360.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_018921445_019234360.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_019234361_019505163.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_019234361_019505163.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_019234361_019505163.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_019505164_019840917.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_019505164_019840917.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_019505164_019840917.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_019840918_020156967.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_019840918_020156967.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_019840918_020156967.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_020156968_020569110.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_020156968_020569110.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_020156968_020569110.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_020569111_020911283.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_020569111_020911283.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_020569111_020911283.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_020911284_021393083.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_020911284_021393083.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_020911284_021393083.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_021393084_021880883.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_021393084_021880883.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_021393084_021880883.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_021880884_022537444.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_021880884_022537444.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_021880884_022537444.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_022537445_023549227.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_022537445_023549227.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_022537445_023549227.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_023549229_024148433.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_023549229_024148433.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_023549229_024148433.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_024148434_024746151.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_024148434_024746151.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_024148434_024746151.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_024746152_026007441.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_024746152_026007441.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_024746152_026007441.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_026007442_026955035.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_026007442_026955035.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_026007442_026955035.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_026955036_027951181.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_026955036_027951181.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_026955036_027951181.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_027951183_028031419.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_027951183_028031419.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_027951183_028031419.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028031420_028065178.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028031420_028065178.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028031420_028065178.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028065179_028171361.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028065179_028171361.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028065179_028171361.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028171362_028204557.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028171362_028204557.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028171362_028204557.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028204558_028260925.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028204558_028260925.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028204558_028260925.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028260926_028361890.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028260926_028361890.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028260926_028361890.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028361891_028428662.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028361891_028428662.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028361891_028428662.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028428663_028567857.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028428663_028567857.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028428663_028567857.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028567858_028644911.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028567858_028644911.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028567858_028644911.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028644912_028750894.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028644912_028750894.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028644912_028750894.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028750895_028763479.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028750895_028763479.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028750895_028763479.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028763480_028816239.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028763480_028816239.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028763480_028816239.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028816240_028849705.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028816240_028849705.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028816240_028849705.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028849706_028934611.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028849706_028934611.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028849706_028934611.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_028934612_029014103.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028934612_029014103.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_028934612_029014103.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029014104_029048865.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029014104_029048865.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029014104_029048865.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029048866_029081108.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029048866_029081108.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029048866_029081108.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029081109_029130260.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029081109_029130260.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029081109_029130260.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029130261_029195437.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029130261_029195437.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029130261_029195437.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029195438_029199346.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029195438_029199346.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029195438_029199346.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029199347_029228286.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029199347_029228286.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029199347_029228286.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_029228287_029263078.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029228287_029263078.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_029228287_029263078.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/history/streamline__decode_logs_history_07500134_013840177.sql b/models/silver/streamline/decoder/history/streamline__decode_logs_history_07500134_013840177.sql new file mode 100644 index 0000000..cb4ad80 --- /dev/null +++ b/models/silver/streamline/decoder/history/streamline__decode_logs_history_07500134_013840177.sql @@ -0,0 +1,12 @@ +{{ config ( + materialized = "view", + post_hook = [if_data_call_function( func = "{{model.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{model.alias}}','producer_batch_size', 20000000,'producer_limit_size', {{var('row_limit',7500000)}}))", target = "{{model.schema}}.{{model.alias}}" ) ,if_data_call_wait()], + tags = ['streamline_decoded_logs_history'] +) }} + +{% set start = this.identifier.split("_") [-2] %} +{% set stop = this.identifier.split("_") [-1] %} +{{ fsc_utils.decode_logs_history( + start, + stop +) }} diff --git a/models/silver/streamline/decoder/streamline__decode_logs_realtime.sql b/models/silver/streamline/decoder/realtime/streamline__decode_logs_realtime.sql similarity index 52% rename from models/silver/streamline/decoder/streamline__decode_logs_realtime.sql rename to models/silver/streamline/decoder/realtime/streamline__decode_logs_realtime.sql index 6ec8278..95410ef 100644 --- a/models/silver/streamline/decoder/streamline__decode_logs_realtime.sql +++ b/models/silver/streamline/decoder/realtime/streamline__decode_logs_realtime.sql @@ -1,5 +1,7 @@ {{ config ( - materialized = "view" + materialized = "view", + post_hook = [if_data_call_function( func = "{{this.schema}}.udf_bulk_decode_logs(object_construct('sql_source', '{{this.identifier}}','producer_batch_size', 20000000,'producer_limit_size', 20000000))", target = "{{this.schema}}.{{this.identifier}}" ),"call system$wait(" ~ var("WAIT", 400) ~ ")" ], + tags = ['streamline_decoded_logs_realtime'] ) }} WITH look_back AS ( @@ -43,3 +45,18 @@ WHERE ) ) AND l.block_number IS NOT NULL + AND l.block_timestamp >= DATEADD('day', -2, CURRENT_DATE()) + AND _log_id NOT IN ( + SELECT + _log_id + FROM + {{ ref("streamline__complete_decode_logs") }} + WHERE + block_number >= ( + SELECT + block_number + FROM + look_back + ) + AND _inserted_timestamp >= DATEADD('day', -2, CURRENT_DATE())) + diff --git a/models/sources.yml b/models/sources.yml index f9e11d2..79f8730 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -46,4 +46,3 @@ sources: - name: debug_traceBlockByNumber - name: decoded_logs - name: confirm_blocks - \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec44b06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dbt-snowflake>=1.4,<1.5 \ No newline at end of file