From 9fe8ac4ba8dd20475bead42678199c950c2a55f3 Mon Sep 17 00:00:00 2001 From: gregoriustanleyy Date: Fri, 29 Aug 2025 14:20:00 +0700 Subject: [PATCH] v1 namespace --- .../v1/bronze__streamline_blocks_v1.sql | 34 +++++++++++++++++++ .../v1/bronze__streamline_collections_v1.sql | 11 ++++++ .../v1/bronze__streamline_fr_blocks_v1.sql | 34 +++++++++++++++++++ .../bronze__streamline_fr_collections_v1.sql | 11 ++++++ ...__streamline_fr_transaction_results_v1.sql | 13 +++++++ .../bronze__streamline_fr_transactions_v1.sql | 11 ++++++ ...nze__streamline_transaction_results_v1.sql | 11 ++++++ .../v1/bronze__streamline_transactions_v1.sql | 11 ++++++ 8 files changed, 136 insertions(+) create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_blocks_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_collections_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_fr_blocks_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_fr_collections_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_fr_transaction_results_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_fr_transactions_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_transaction_results_v1.sql create mode 100644 models/bronze/streamline/realtime/v1/bronze__streamline_transactions_v1.sql diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_blocks_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_blocks_v1.sql new file mode 100644 index 0000000..fffa5d5 --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_blocks_v1.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + SELECT + last_modified AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", "blocks") }}') + ) A + ) + SELECT + block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE + FROM + {{ source("bronze_streamline","blocks") }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id + WHERE + b._partition_by_block_id = s._partition_by_block_id \ No newline at end of file diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_collections_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_collections_v1.sql new file mode 100644 index 0000000..a2614fa --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_collections_v1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + model = "collections", + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_fr_blocks_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_blocks_v1.sql new file mode 100644 index 0000000..dc349d9 --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_blocks_v1.sql @@ -0,0 +1,34 @@ +{{ config ( +materialized = 'view' +) }} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "blocks") }}' + ) + ) A +) +SELECT + block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source("bronze_streamline","blocks") }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id \ No newline at end of file diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_fr_collections_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_collections_v1.sql new file mode 100644 index 0000000..ec83dce --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_collections_v1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'collections', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_fr_transaction_results_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_transaction_results_v1.sql new file mode 100644 index 0000000..230bc83 --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_transaction_results_v1.sql @@ -0,0 +1,13 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-2:] | join('_') %} +{{ streamline_external_table_FR_query( + model = 'transaction_results', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} + + diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_fr_transactions_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_transactions_v1.sql new file mode 100644 index 0000000..c8c73c8 --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_fr_transactions_v1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_FR_query( + model = 'transactions', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_transaction_results_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_transaction_results_v1.sql new file mode 100644 index 0000000..08b2434 --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_transaction_results_v1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-2:] | join('_') %} +{{ streamline_external_table_query( + model = 'transaction_results', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/realtime/v1/bronze__streamline_transactions_v1.sql b/models/bronze/streamline/realtime/v1/bronze__streamline_transactions_v1.sql new file mode 100644 index 0000000..8a3fef0 --- /dev/null +++ b/models/bronze/streamline/realtime/v1/bronze__streamline_transactions_v1.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_") [-1] %} +{{ streamline_external_table_query( + model = 'transactions', + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }}