remove v1

models/bronze/core/bronze__blocks.sql

@@ -1,6 +1,6 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
 WITH meta AS (

models/bronze/core/bronze__blocks_fr.sql

@@ -1,26 +1,39 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
-SELECT
-    partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
+    WITH meta AS (
+        SELECT
+            registered_on AS _inserted_timestamp,
             file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__blocks_fr_v2') }}
-UNION ALL
+            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
+        FROM
+            TABLE(
+                information_schema.external_table_files(
+                    table_name => '{{ source( "bronze_streamline", "blocks_v2") }}'
+                )
+            ) A
+    )
 SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
+    s.*,
+    b.file_name,
+    b._inserted_timestamp,
+    COALESCE(
+        s.value :"BLOCK_NUMBER" :: STRING,
+        s.value :"block_number" :: STRING,
+        s.metadata :request :"data" :id :: STRING,
+        PARSE_JSON(
+            s.metadata :request :"data"
+        ) :id :: STRING
+    ) :: INT AS block_number
 FROM
-   {{ ref('bronze__blocks_fr_v1') }}
+    {{ source( "bronze_streamline", "blocks_v2") }}
+    s
+    JOIN meta b
+    ON b.file_name = metadata$filename
+    AND b.partition_key = s.partition_key
+WHERE
+    b.partition_key = s.partition_key
+    AND DATA :error IS NULL
+    AND DATA IS NOT NULL

models/bronze/core/bronze__blocks_fr_v1.sql

@@ -1,54 +0,0 @@
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-    WITH meta AS (
-        SELECT
-            registered_on AS _inserted_timestamp,
-            file_name,
-            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS _partition_by_block_id
-        FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "blocks") }}'
-                )
-            ) A
-    )
-SELECT
-    block_number,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp,
-    MD5(
-        CAST(
-            COALESCE(CAST(block_number AS text), '' :: STRING) AS text
-        )
-    ) AS id,
-    s._partition_by_block_id,
-    s.value AS VALUE
-FROM
-    {{ source( "bronze_streamline", "blocks") }}
-    s
-    JOIN meta b
-    ON b.file_name = metadata$filename
-    AND b._partition_by_block_id = s._partition_by_block_id
-WHERE
-    b._partition_by_block_id = s._partition_by_block_id
-    AND (
-        DATA :error :code IS NULL
-        OR DATA :error :code NOT IN (
-            '-32000',
-            '-32001',
-            '-32002',
-            '-32003',
-            '-32004',
-            '-32005',
-            '-32006',
-            '-32007',
-            '-32008',
-            '-32009',
-            '-32010'
-        )
-    )

models/bronze/core/bronze__blocks_fr_v2.sql

@@ -1,39 +0,0 @@
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-    WITH meta AS (
-        SELECT
-            registered_on AS _inserted_timestamp,
-            file_name,
-            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
-        FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "blocks_v2") }}'
-                )
-            ) A
-    )
-SELECT
-    s.*,
-    b.file_name,
-    b._inserted_timestamp,
-    COALESCE(
-        s.value :"BLOCK_NUMBER" :: STRING,
-        s.value :"block_number" :: STRING,
-        s.metadata :request :"data" :id :: STRING,
-        PARSE_JSON(
-            s.metadata :request :"data"
-        ) :id :: STRING
-    ) :: INT AS block_number
-FROM
-    {{ source( "bronze_streamline", "blocks_v2") }}
-    s
-    JOIN meta b
-    ON b.file_name = metadata$filename
-    AND b.partition_key = s.partition_key
-WHERE
-    b.partition_key = s.partition_key
-    AND DATA :error IS NULL
-    AND DATA IS NOT NULL

models/bronze/core/bronze__receipts.sql

@@ -1,6 +1,6 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
 WITH meta AS (

models/bronze/core/bronze__receipts_fr.sql

@@ -3,26 +3,38 @@
     tags = ['bronze','core','streamline_v1','phase_1']
 ) }}
 
-SELECT
-    partition_key,
-    block_number,
-    tx_hash,
-    VALUE,
-    DATA,
-    metadata,
+    WITH meta AS (
+        SELECT
+            registered_on AS _inserted_timestamp,
             file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__receipts_fr_v2') }}
-UNION ALL
+            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
+        FROM
+            TABLE(
+                information_schema.external_table_files(
+                    table_name => '{{ source( "bronze_streamline", "receipts_by_hash") }}'
+                )
+            ) A
+    )
 SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    tx_hash,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
+    s.*,
+    b.file_name,
+    b._inserted_timestamp,
+    COALESCE(
+        s.value :"BLOCK_NUMBER" :: STRING,
+        s.value :"block_number" :: STRING,
+        s.metadata :request :"data" :id :: STRING,
+        PARSE_JSON(
+            s.metadata :request :"data"
+        ) :id :: STRING
+    ) :: INT AS block_number,
+    s.value: "TX_HASH" :: STRING AS tx_hash
 FROM
-   {{ ref('bronze__receipts_fr_v1') }}
+    {{ source( "bronze_streamline", "receipts_by_hash") }}
+    s
+    JOIN meta b
+    ON b.file_name = metadata$filename
+    AND b.partition_key = s.partition_key
+WHERE
+    b.partition_key = s.partition_key
+    AND DATA :error IS NULL
+    AND DATA IS NOT NULL

models/bronze/core/bronze__receipts_fr_v1.sql

@@ -1,58 +0,0 @@
-{{ config (
-    materialized = 'view'
-) }}
-
-WITH meta AS (
-
-    SELECT
-        last_modified AS _inserted_timestamp,
-        file_name,
-        CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER) AS _partition_by_block_id
-    FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "tx_receipts") }}'
-                )
-            ) A
-)
-SELECT
-    block_number,
-    value :data :result :transactionHash ::STRING AS tx_hash,
-    _inserted_timestamp,
-    MD5(
-        CAST(
-            COALESCE(CAST(CONCAT(block_number, '_-_', COALESCE(value :data :result :transactionHash ::STRING, '')) AS text), '' :: STRING) AS text
-        )
-    ) AS id,
-    s._partition_by_block_id,
-    s.value:data:result AS VALUE,
-    data,
-    metadata,
-    file_name
-FROM
-    {{ source(
-        "bronze_streamline",
-        "tx_receipts"
-    ) }}
-    s
-    JOIN meta b
-    ON b.file_name = metadata$filename
-    AND b._partition_by_block_id = s._partition_by_block_id
-WHERE
-    b._partition_by_block_id = s._partition_by_block_id
-    AND (
-        DATA :error :code IS NULL
-        OR DATA :error :code NOT IN (
-            '-32000',
-            '-32001',
-            '-32002',
-            '-32003',
-            '-32004',
-            '-32005',
-            '-32006',
-            '-32007',
-            '-32008',
-            '-32009',
-            '-32010'
-        )
-    )

models/bronze/core/bronze__receipts_fr_v2.sql

@@ -1,40 +0,0 @@
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-    WITH meta AS (
-        SELECT
-            registered_on AS _inserted_timestamp,
-            file_name,
-            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
-        FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "receipts_by_hash") }}'
-                )
-            ) A
-    )
-SELECT
-    s.*,
-    b.file_name,
-    b._inserted_timestamp,
-    COALESCE(
-        s.value :"BLOCK_NUMBER" :: STRING,
-        s.value :"block_number" :: STRING,
-        s.metadata :request :"data" :id :: STRING,
-        PARSE_JSON(
-            s.metadata :request :"data"
-        ) :id :: STRING
-    ) :: INT AS block_number,
-    s.value: "TX_HASH" :: STRING AS tx_hash
-FROM
-    {{ source( "bronze_streamline", "receipts_by_hash") }}
-    s
-    JOIN meta b
-    ON b.file_name = metadata$filename
-    AND b.partition_key = s.partition_key
-WHERE
-    b.partition_key = s.partition_key
-    AND DATA :error IS NULL
-    AND DATA IS NOT NULL

models/bronze/core/bronze__traces.sql

@@ -1,6 +1,6 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
 WITH meta AS (

models/bronze/core/bronze__traces_fr.sql

@@ -1,28 +1,40 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
-SELECT
-    partition_key,
-    block_number,
-    tx_hash,
-    VALUE,
-    DATA,
-    metadata,
+    WITH meta AS (
+        SELECT
+            registered_on AS _inserted_timestamp,
             file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__traces_fr_v2') }}
-UNION ALL
+            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
+        FROM
+            TABLE(
+                information_schema.external_table_files(
+                    table_name => '{{ source( "bronze_streamline", "traces_by_hash") }}'
+                )
+            ) A
+    )
 SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    tx_hash,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
+    s.*,
+    b.file_name,
+    b._inserted_timestamp,
+    COALESCE(
+        s.value :"BLOCK_NUMBER" :: STRING,
+        s.value :"block_number" :: STRING,
+        s.metadata :request :"data" :id :: STRING,
+        PARSE_JSON(
+            s.metadata :request :"data"
+        ) :id :: STRING
+    ) :: INT AS block_number,
+    s.value: "TX_HASH" :: STRING AS tx_hash
 FROM
-   {{ ref('bronze__traces_fr_v1') }}
+    {{ source( "bronze_streamline", "traces_by_hash") }}
+    s
+    JOIN meta b
+    ON b.file_name = metadata$filename
+    AND b.partition_key = s.partition_key
+WHERE
+    b.partition_key = s.partition_key
+    AND DATA :error IS NULL
+    AND DATA IS NOT NULL

models/bronze/core/bronze__traces_fr_v1.sql

@@ -1,59 +0,0 @@
-{{ config (
-    materialized = 'view'
-) }}
-
-WITH meta AS (
-
-    SELECT
-        last_modified AS _inserted_timestamp,
-        file_name,
-        CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER) AS _partition_by_block_id
-    FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "traces") }}'
-                )
-            ) A
-    )
-    SELECT
-        block_number,
-        s.value :metadata :request :params [0] :: STRING AS tx_hash,
-        DATA,
-        _inserted_timestamp,
-        MD5(
-            CAST(
-                COALESCE(CAST(tx_hash AS text), '' :: STRING) AS text
-            )
-        ) AS id,
-        s._partition_by_block_id,
-        s.value AS VALUE,
-        metadata,
-        file_name
-    FROM
-        {{ source(
-            "bronze_streamline",
-            "traces"
-        ) }}
-        s
-        JOIN meta b
-        ON b.file_name = metadata$filename
-        AND b._partition_by_block_id = s._partition_by_block_id
-    WHERE
-        b._partition_by_block_id = s._partition_by_block_id
-        AND (
-            DATA :error :code IS NULL
-            OR DATA :error :code NOT IN (
-                '-32000',
-                '-32001',
-                '-32002',
-                '-32003',
-                '-32004',
-                '-32005',
-                '-32006',
-                '-32007',
-                '-32008',
-                '-32009',
-                '-32010',
-                '-32608'
-            )
-        )

models/bronze/core/bronze__traces_fr_v2.sql

@@ -1,40 +0,0 @@
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-    WITH meta AS (
-        SELECT
-            registered_on AS _inserted_timestamp,
-            file_name,
-            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
-        FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "traces_by_hash") }}'
-                )
-            ) A
-    )
-SELECT
-    s.*,
-    b.file_name,
-    b._inserted_timestamp,
-    COALESCE(
-        s.value :"BLOCK_NUMBER" :: STRING,
-        s.value :"block_number" :: STRING,
-        s.metadata :request :"data" :id :: STRING,
-        PARSE_JSON(
-            s.metadata :request :"data"
-        ) :id :: STRING
-    ) :: INT AS block_number,
-    s.value: "TX_HASH" :: STRING AS tx_hash
-FROM
-    {{ source( "bronze_streamline", "traces_by_hash") }}
-    s
-    JOIN meta b
-    ON b.file_name = metadata$filename
-    AND b.partition_key = s.partition_key
-WHERE
-    b.partition_key = s.partition_key
-    AND DATA :error IS NULL
-    AND DATA IS NOT NULL

models/bronze/core/bronze__transactions.sql

@@ -1,6 +1,6 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
 WITH meta AS (

models/bronze/core/bronze__transactions_fr.sql

@@ -1,26 +1,39 @@
 {{ config (
     materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
+    tags = ['bronze','core','streamline_v2','phase_1']
 ) }}
 
-SELECT
-    partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
+    WITH meta AS (
+        SELECT
+            registered_on AS _inserted_timestamp,
             file_name,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__transactions_fr_v2') }}
-UNION ALL
+            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
+        FROM
+            TABLE(
+                information_schema.external_table_files(
+                    table_name => '{{ source( "bronze_streamline", "transactions_v2") }}'
+                )
+            ) A
+    )
 SELECT
-    _partition_by_block_id AS partition_key,
-    block_number,
-    VALUE,
-    DATA,
-    metadata,
-    file_name,
-    _inserted_timestamp
+    s.*,
+    b.file_name,
+    b._inserted_timestamp,
+    COALESCE(
+        s.value :"BLOCK_NUMBER" :: STRING,
+        s.value :"block_number" :: STRING,
+        s.metadata :request :"data" :id :: STRING,
+        PARSE_JSON(
+            s.metadata :request :"data"
+        ) :id :: STRING
+    ) :: INT AS block_number
 FROM
-   {{ ref('bronze__transactions_fr_v1') }}
+    {{ source( "bronze_streamline", "transactions_v2") }}
+    s
+    JOIN meta b
+    ON b.file_name = metadata$filename
+    AND b.partition_key = s.partition_key
+WHERE
+    b.partition_key = s.partition_key
+    AND DATA :error IS NULL
+    AND DATA IS NOT NULL

models/bronze/core/bronze__transactions_fr_v1.sql

@@ -1,86 +0,0 @@
-{{ config (
-    materialized = 'view'
-) }}
-
-WITH meta AS (
-
-    SELECT
-        last_modified AS _inserted_timestamp,
-        file_name,
-        CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER) AS _partition_by_block_id
-    FROM
-        TABLE(
-            information_schema.external_table_file_registration_history(
-                start_time => GREATEST(DATEADD('day', -1, CURRENT_TIMESTAMP), '2023-08-01 18:44:00.000' :: timestamp_ntz),
-                table_name => '{{ source( "bronze_streamline", "transactions") }}'
-            )
-        ) A
-),
-tbl AS (
-    SELECT
-        block_number,
-        COALESCE(s.data :hash :: STRING, s.data :result :hash :: STRING) AS tx_hash,
-        _inserted_timestamp,
-        metadata,
-        file_name,
-        s._partition_by_block_id,
-        s.value AS VALUE,
-        s.data AS DATA
-    FROM
-        {{ source(
-            "bronze_streamline",
-            "transactions"
-        ) }}
-        s
-        JOIN meta b
-        ON b.file_name = metadata$filename
-        AND b._partition_by_block_id = s._partition_by_block_id
-    WHERE
-        b._partition_by_block_id = s._partition_by_block_id
-        AND (
-            DATA :error :code IS NULL
-            OR DATA :error :code NOT IN (
-                '-32000',
-                '-32001',
-                '-32002',
-                '-32003',
-                '-32004',
-                '-32005',
-                '-32006',
-                '-32007',
-                '-32008',
-                '-32009',
-                '-32010'
-            )
-        )
-)
-SELECT
-    block_number,
-    COALESCE(f.value :hash :: STRING, tx_hash) AS tx_hash,
-    _inserted_timestamp,
-    MD5(
-        CAST(
-            COALESCE(
-                CAST(
-                    CONCAT(
-                        block_number,
-                        '_-_',
-                        COALESCE(COALESCE(f.value :hash :: STRING, tx_hash), '')
-                    ) AS text
-                ),
-                '' :: STRING
-            ) AS text
-        )
-    ) AS id,
-    data,
-    metadata,
-    file_name,
-    _partition_by_block_id,
-    _inserted_timestamp,
-    COALESCE(f.value, tbl.data:result, tbl.data) AS VALUE
-FROM
-    tbl,
-    LATERAL FLATTEN(
-        input => VALUE :data :result :transactions, OUTER => TRUE
-    ) f
-WHERE f.value IS NOT NULL OR tbl.data :transactionIndex IS NOT NULL 

models/bronze/core/bronze__transactions_fr_v2.sql

@@ -1,39 +0,0 @@
-{{ config (
-    materialized = 'view',
-    tags = ['bronze','core','streamline_v1','phase_1']
-) }}
-
-    WITH meta AS (
-        SELECT
-            registered_on AS _inserted_timestamp,
-            file_name,
-            CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 4), '_', 1) AS INTEGER ) AS partition_key
-        FROM
-            TABLE(
-                information_schema.external_table_files(
-                    table_name => '{{ source( "bronze_streamline", "transactions_v2") }}'
-                )
-            ) A
-    )
-SELECT
-    s.*,
-    b.file_name,
-    b._inserted_timestamp,
-    COALESCE(
-        s.value :"BLOCK_NUMBER" :: STRING,
-        s.value :"block_number" :: STRING,
-        s.metadata :request :"data" :id :: STRING,
-        PARSE_JSON(
-            s.metadata :request :"data"
-        ) :id :: STRING
-    ) :: INT AS block_number
-FROM
-    {{ source( "bronze_streamline", "transactions_v2") }}
-    s
-    JOIN meta b
-    ON b.file_name = metadata$filename
-    AND b.partition_key = s.partition_key
-WHERE
-    b.partition_key = s.partition_key
-    AND DATA :error IS NULL
-    AND DATA IS NOT NULL

models/sources.yml

@@ -14,10 +14,6 @@ sources:
     schema: |
       {{ "AURORA_DEV" if var("STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES", False) else "AURORA" }}
     tables:
-      - name: blocks
-      - name: transactions
-      - name: tx_receipts
-      - name: traces
       - name: blocks_v2
       - name: transactions_v2
       - name: receipts_by_hash