An 2604/polygon optimize (#71)

* optimize polygon

* tests + docs

* change SO

models/gold/core__ez_matic_transfers.sql

@@ -1,5 +1,9 @@
 {{ config(
-    materialized = 'view'
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'tx_hash',
+    cluster_by = ['block_timestamp::DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 
 WITH matic_base AS (
@@ -8,19 +12,29 @@ WITH matic_base AS (
         tx_hash,
         block_number,
         block_timestamp,
+        identifier,
         from_address,
         to_address,
         matic_value,
-        identifier,
         _call_id,
-        ingested_at,
-        input
+        _inserted_timestamp
     FROM
         {{ ref('silver__traces') }}
     WHERE
         matic_value > 0
         AND tx_status = 'SUCCESS'
         AND gas_used IS NOT NULL
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        ) :: DATE - 2
+    FROM
+        {{ this }}
+)
+{% endif %}
 ),
 matic_price AS (
     SELECT
@@ -35,22 +49,51 @@ matic_price AS (
         token_address = LOWER('0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0')
     GROUP BY
         HOUR
+),
+tx_table AS (
+    SELECT
+        tx_hash,
+        from_address AS origin_from_address,
+        to_address AS origin_to_address,
+        origin_function_signature
+    FROM
+        {{ ref('silver__transactions') }}
+    WHERE
+        tx_hash IN (
+            SELECT
+                DISTINCT tx_hash
+            FROM
+                matic_base
+        )
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(
+            _inserted_timestamp
+        ) :: DATE - 2
+    FROM
+        {{ this }}
+)
+{% endif %}
 )
 SELECT
     A.tx_hash AS tx_hash,
     A.block_number AS block_number,
     A.block_timestamp AS block_timestamp,
     A.identifier AS identifier,
-    tx.from_address AS origin_from_address,
-    tx.to_address AS origin_to_address,
-    tx.origin_function_signature AS origin_function_signature,
+    origin_from_address,
+    origin_to_address,
+    origin_function_signature,
     A.from_address AS matic_from_address,
     A.to_address AS matic_to_address,
     A.matic_value AS amount,
     ROUND(
         A.matic_value * matic_price,
         2
-    ) AS amount_usd
+    ) AS amount_usd,
+    _call_id,
+    _inserted_timestamp
 FROM
     matic_base A
     LEFT JOIN matic_price
@@ -58,6 +101,5 @@ FROM
         'hour',
         block_timestamp
     ) = HOUR
-    JOIN {{ ref('silver__transactions') }}
-    tx
-    ON A.tx_hash = tx.tx_hash
+    LEFT JOIN tx_table
+    ON A.tx_hash = tx_table.tx_hash

models/gold/core__ez_matic_transfers.yml

@@ -2,26 +2,45 @@ version: 2
 models:
   - name: core__ez_matic_transfers
     description: '{{ doc("poly_ez_eth_transfers_table_doc") }}'
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - _CALL_ID
       
     columns:
       - name: BLOCK_NUMBER
         description: '{{ doc("poly_block_number") }}'
       - name: BLOCK_TIMESTAMP
         description: '{{ doc("poly_block_timestamp") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ 
       - name: TX_HASH
         description: '{{ doc("poly_transfer_tx_hash") }}'
-      - name: ETH_FROM_ADDRESS
+        tests:
+          - not_null
+      - name: MATIC_FROM_ADDRESS
         description: '{{ doc("poly_transfer_from_address") }}'
-      - name: ETH_TO_ADDRESS
+        tests:
+          - not_null
+      - name: MATIC_TO_ADDRESS
         description: '{{ doc("poly_transfer_to_address") }}'
+        tests:
+          - not_null
       - name: AMOUNT
         description: '{{ doc("poly_eth_amount") }}'
-      - name: TOKEN_PRICE
-        description: '{{ doc("poly_transfer_token_price") }}'
+        tests:
+          - not_null
       - name: AMOUNT_USD
-        description: '{{ doc("poly_eth_amount_usd") }}' 
-      - name: HAS_PRICE
-        description: '{{ doc("poly_transfer_has_price") }}' 
+        description: '{{ doc("poly_eth_amount_usd") }}'
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP::date > '2021-01-01' AND BLOCK_TIMESTAMP < CURRENT_DATE - 1
       - name: ORIGIN_FUNCTION_SIGNATURE
         description: '{{ doc("poly_origin_sig") }}'
       - name: ORIGIN_FROM_ADDRESS

models/silver/silver__logs.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = 'tx_hash',
     cluster_by = ['block_timestamp::DATE'],
-    incremental_strategy = 'delete+insert'
+    incremental_strategy = 'delete+insert',
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 
 WITH base_txs AS (

models/silver/silver__traces.sql

@@ -2,7 +2,8 @@
     materialized = 'incremental',
     unique_key = 'tx_hash',
     cluster_by = ['block_timestamp::DATE'],
-    incremental_strategy = 'delete+insert'
+    incremental_strategy = 'delete+insert',
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 
 WITH new_blocks AS (

models/silver/silver__transactions.sql

@@ -1,7 +1,8 @@
 {{ config(
     materialized = 'incremental',
     unique_key = "tx_hash",
-    cluster_by = ['block_timestamp::DATE']
+    cluster_by = ['block_timestamp::DATE'],
+    post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION"
 ) }}
 
 WITH base_table AS (