new table

models/doc_descriptions/dex - (imported from layer 1)/arb_sushi__ez_swaps.md

@@ -1,6 +1,6 @@
-{% docs eth_sushi__ez_swaps %}
+{% docs arb_sushi__ez_swaps %}
 
-This table currently contains swap events from the ```fact_event_logs``` table for SushiSwap, along with other helpful columns including an amount USD where possible. 
+This table currently contains swap events from the ```logs``` table for SushiSwap on arbitrum, along with other helpful columns including an amount USD where possible. 
 Note: A rule has been put in place to null out the amount_USD if that number is too divergent between amount_in_USD and amount_out_usd. This can happen for swaps of less liquid tokens during very high fluctuation of price.
 
 {% enddocs %}

models/sushi/sushi__dim_dex_pools.sql

@@ -3,17 +3,18 @@
 ) }}
 
     SELECT
-        pair_contract_address as pool_address,
-        token0_contract_address as token0_address,
+        lower(pair_contract_address) as pool_address,
+        lower(token0_contract_address) as token0_address,
+        token0_symbol||'-'||token1_symbol as pool_name,
         token0_symbol,
         token0_name,
-        tokens1_contract_address as token1_address,
+        lower(token1_contract_address) as token1_address,
         token1_symbol,
-        token0_name,
+        token1_name,
         token0_decimals,
         token1_decimals 
     FROM
          {{ source(
             'arbitrum_pools',
-            'SUSHI_DIM_KASHI_POOLS'
+            'SUSHI_DIM_DEX_POOLS'
         ) }} 

models/sushi/sushi__dim_kashi_pairs.sql

@@ -3,12 +3,12 @@
 ) }}
 
     SELECT
-        pair_address, 
+        lower(pair_address) as pair_address, 
         pair_symbol as pair_name,
         asset_symbol,
-        asset_address,
+        lower(asset_address) as asset_address,
         collateral_symbol,
-        collateral_address,
+        lower(collateral_address) as collateral_address,
         asset_decimal as asset_decimals,
         collateral_decimal as collateral_decimals  
     FROM

models/sushi/sushi__ez_swaps.sql

@@ -0,0 +1,277 @@
+{{ config(
+    materialized = 'incremental',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    unique_key = '_log_id',
+    cluster_by = ['block_timestamp::DATE']
+) }}
+
+WITH swap_events AS (
+
+    SELECT
+        block_number,
+        origin_function_signature,
+        origin_from_address,
+        origin_to_address,
+        block_timestamp,
+        tx_hash,
+        contract_address,
+        event_name,
+        TRY_TO_NUMBER(
+            event_inputs :amount0In :: STRING
+        ) AS amount0In,
+        TRY_TO_NUMBER(
+            event_inputs :amount1In :: STRING
+        ) AS amount1In,
+        TRY_TO_NUMBER(
+            event_inputs :amount0Out :: STRING
+        ) AS amount0Out,
+        TRY_TO_NUMBER(
+            event_inputs :amount1Out :: STRING
+        ) AS amount1Out,
+        event_inputs :sender :: STRING AS sender,
+        event_inputs :to :: STRING AS tx_to,
+        event_index,
+        _log_id,
+        _inserted_timestamp
+    FROM
+        {{ ref('silver__logs') }}
+    WHERE
+        event_name = 'Swap'
+        AND tx_status = 'SUCCESS'
+        AND contract_address IN (
+            SELECT
+                DISTINCT pool_address
+            FROM
+                {{ ref('sushi__dim_dex_pools') }}
+        )
+
+{% if is_incremental() %}
+AND _inserted_timestamp >= (
+    SELECT
+        MAX(_inserted_timestamp) :: DATE - 2
+    FROM
+        {{ this }}
+)
+{% endif %}
+),
+FINAL AS (
+    SELECT
+        block_number,
+        block_timestamp,
+        origin_function_signature,
+        origin_from_address,
+        origin_to_address,
+        tx_hash,
+        contract_address,
+        event_name,
+        CASE
+            WHEN amount0In <> 0
+            AND amount1In <> 0
+            AND token1_decimals IS NOT NULL THEN amount1In / power(
+                10,
+                token1_decimals
+            ) :: FLOAT
+            WHEN amount0In <> 0
+            AND token0_decimals IS NOT NULL THEN amount0In / power(
+                10,
+                token0_decimals
+            ) :: FLOAT
+            WHEN amount1In <> 0
+            AND token1_decimals IS NOT NULL THEN amount1In / power(
+                10,
+                token1_decimals
+            ) :: FLOAT
+            WHEN amount0In <> 0
+            AND token0_decimals IS NULL THEN amount0In
+            WHEN amount1In <> 0
+            AND token1_decimals IS NULL THEN amount1In
+        END AS amount_in,
+        CASE
+            WHEN amount0Out <> 0
+            AND token0_decimals IS NOT NULL THEN amount0Out / power(
+                10,
+                token0_decimals
+            ) :: FLOAT
+            WHEN amount1Out <> 0
+            AND token1_decimals IS NOT NULL THEN amount1Out / power(
+                10,
+                token1_decimals
+            ) :: FLOAT
+            WHEN amount0Out <> 0
+            AND token0_decimals IS NULL THEN amount0Out
+            WHEN amount1Out <> 0
+            AND token1_decimals IS NULL THEN amount1Out
+        END AS amount_out,
+        sender,
+        tx_to,
+        event_index,
+        _log_id,
+        CASE
+            WHEN amount0In <> 0
+            AND amount1In <> 0 THEN token1_address
+            WHEN amount0In <> 0 THEN token0_address
+            WHEN amount1In <> 0 THEN token1_address
+        END AS token_in,
+        CASE
+            WHEN amount0Out <> 0 THEN token0_address
+            WHEN amount1Out <> 0 THEN token1_address
+        END AS token_out,
+        CASE
+            WHEN amount0In <> 0
+            AND amount1In <> 0 THEN token1_symbol
+            WHEN amount0In <> 0 THEN token0_symbol
+            WHEN amount1In <> 0 THEN token1_symbol
+        END AS symbol_in,
+        CASE
+            WHEN amount0Out <> 0 THEN token0_symbol
+            WHEN amount1Out <> 0 THEN token1_symbol
+        END AS symbol_out,
+        CASE
+            WHEN amount0In <> 0
+            AND amount1In <> 0 THEN token1_decimals
+            WHEN amount0In <> 0 THEN token0_decimals
+            WHEN amount1In <> 0 THEN token1_decimals
+        END AS decimals_in,
+        CASE
+            WHEN amount0Out <> 0 THEN token0_decimals
+            WHEN amount1Out <> 0 THEN token1_decimals
+        END AS decimals_out,
+        token0_decimals,
+        token1_decimals,
+        token0_symbol,
+        token1_symbol,
+        pool_name,
+        _inserted_timestamp
+    FROM
+        swap_events
+        LEFT JOIN {{ ref('sushi__dim_dex_pools') }}
+        bb
+        ON swap_events.contract_address = bb.pool_address
+),
+eth_prices AS (
+    SELECT
+        token_address,
+        HOUR,
+        symbol,
+        AVG(price) AS price
+    FROM
+        {{ source(
+            'ethereum',
+            'fact_hourly_token_prices'
+        ) }}
+    WHERE
+        1 = 1
+
+{% if is_incremental() %}
+AND HOUR :: DATE IN (
+    SELECT
+        DISTINCT block_timestamp :: DATE
+    FROM
+        swap_events
+)
+{% else %}
+    AND HOUR :: DATE >= '2021-09-01'
+{% endif %}
+GROUP BY
+    token_address,
+    HOUR,
+    symbol
+),
+arbitrum_eth_crosstab AS (
+    SELECT
+        NAME,
+        symbol,
+        MAX (
+            CASE
+                WHEN platform_id = 'arbitrum-one' THEN token_address
+                ELSE ''
+            END
+        ) AS arbitrum_address,
+        MAX (
+            CASE
+                WHEN platform = 'ethereum' THEN token_address
+                ELSE ''
+            END
+        ) AS eth_address
+    FROM
+        {{ source(
+            'symbols_cross_tab',
+            'MARKET_ASSET_METADATA'
+        ) }}
+    GROUP BY
+        1,
+        2
+    HAVING
+        arbitrum_address <> ''
+        AND eth_address <> ''
+    ORDER BY
+        1,
+        2
+),
+arbitrum_prices AS (
+    SELECT
+        DISTINCT ep.token_address,
+        ep.hour,
+        ep.symbol,
+        ep.price,
+        pec.arbitrum_address AS arbitrum_address
+    FROM
+        eth_prices ep
+        LEFT JOIN arbitrum_eth_crosstab pec
+        ON ep.token_address = pec.eth_Address
+)
+SELECT
+    block_number,
+    block_timestamp,
+    tx_hash,
+    origin_function_signature,
+    origin_from_address,
+    origin_to_address,
+    contract_address,
+    'sushiswap' AS platform,
+    pool_name,
+    event_name,
+    amount_in,
+    CASE
+        WHEN decimals_in IS NOT NULL
+        AND amount_in * pIn.price <= 5 * amount_out * pOut.price
+        AND amount_out * pOut.price <= 5 * amount_in * pIn.price THEN amount_in * pIn.price
+        WHEN decimals_in IS NOT NULL and decimals_out is null then amount_in * pIn.price
+        ELSE NULL
+    END AS amount_in_usd,
+    amount_out,
+    CASE
+        WHEN decimals_out IS NOT NULL
+        AND amount_in * pIn.price <= 5 * amount_out * pOut.price
+        AND amount_out * pOut.price <= 5 * amount_in * pIn.price THEN amount_out * pOut.price
+        WHEN decimals_out IS NOT NULL and decimals_in is null then amount_out * pOut.price
+        ELSE NULL
+    END AS amount_out_usd,
+    sender,
+    tx_to,
+    event_index,
+    token_in,
+    token_out,
+    symbol_in,
+    symbol_out,
+    _log_id,
+    _inserted_timestamp
+FROM
+    FINAL wp
+    LEFT JOIN arbitrum_prices pIn
+    ON LOWER(token_in) = LOWER(
+        pIn.arbitrum_address
+    )
+    AND DATE_TRUNC(
+        'hour',
+        wp.block_timestamp
+    ) = pIn.hour
+    LEFT JOIN arbitrum_prices pOut
+    ON LOWER(token_out) = LOWER(
+        pOut.arbitrum_address
+    )
+    AND DATE_TRUNC(
+        'hour',
+        wp.block_timestamp
+    ) = pOut.hour

models/sushi/sushi__ez_swaps.yml

@@ -0,0 +1,148 @@
+version: 2
+models:
+  - name: sushi__ez_swaps
+    description: '{{ doc("arb_sushi__ez_swaps") }}'
+
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - _LOG_ID
+    columns:
+      - name: BLOCK_NUMBER
+        description: '{{ doc("arb_block_number") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: BLOCK_TIMESTAMP
+        description: '{{ doc("arb_block_timestamp") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 2 #might be normal for swaps not to happen on a day
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_NTZ
+      - name: TX_HASH
+        description: '{{ doc("arb_logs_tx_hash") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: CONTRACT_ADDRESS
+        description: '{{ doc("arb_logs_contract_address") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: EVENT_NAME
+        description: '{{ doc("arb_event_name") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: AMOUNT_IN
+        description: '{{ doc("eth_dex_swaps_amount_in") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: AMOUNT_OUT
+        description: '{{ doc("eth_dex_swaps_amount_out") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: AMOUNT_IN_USD
+        description: '{{ doc("eth_dex_swaps_amount_in_usd") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: AMOUNT_OUT_USD
+        description: '{{ doc("eth_dex_swaps_amount_out_usd") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: TOKEN_IN
+        description: '{{ doc("eth_dex_swaps_token_in") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TOKEN_OUT
+        description: '{{ doc("eth_dex_swaps_token_out") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: SYMBOL_IN
+        description: '{{ doc("eth_dex_swaps_symbol_in") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: SYMBOL_OUT
+        description: '{{ doc("eth_dex_swaps_symbol_out") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: SENDER
+        description: '{{ doc("eth_dex_swaps_sender") }}'
+        tests:
+          - not_null:
+              where: BLOCK_TIMESTAMP > '2021-08-01'
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: TX_TO
+        description: '{{ doc("eth_dex_swaps_tx_to") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: PLATFORM
+        description: '{{ doc("eth_dex_platform") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - STRING
+                - VARCHAR
+      - name: EVENT_INDEX
+        description: '{{ doc("arb_event_index") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - NUMBER
+                - FLOAT
+      - name: _LOG_ID
+        description: '{{ doc("arb_log_id_events") }}'
+        tests:
+          - not_null
+      - name: ORIGIN_FUNCTION_SIGNATURE
+        description: '{{ doc("arb_tx_origin_sig") }}'
+        tests:
+          - not_null
+      - name: ORIGIN_FROM_ADDRESS
+        description: '{{ doc("arb_origin_from") }}'
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: ORIGIN_TO_ADDRESS
+        description: '{{ doc("arb_origin_to") }}'
+        tests:
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+