AN-2969/snapshot-upgrade (#36)

.github/workflows/dbt_run_bi_hourly.yml

@@ -0,0 +1,45 @@
+name: dbt_run_scheduled_bi_hourly
+run-name: dbt_run_scheduled_bi_hourly
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs at minute 15 and 45 past every hour (see https://crontab.guru)
+    - cron: '15,45 */1 * * *'
+    
+env:
+  DBT_PROFILES_DIR: ${{ vars.DBT_PROFILES_DIR }}
+
+  ACCOUNT: "${{ vars.ACCOUNT }}"
+  ROLE: "${{ vars.ROLE }}"
+  USER: "${{ vars.USER }}"
+  PASSWORD: "${{ secrets.PASSWORD }}"
+  REGION: "${{ vars.REGION }}"
+  DATABASE: "${{ vars.DATABASE }}"
+  WAREHOUSE: "${{ vars.WAREHOUSE }}"
+  SCHEMA: "${{ vars.SCHEMA }}"
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+      - name: Run DBT Jobs
+        run: |
+          dbt run -m tag:snapshot

.github/workflows/dbt_run_daily.yml

@@ -42,4 +42,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m tag:defillama models/deepnftvalue models/silver__dates.sql models/snapshot
+          dbt run -m tag:defillama tag:deepnftvalue tag:core

.github/workflows/dbt_run_weekly.yml

@@ -4,8 +4,8 @@ run-name: dbt_run_scheduled_weekly
 on:
   workflow_dispatch:
   schedule:
-    # Runs "at 08:00 UTC on Mondays" (see https://crontab.guru)
-    - cron: '0 8 * * 1'
+    # Runs "at 07:00 UTC on Mondays" (see https://crontab.guru)
+    - cron: '0 7 * * 1'
     
 env:
   DBT_PROFILES_DIR: ${{ vars.DBT_PROFILES_DIR }}
@@ -42,4 +42,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --exclude tag:defillama models/deepnftvalue models/snapshot
+          dbt run -m tag:tokenflow

.github/workflows/dbt_test.yml

@@ -45,6 +45,4 @@ jobs:
       - name: Run DBT Jobs
         run: |
           dbt test
-  
-
-
+  

models/deepnftvalue/bronze/bronze__dnv_collection_slugs.sql

@@ -1,5 +1,6 @@
 {{ config(
-    materialized = 'table'
+    materialized = 'table',
+    tags = ['deepnftvalue']
 ) }}
 
 SELECT

models/deepnftvalue/bronze/bronze__dnv_collections.sql

@@ -1,7 +1,8 @@
 {{ config(
     materialized = 'incremental',
     unique_key = '_id',
-    full_refresh = false
+    full_refresh = false,
+    tags = ['deepnftvalue']
 ) }}
 
 WITH slugs AS (

models/deepnftvalue/bronze/bronze__dnv_tokens.sql

@@ -1,7 +1,8 @@
 {{ config(
     materialized = 'incremental',
     unique_key = '_id',
-    full_refresh = false
+    full_refresh = false,
+    tags = ['deepnftvalue']
 ) }}
 
 WITH requests AS (

models/deepnftvalue/bronze/bronze__streamline_fr_valuations.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['deepnftvalue']
 ) }}
 
 WITH meta AS (

models/deepnftvalue/bronze/bronze__streamline_valuations.sql

@@ -1,5 +1,6 @@
 {{ config (
-    materialized = 'view'
+    materialized = 'view',
+    tags = ['deepnftvalue']
 ) }}
 
 WITH meta AS (

models/deepnftvalue/gold/deepnftvalue__fact_collections.sql

@@ -2,7 +2,8 @@
     materialized = 'view',
     persist_docs ={ "relation": true,
     "columns": true },
-    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'DEEPNFTVALUE' } } }
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'DEEPNFTVALUE' } } },
+    tags = ['deepnftvalue']
 ) }}
 
 SELECT

models/deepnftvalue/gold/deepnftvalue__fact_tokens.sql

@@ -2,7 +2,8 @@
     materialized = 'view',
     persist_docs ={ "relation": true,
     "columns": true },
-    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'DEEPNFTVALUE' } } }
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'DEEPNFTVALUE' } } },
+    tags = ['deepnftvalue']
 ) }}
 
 SELECT

models/deepnftvalue/gold/deepnftvalue__fact_valuations.sql

@@ -2,7 +2,8 @@
     materialized = 'view',
     persist_docs ={ "relation": true,
     "columns": true },
-    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'DEEPNFTVALUE' } } }
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'DEEPNFTVALUE' } } },
+    tags = ['deepnftvalue']
 ) }}
 
 SELECT

models/deepnftvalue/silver/requests/silver__dnv_token_requests.sql

@@ -1,5 +1,6 @@
 {{ config(
-    materialized = 'table'
+    materialized = 'table',
+    tags = ['deepnftvalue']
 ) }}
 
 WITH slugs AS (

models/deepnftvalue/silver/silver__dnv_collections.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = '_id'
+    unique_key = '_id',
+    tags = ['deepnftvalue']
 ) }}
 
 WITH base AS (

models/deepnftvalue/silver/silver__dnv_tokens.sql

@@ -1,6 +1,7 @@
 {{ config(
     materialized = 'incremental',
-    unique_key = '_id'
+    unique_key = '_id',
+    tags = ['deepnftvalue']
 ) }}
 
 WITH base AS (

models/deepnftvalue/silver/silver__dnv_valuations.sql

@@ -3,7 +3,8 @@
     materialized = 'incremental',
     unique_key = "_id",
     incremental_strategy = "delete+insert",
-    cluster_by = ['valuation_date::DATE']
+    cluster_by = ['valuation_date::DATE'],
+    tags = ['deepnftvalue']
 ) }}
 
 WITH base AS (

models/defillama/silver/silver__defillama_chains_tvl.sql

@@ -52,8 +52,11 @@ SELECT
     TO_TIMESTAMP(VALUE:date::INTEGER) AS timestamp,
     VALUE:totalLiquidityUSD::INTEGER AS tvl_usd,
     _inserted_timestamp,
-     {{ dbt_utils.surrogate_key(
+     {{ dbt_utils.generate_surrogate_key(
         ['chain_id', 'chain', 'timestamp']
     ) }} AS id
 FROM tvl_base,
-    LATERAL FLATTEN (input=> read:data)
+    LATERAL FLATTEN (input=> read:data)
+qualify (ROW_NUMBER () over (PARTITION BY chain_id, chain, TIMESTAMP
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/defillama/silver/silver__defillama_dex_volume.sql

@@ -76,7 +76,7 @@ SELECT
     value::INTEGER AS daily_volume,
     dex_object,
     _inserted_timestamp,
-    {{ dbt_utils.surrogate_key(
+    {{ dbt_utils.generate_surrogate_key(
         ['chain', 'protocol', 'timestamp']
     ) }} AS id
 FROM reads_output,

models/defillama/silver/silver__defillama_options_notional.sql

@@ -76,7 +76,7 @@ SELECT
     value::INTEGER AS daily_volume_notional,
     options_object,
     _inserted_timestamp,
-    {{ dbt_utils.surrogate_key(
+    {{ dbt_utils.generate_surrogate_key(
         ['chain', 'protocol', 'timestamp']
     ) }} AS id
 FROM reads_output,

models/defillama/silver/silver__defillama_options_premium.sql

@@ -76,7 +76,7 @@ SELECT
     value::INTEGER AS daily_volume_premium,
     options_object,
     _inserted_timestamp,
-    {{ dbt_utils.surrogate_key(
+    {{ dbt_utils.generate_surrogate_key(
         ['chain', 'protocol', 'timestamp']
     ) }} AS id
 FROM reads_output,

models/defillama/silver/silver__defillama_protocol_fees.sql

@@ -76,7 +76,7 @@ SELECT
     value::INTEGER AS daily_fees,
     fees_object,
     _inserted_timestamp,
-    {{ dbt_utils.surrogate_key(
+    {{ dbt_utils.generate_surrogate_key(
         ['chain', 'protocol', 'timestamp']
     ) }} AS id
 FROM reads_output,

models/defillama/silver/silver__defillama_protocol_revenue.sql

@@ -76,7 +76,7 @@ SELECT
     value::INTEGER AS daily_rev,
     rev_object,
     _inserted_timestamp,
-    {{ dbt_utils.surrogate_key(
+    {{ dbt_utils.generate_surrogate_key(
         ['chain', 'protocol', 'timestamp']
     ) }} AS id
 FROM reads_output,

models/doc_descriptions/general/__overview__.md

@@ -62,6 +62,10 @@ The documentation included here details the design of the External tables and vi
 [Snapshot Documentation](https://docs.snapshot.org/)
 
 - [snapshot__ez_snapshot](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.snapshot__ez_snapshot)
+- [snapshot__dim_spaces](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.snapshot__snapshot__dim_spaces)
+- [snapshot__dim_users](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.snapshot__dim_users)
+- [snapshot__fact_proposals](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.snapshot__fact_proposals)
+- [snapshot__fact_votes](https://flipsidecrypto.github.io/external-models/#!/model/model.external_models.snapshot__fact_votes)
 
 ## **Helpful User-Defined Functions (UDFs)**
 

models/doc_descriptions/general/snapshot/snapshot_proposals.md

@@ -0,0 +1,55 @@
+{% docs snapshot_proposal_id %}
+A unique identifier for the proposal being voted on.
+{% enddocs %}
+
+{% docs snapshot_choices %}
+An array containing the voting options on the proposal.
+{% enddocs %}
+
+{% docs snapshot_proposal_author %}
+The address of the wallet that wrote the proposal.
+{% enddocs %}
+
+{% docs snapshot_proposal_title %}
+The title or name of the proposal.
+{% enddocs %}
+
+{% docs snapshot_proposal_text %}
+The body of the proposal.
+{% enddocs %}
+
+{% docs snapshot_network %}
+The default network or blockchain used for the space.
+{% enddocs %}
+
+{% docs snapshot_network_id %}
+The default network or blockchain unique identifier used for the space.
+{% enddocs %}
+
+{% docs snapshot_delay %}
+The default number of hours of voting delay for the space. The voting delay is the time between proposal submission and the opening of voting.
+{% enddocs %}
+
+{% docs snapshot_quorum %}
+The minimum amount of voting power for a proposal to pass.
+{% enddocs %}
+
+{% docs snapshot_voting_period %}
+The default time in hours within a space for a proposal to be open for voting.
+{% enddocs %}
+
+{% docs snapshot_voting_type %}
+The type of voting system used for this space. If this column is not null, the same voting system will be used for all proposals in this space.
+{% enddocs %}
+
+{% docs snapshot_proposal_start_time %}
+The time at which voting on the proposal begins.
+{% enddocs %}
+
+{% docs snapshot_proposal_end_time %}
+The time at which voting on the proposal ends.
+{% enddocs %}
+
+{% docs snapshot_proposal_created_at %}
+The time at which the proposal was generated.
+{% enddocs %}

models/doc_descriptions/general/snapshot/snapshot_spaces.md

@@ -0,0 +1,43 @@
+{% docs snapshot_space_id %}
+A way to identify which group, or space, on Snapshot a proposal belongs to.
+{% enddocs %}
+
+{% docs snapshot_space %}
+The name of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_about %}
+The description of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_symbol %}
+The symbol of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_categories %}
+The category of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_domain %}
+The domain name of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_is_private %}
+Boolean column to signal whether the space is private or not.
+{% enddocs %}
+
+{% docs snapshot_spaces_is_verified %}
+Boolean column to signal whether the space is verified or not.
+{% enddocs %}
+
+{% docs snapshot_spaces_admins %}
+An array of the admin addresses of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_members %}
+An array of the member addresses of the space.
+{% enddocs %}
+
+{% docs snapshot_spaces_treasuries %}
+An object showcasing treasury details for the space.
+{% enddocs %}

models/doc_descriptions/general/snapshot/snapshot_users.md

@@ -0,0 +1,19 @@
+{% docs snapshot_user_created_at %}
+The time at which the user was created.
+{% enddocs %}
+
+{% docs snapshot_user_address %}
+The address of the user.
+{% enddocs %}
+
+{% docs snapshot_user_name %}
+The name of the user.
+{% enddocs %}
+
+{% docs snapshot_user_about %}
+The description of the user.
+{% enddocs %}
+
+{% docs snapshot_user_avatar %}
+The link for the avatar or profile picture of the user.
+{% enddocs %}

models/doc_descriptions/general/snapshot/snapshot_votes.md

@@ -0,0 +1,23 @@
+{% docs snapshot_vote_id %}
+A unique identifier assigned to a vote.
+{% enddocs %}
+
+{% docs snapshot_voter %}
+The wallet address of the user that voted on a proposal.
+{% enddocs %}
+
+{% docs snapshot_vote_option %}
+How the user voted on the proposal. Corresponds to the choices array.
+{% enddocs %}
+
+{% docs snapshot_voting_power %}
+The amount of voting power one holds is proportional to the amount of the token they hold.
+{% enddocs %}
+
+{% docs snapshot_vote_timestamp %}
+The time at which the vote was cast.
+{% enddocs %}
+
+{% docs snapshot_ipfs %}
+The corresponding IPFS hash.
+{% enddocs %}

models/silver/silver__dates.sql

@@ -1,5 +1,6 @@
 {{ config(
-    materialized = "table"
+    materialized = "table",
+    tags = ['core']
 ) }}
 {{ dbt_date.get_date_dimension(
     '2017-01-01',

models/snapshot/bronze/bronze__snapshot.sql

@@ -1,29 +0,0 @@
-{{ config (
-    materialized = 'view'
-) }}
-
-SELECT
-    id, 
-    proposal_id, 
-    voter, 
-    vote_option, 
-    voting_power, 
-    vote_timestamp, 
-    choices, 
-    proposal_author, 
-    proposal_title, 
-    proposal_text, 
-    space_id, 
-    network, 
-    delay, 
-    quorum, 
-    voting_period, 
-    voting_type,
-    proposal_start_time, 
-    proposal_end_time,
-    _inserted_timestamp
-FROM
-    {{ source(
-        'ethereum_silver',
-        'snapshot'
-    ) }}

models/snapshot/bronze/bronze__snapshot_proposals.sql

@@ -0,0 +1,94 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'proposal_id',
+    tags = ['snapshot']
+) }}
+--    full_refresh = false,
+-- backfill to complete in 24hrs
+
+WITH requests AS ({% for item in range(6) %}
+    (
+
+    SELECT
+        ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': (
+    SELECT
+        api_key
+    FROM
+        {{ source('crosschain_silver', 'apis_keys') }}
+    WHERE
+        api_name = 'snapshot') },{ 'query':'query { proposals(orderBy: "created", orderDirection: asc,first:1000, skip: ' || {{ item * 1000 }} || ',where:{created_gte: ' || max_time_start || '}) { id space{id voting {delay quorum period type}} ipfs author created network type title body start end state votes choices scores_state scores } }' }) AS resp, SYSDATE() AS _inserted_timestamp
+    FROM
+        (
+    SELECT
+        DATE_PART(epoch_second, max_prop_created :: TIMESTAMP) AS max_time_start
+    FROM
+        (
+
+{% if is_incremental() %}
+SELECT
+    MAX(created_at) AS max_prop_created
+FROM
+    {{ this }}
+{% else %}
+SELECT
+    1595080000 AS max_prop_created
+{% endif %}) AS max_time)) {% if not loop.last %}
+UNION ALL
+{% endif %}
+{% endfor %}),
+FINAL AS (
+    SELECT
+        VALUE :id :: STRING AS proposal_id,
+        VALUE :ipfs :: STRING AS ipfs,
+        STRTOK_TO_ARRAY(
+            VALUE :choices,
+            ';'
+        ) AS choices,
+        VALUE :author :: STRING AS proposal_author,
+        VALUE :title :: STRING AS proposal_title,
+        VALUE :body :: STRING AS proposal_text,
+        VALUE :space :id :: STRING AS space_id,
+        VALUE :space :voting :delay :: INTEGER AS delay,
+        VALUE :space :voting :quorum :: INTEGER AS quorum,
+        VALUE :space :voting :period :: INTEGER AS voting_period,
+        VALUE :space :voting :type :: STRING AS voting_type,
+        VALUE :network :: STRING AS network,
+        TO_TIMESTAMP_NTZ(
+            VALUE :created
+        ) AS created_at,
+        TO_TIMESTAMP_NTZ(
+            VALUE :start
+        ) AS proposal_start_time,
+        TO_TIMESTAMP_NTZ(
+            VALUE :end
+        ) AS proposal_end_time,
+        VALUE,
+        _inserted_timestamp
+    FROM
+        requests,
+        LATERAL FLATTEN(
+            input => resp :data :data :proposals
+        )
+    qualify(ROW_NUMBER() over (PARTITION BY proposal_id
+    ORDER BY
+        TO_TIMESTAMP_NTZ(VALUE :created) DESC)) = 1
+)
+SELECT
+    proposal_id,
+    ipfs,
+    choices,
+    proposal_author,
+    proposal_title,
+    proposal_text,
+    (delay / 3600) :: INTEGER AS delay,
+    quorum,
+    (voting_period / 3600) :: INTEGER AS voting_period,
+    LOWER(voting_type) AS voting_type,
+    space_id,
+    network,
+    created_at,
+    proposal_start_time,
+    proposal_end_time,
+    _inserted_timestamp
+FROM
+    FINAL

models/snapshot/bronze/bronze__snapshot_proposals.yml

@@ -0,0 +1,33 @@
+version: 2
+models:
+  - name: bronze__snapshot_proposals
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - PROPOSAL_ID
+    columns:
+      - name: PROPOSAL_ID
+        tests:
+          - not_null
+      - name: CHOICES
+        tests:
+          - not_null
+      - name: SPACE_ID
+        tests:
+          - not_null
+      - name: CREATED_AT
+        tests:
+          - not_null
+      - name: PROPOSAL_START_TIME
+        tests:
+          - not_null
+      - name: PROPOSAL_END_TIME
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ

models/snapshot/bronze/bronze__snapshot_spaces.sql

@@ -0,0 +1,62 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'space_id',
+    tags = ['snapshot']
+) }}
+--    full_refresh = false,
+-- after initial fill, reduce range to 6 and change orderDirection to desc
+
+WITH requests AS ({% for item in range(32) %}
+    (
+
+    SELECT
+        ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': (
+    SELECT
+        api_key
+    FROM
+        {{ source('crosschain_silver', 'apis_keys') }}
+    WHERE
+        api_name = 'snapshot') },{ 'query': 'query { spaces(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ') { id name about network symbol admins members categories domain private treasuries { address name network } verified } }' }) AS resp, SYSDATE() AS _inserted_timestamp) {% if not loop.last %}
+        UNION ALL
+        {% endif %}
+    {% endfor %}),
+    FINAL AS (
+        SELECT
+            VALUE :id :: STRING AS space_id,
+            VALUE :name :: STRING AS SPACE,
+            VALUE :about :: STRING AS about,
+            VALUE :network :: STRING AS network,
+            VALUE :symbol :: STRING AS symbol,
+            VALUE :admins :: variant AS admins,
+            VALUE :members :: variant AS members,
+            VALUE :categories :: variant AS categories,
+            VALUE :domain :: STRING AS domain,
+            VALUE :private :: BOOLEAN AS is_private,
+            VALUE :treasuries :: variant AS treasuries,
+            VALUE :verified :: BOOLEAN AS is_verified,
+            _inserted_timestamp
+        FROM
+            requests,
+            LATERAL FLATTEN(
+                input => resp :data :data :spaces
+            ) qualify(ROW_NUMBER() over(PARTITION BY space_id
+        ORDER BY
+            _inserted_timestamp DESC)) = 1
+    )
+SELECT
+    space_id,
+    SPACE,
+    about,
+    symbol,
+    network,
+    categories,
+    domain,
+    is_private,
+    is_verified,
+    admins,
+    members,
+    treasuries,
+    _inserted_timestamp
+FROM
+    FINAL

models/snapshot/bronze/bronze__snapshot_spaces.yml

@@ -0,0 +1,21 @@
+version: 2
+models:
+  - name: bronze__snapshot_spaces
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - SPACE_ID
+    columns:
+      - name: SPACE_ID
+        tests:
+          - not_null
+      - name: SPACE
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ

models/snapshot/bronze/bronze__snapshot_users.sql

@@ -0,0 +1,127 @@
+{{ config(
+    materialized = 'incremental',
+    incremental_strategy = 'delete+insert',
+    unique_key = 'address',
+    tags = ['snapshot']
+) }}
+--    full_refresh = false,
+-- backfill to complete in 1.5 weeks (estimate)
+
+WITH initial_request AS ({% for item in range(6) %}
+    (
+
+    SELECT
+        ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': (
+    SELECT
+        api_key
+    FROM
+        {{ source('crosschain_silver', 'apis_keys') }}
+    WHERE
+        api_name = 'snapshot') },{ 'query': 'query { users(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || '}) { id name about avatar ipfs created } }' }) AS resp, SYSDATE() AS _inserted_timestamp
+    FROM
+        (
+    SELECT
+        DATE_PART(epoch_second, max_start :: TIMESTAMP) AS max_time_start
+    FROM
+        (
+
+{% if is_incremental() %}
+SELECT
+    MAX(created_at) AS max_start
+FROM
+    {{ this }}
+{% else %}
+SELECT
+    1595080000 AS max_start
+{% endif %}) AS max_time)) {% if not loop.last %}
+UNION ALL
+{% endif %}
+{% endfor %}),
+users_initial AS (
+    SELECT
+        LOWER(
+            VALUE :id :: STRING
+        ) AS address,
+        VALUE :name :: STRING AS NAME,
+        VALUE :about :: STRING AS about,
+        VALUE :avatar :: STRING AS avatar,
+        VALUE :ipfs :: STRING AS ipfs,
+        TO_TIMESTAMP_NTZ(
+            VALUE :created
+        ) AS created_at,
+        _inserted_timestamp
+    FROM
+        initial_request,
+        LATERAL FLATTEN(
+            input => resp :data :data :users
+        ) qualify(ROW_NUMBER() over(PARTITION BY address
+    ORDER BY
+        TO_TIMESTAMP_NTZ(VALUE :created) DESC)) = 1
+),
+final_request AS ({% for item in range(6) %}
+    (
+SELECT
+    ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': (
+SELECT
+    api_key
+FROM
+    {{ source('crosschain_silver', 'apis_keys') }}
+WHERE
+    api_name = 'snapshot') },{ 'query': 'query { users(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || '}) { id name about avatar ipfs created } }' }) AS resp, SYSDATE() AS _inserted_timestamp
+FROM
+    (
+SELECT
+    DATE_PART(epoch_second, max_start :: TIMESTAMP) AS max_time_start
+FROM
+    (
+SELECT
+    MAX(created_at) AS max_start
+FROM
+    users_initial) AS max_time)) {% if not loop.last %}
+    UNION ALL
+    {% endif %}
+{% endfor %}),
+users_final AS (
+    SELECT
+        LOWER(
+            VALUE :id :: STRING
+        ) AS address,
+        VALUE :name :: STRING AS NAME,
+        VALUE :about :: STRING AS about,
+        VALUE :avatar :: STRING AS avatar,
+        VALUE :ipfs :: STRING AS ipfs,
+        TO_TIMESTAMP_NTZ(
+            VALUE :created
+        ) AS created_at,
+        _inserted_timestamp
+    FROM
+        final_request,
+        LATERAL FLATTEN(
+            input => resp :data :data :users
+        ) qualify(ROW_NUMBER() over(PARTITION BY address
+    ORDER BY
+        TO_TIMESTAMP_NTZ(VALUE :created) DESC)) = 1
+),
+users_merged AS (
+    SELECT
+        *
+    FROM
+        users_initial
+    UNION ALL
+    SELECT
+        *
+    FROM
+        users_final
+)
+SELECT
+    address,
+    NAME,
+    about,
+    avatar,
+    ipfs,
+    created_at,
+    _inserted_timestamp
+FROM
+    users_merged qualify(ROW_NUMBER() over(PARTITION BY address
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/snapshot/bronze/bronze__snapshot_users.yml

@@ -0,0 +1,21 @@
+version: 2
+models:
+  - name: bronze__snapshot_users
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - ADDRESS
+    columns:
+      - name: ADDRESS
+        tests:
+          - not_null
+      - name: CREATED_AT
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ

models/snapshot/bronze/bronze__snapshot_votes.sql

@@ -0,0 +1,131 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'id',
+    full_refresh = false,
+    tags = ['snapshot']
+) }}
+-- backfill from ethereum bronze_api__snapshot_votes
+
+WITH initial_votes_request AS ({% for item in range(6) %}
+    (
+
+    SELECT
+        ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': (
+    SELECT
+        api_key
+    FROM
+        {{ source('crosschain_silver', 'apis_keys') }}
+    WHERE
+        api_name = 'snapshot') },{ 'query': 'query { votes(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || ',created_lt: ' || max_time_end || '}) { id proposal{id} ipfs voter created choice vp } }' }) AS resp, SYSDATE() AS _inserted_timestamp
+    FROM
+        (
+    SELECT
+        DATE_PART(epoch_second, max_vote_start :: TIMESTAMP) AS max_time_start, DATE_PART(epoch_second, max_vote_start :: TIMESTAMP) + 86400 AS max_time_end
+    FROM
+        (
+
+{% if is_incremental() %}
+SELECT
+    MAX(vote_timestamp) AS max_vote_start
+FROM
+    {{ this }}
+{% else %}
+SELECT
+    1595080000 AS max_vote_start
+{% endif %}) AS max_time)) {% if not loop.last %}
+UNION ALL
+{% endif %}
+{% endfor %}),
+votes_initial AS (
+    SELECT
+        SPLIT(
+            VALUE :choice :: STRING,
+            ';'
+        ) AS vote_option,
+        VALUE :id :: STRING AS id,
+        VALUE :ipfs :: STRING AS ipfs,
+        VALUE :proposal :id :: STRING AS proposal_id,
+        VALUE :voter :: STRING AS voter,
+        VALUE :vp :: NUMBER AS voting_power,
+        TO_TIMESTAMP_NTZ(
+            VALUE :created
+        ) AS vote_timestamp,
+        _inserted_timestamp
+    FROM
+        initial_votes_request,
+        LATERAL FLATTEN(
+            input => resp :data :data :votes
+        ) qualify(ROW_NUMBER() over(PARTITION BY id
+    ORDER BY
+        TO_TIMESTAMP_NTZ(VALUE :created) DESC)) = 1
+),
+final_votes_request AS ({% for item in range(6) %}
+    (
+SELECT
+    ethereum.streamline.udf_api('GET', 'https://hub.snapshot.org/graphql',{ 'apiKey': (
+SELECT
+    api_key
+FROM
+    {{ source('crosschain_silver', 'apis_keys') }}
+WHERE
+    api_name = 'snapshot') },{ 'query': 'query { votes(orderBy: "created", orderDirection: asc, first: 1000, skip: ' || {{ item * 1000 }} || ', where:{created_gte: ' || max_time_start || ',created_lt: ' || max_time_end || '}) { id proposal{id} ipfs voter created choice vp } }' }) AS resp, SYSDATE() AS _inserted_timestamp
+FROM
+    (
+SELECT
+    DATE_PART(epoch_second, max_vote_start :: TIMESTAMP) AS max_time_start, DATE_PART(epoch_second, max_vote_start :: TIMESTAMP) + 86400 AS max_time_end
+FROM
+    (
+SELECT
+    MAX(vote_timestamp) AS max_vote_start
+FROM
+    votes_initial) AS max_time)) {% if not loop.last %}
+    UNION ALL
+    {% endif %}
+{% endfor %}),
+votes_final AS (
+    SELECT
+        SPLIT(
+            VALUE :choice :: STRING,
+            ';'
+        ) AS vote_option,
+        VALUE :id :: STRING AS id,
+        VALUE :ipfs :: STRING AS ipfs,
+        VALUE :proposal :id :: STRING AS proposal_id,
+        VALUE :voter :: STRING AS voter,
+        VALUE :vp :: NUMBER AS voting_power,
+        TO_TIMESTAMP_NTZ(
+            VALUE :created
+        ) AS vote_timestamp,
+        _inserted_timestamp
+    FROM
+        final_votes_request,
+        LATERAL FLATTEN(
+            input => resp :data :data :votes
+        ) qualify(ROW_NUMBER() over(PARTITION BY id
+    ORDER BY
+        TO_TIMESTAMP_NTZ(VALUE :created) DESC)) = 1
+),
+votes_merged AS (
+    SELECT
+        *
+    FROM
+        votes_initial
+    UNION ALL
+    SELECT
+        *
+    FROM
+        votes_final
+)
+SELECT
+    id,
+    ipfs,
+    proposal_id,
+    voter,
+    voting_power,
+    vote_timestamp,
+    vote_option,
+    _inserted_timestamp
+FROM
+    votes_merged qualify(ROW_NUMBER() over(PARTITION BY id
+ORDER BY
+    _inserted_timestamp DESC)) = 1

models/snapshot/bronze/bronze__snapshot_votes.yml

@@ -0,0 +1,30 @@
+version: 2
+models:
+  - name: bronze__snapshot_votes
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - ID
+    columns:
+      - name: ID
+        tests:
+          - not_null
+      - name: PROPOSAL_ID
+        tests:
+          - not_null
+      - name: VOTER
+        tests:
+          - not_null
+      - name: VOTE_TIMESTAMP
+        tests:
+          - not_null
+      - name: VOTE_OPTION
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ

models/snapshot/gold/snapshot__dim_spaces.sql

@@ -0,0 +1,24 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'SNAPSHOT',
+    'PURPOSE': 'GOVERNANCE' } } },
+    tags = ['snapshot']
+) }}
+
+SELECT
+    space_id,
+    SPACE,
+    about,
+    symbol,
+    network,
+    categories,
+    domain,
+    is_private,
+    is_verified,
+    admins,
+    members,
+    treasuries
+FROM
+    {{ ref('silver__snapshot_spaces') }}

models/snapshot/gold/snapshot__dim_spaces.yml

@@ -0,0 +1,29 @@
+version: 2
+models:
+  - name: snapshot__dim_spaces
+    description: A dimensional view that contains all spaces on Snapshot and related information.
+    columns:
+      - name: SPACE_ID
+        description: '{{ doc("snapshot_space_id") }}'
+      - name: SPACE
+        description: '{{ doc("snapshot_space") }}'
+      - name: ABOUT
+        description: '{{ doc("snapshot_spaces_about") }}'
+      - name: SYMBOL
+        description: '{{ doc("snapshot_spaces_symbol") }}'
+      - name: NETWORK
+        description: '{{ doc("snapshot_network") }}'
+      - name: CATEGORIES
+        description: '{{ doc("snapshot_spaces_categories") }}'
+      - name: DOMAIN
+        description: '{{ doc("snapshot_spaces_domain") }}'
+      - name: IS_PRIVATE
+        description: '{{ doc("snapshot_spaces_is_private") }}'
+      - name: IS_VERIFIED
+        description: '{{ doc("snapshot_spaces_is_verified") }}'
+      - name: ADMINS
+        description: '{{ doc("snapshot_spaces_admins") }}'
+      - name: MEMBERS
+        description: '{{ doc("snapshot_spaces_members") }}'
+      - name: TREASURIES
+        description: '{{ doc("snapshot_spaces_treasuries") }}'

models/snapshot/gold/snapshot__dim_users.sql

@@ -0,0 +1,18 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'SNAPSHOT',
+    'PURPOSE': 'GOVERNANCE' } } },
+    tags = ['snapshot']
+) }}
+
+SELECT
+    address,
+    NAME,
+    about,
+    avatar,
+    ipfs,
+    created_at
+FROM
+    {{ ref('silver__snapshot_users') }}

models/snapshot/gold/snapshot__dim_users.yml

@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: snapshot__dim_users
+    description: A dimensional view that contains all users on Snapshot and related information.
+    columns:
+      - name: ADDRESS
+        description: '{{ doc("snapshot_user_address") }}'
+      - name: NAME
+        description: '{{ doc("snapshot_user_name") }}'
+      - name: ABOUT
+        description: '{{ doc("snapshot_user_about") }}'
+      - name: AVATAR
+        description: '{{ doc("snapshot_user_avatar") }}'
+      - name: IPFS
+        description: '{{ doc("snapshot_ipfs") }}'
+      - name: CREATED_AT
+        description: '{{ doc("snapshot_user_created_at") }}'

models/snapshot/gold/snapshot__ez_snapshot.sql

@@ -9,18 +9,54 @@
                 'PURPOSE': 'GOVERNANCE'
             }
         }
-    }
+    },
+    tags = ['snapshot']
 ) }}
 
-SELECT 
-    id, 
-    proposal_id, 
-    LOWER(voter) AS voter, 
+WITH votes AS (
+    SELECT
+        vote_id,
+        ipfs,
+        proposal_id,
+        voter,
+        voting_power,
+        vote_timestamp,
+        vote_option,
+        _inserted_timestamp
+    FROM
+        {{ ref('silver__snapshot_votes') }}
+),
+proposals AS (
+
+    SELECT
+        proposal_id,
+        ipfs,
+        choices,
+        proposal_author,
+        proposal_title,
+        proposal_text,
+        delay,
+        quorum,
+        voting_period,
+        voting_type,
+        space_id,
+        network,
+        network_id,
+        proposal_start_time,
+        proposal_end_time,
+        _inserted_timestamp
+    FROM
+        {{ ref('silver__snapshot_proposals') }}
+)
+SELECT
+    vote_id AS id, 
+    v.proposal_id, 
+    voter, 
     vote_option, 
     voting_power, 
     vote_timestamp, 
     choices, 
-    LOWER(proposal_author) AS proposal_author, 
+    proposal_author, 
     proposal_title, 
     proposal_text, 
     space_id,
@@ -31,5 +67,7 @@ SELECT
     voting_type, 
     proposal_start_time, 
     proposal_end_time
-FROM 
-    {{ ref('silver__snapshot') }}
+FROM
+    votes v
+    INNER JOIN proposals p
+    ON v.proposal_id = p.proposal_id

models/snapshot/gold/snapshot__ez_snapshot.yml

@@ -1,77 +1,41 @@
 version: 2
 models:
   - name: snapshot__ez_snapshot
-    description: A table that contains all votes on Snapshot and information about the voting space and proposal. All data in this table is off chain.
+    description: A convenience view that contains all votes on Snapshot and information about the voting space and proposal.
     columns:
       - name: ID
-        description: A pseudo transaction hash assigned to all votes on snapshot.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_vote_id") }}'
       - name: PROPOSAL_ID
-        description: An address that identifies the proposal being voted on.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_proposal_id") }}'
       - name: VOTER
-        description: The wallet address of the user that voted on a proposal.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_voter") }}'
       - name: VOTE_OPTION
-        description: How the user voted on the proposal. Corresponds to the choices array.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_vote_option") }}'
       - name: VOTING_POWER
-        description: The amount of voting power one holds is proportional to the amount of the token they hold. 
-        tests: 
-          - dbt_expectations.expect_column_to_exist 
-      - name: VOTE_TIMESTAMP 
-        description: The time at which the vote was cast. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_voting_power") }}'
+      - name: VOTE_TIMESTAMP
+        description: '{{ doc("snapshot_vote_timestamp") }}'
       - name: CHOICES
-        description: An array containing the voting options on the proposal. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_choices") }}'
       - name: PROPOSAL_AUTHOR
-        description: The address of the wallet that wrote the proposal.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_proposal_author") }}'
       - name: PROPOSAL_TITLE
-        description: The title or name of the proposal. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_proposal_title") }}'
       - name: PROPOSAL_TEXT
-        description: The body of the proposal.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_proposal_text") }}'
       - name: SPACE_ID
-        description: A way to identify which group, or space, on Snapshot a proposal belongs to. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_space_id") }}'
       - name: NETWORK
-        description: The default network or blockchain used for the space.
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_network") }}'
       - name: DELAY
-        description: The default number of hours of voting delay for the space. The voting delay is the time between proposal submission and the opening of voting. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_delay") }}'
       - name: QUORUM
-        description: The minimum amount of voting power for a proposal to pass. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_quorum") }}'
       - name: VOTING_PERIOD
-        description: The default time in hours within a space for a proposal to be open for voting. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_voting_period") }}'
       - name: VOTING_TYPE
-        description: The type of voting system used for this space. If this column is not null, the same voting system will be used for all proposals in this space. 
-        tests:
-          - dbt_expectations.expect_column_to_exist  
+        description: '{{ doc("snapshot_voting_type") }}'
       - name: PROPOSAL_START_TIME
-        description: The time at which voting on the proposal begins. 
-        tests:
-          - dbt_expectations.expect_column_to_exist
+        description: '{{ doc("snapshot_proposal_start_time") }}'
       - name: PROPOSAL_END_TIME
-        description: The time at which voting on the proposal ends.
-        tests:
-          - dbt_expectations.expect_column_to_exist 
+        description: '{{ doc("snapshot_proposal_end_time") }}'

models/snapshot/gold/snapshot__fact_proposals.sql

@@ -0,0 +1,28 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'SNAPSHOT',
+    'PURPOSE': 'GOVERNANCE' } } },
+    tags = ['snapshot']
+) }}
+
+SELECT
+    created_at,
+    proposal_title,
+    proposal_author,
+    proposal_text,
+    choices,
+    delay,
+    quorum,
+    voting_period,
+    voting_type,
+    ipfs,
+    proposal_start_time,
+    proposal_end_time,
+    space_id,
+    network,
+    network_id,
+    proposal_id
+FROM
+    {{ ref('silver__snapshot_proposals') }}

models/snapshot/gold/snapshot__fact_proposals.yml

@@ -0,0 +1,37 @@
+version: 2
+models:
+  - name: snapshot__fact_proposals
+    description: A fact view that contains all proposals on Snapshot and related information.
+    columns:
+      - name: CREATED_AT
+        description: '{{ doc("snapshot_proposal_created_at") }}'
+      - name: PROPOSAL_TITLE
+        description: '{{ doc("snapshot_proposal_title") }}'
+      - name: PROPOSAL_AUTHOR
+        description: '{{ doc("snapshot_proposal_author") }}'
+      - name: PROPOSAL_TEXT
+        description: '{{ doc("snapshot_proposal_text") }}'
+      - name: CHOICES
+        description: '{{ doc("snapshot_choices") }}'
+      - name: DELAY
+        description: '{{ doc("snapshot_delay") }}'
+      - name: QUORUM
+        description: '{{ doc("snapshot_quorum") }}'
+      - name: VOTING_PERIOD
+        description: '{{ doc("snapshot_voting_period") }}'
+      - name: VOTING_TYPE
+        description: '{{ doc("snapshot_voting_type") }}'
+      - name: IPFS
+        description: '{{ doc("snapshot_ipfs") }}'
+      - name: PROPOSAL_START_TIME
+        description: '{{ doc("snapshot_proposal_start_time") }}'
+      - name: PROPOSAL_END_TIME
+        description: '{{ doc("snapshot_proposal_end_time") }}'
+      - name: SPACE_ID
+        description: '{{ doc("snapshot_space_id") }}'
+      - name: NETWORK
+        description: '{{ doc("snapshot_network") }}'
+      - name: NETWORK_ID
+        description: '{{ doc("snapshot_network_id") }}'
+      - name: PROPOSAL_ID
+        description: '{{ doc("snapshot_proposal_id") }}'

models/snapshot/gold/snapshot__fact_votes.sql

@@ -0,0 +1,19 @@
+{{ config(
+    materialized = 'view',
+    persist_docs ={ "relation": true,
+    "columns": true },
+    meta ={ 'database_tags':{ 'table':{ 'PROTOCOL': 'SNAPSHOT',
+    'PURPOSE': 'GOVERNANCE' } } },
+    tags = ['snapshot']
+) }}
+
+SELECT
+    vote_timestamp,
+    voter,
+    proposal_id,
+    vote_option,
+    voting_power,
+    ipfs,
+    vote_id
+FROM
+    {{ ref('silver__snapshot_votes') }}

models/snapshot/gold/snapshot__fact_votes.yml

@@ -0,0 +1,19 @@
+version: 2
+models:
+  - name: snapshot__fact_votes
+    description: A fact view that contains all votes on Snapshot and related information.
+    columns:
+      - name: VOTE_TIMESTAMP
+        description: '{{ doc("snapshot_vote_timestamp") }}'
+      - name: VOTER
+        description: '{{ doc("snapshot_voter") }}'
+      - name: PROPOSAL_ID
+        description: '{{ doc("snapshot_proposal_id") }}'
+      - name: VOTE_OPTION
+        description: '{{ doc("snapshot_vote_option") }}'
+      - name: VOTING_POWER
+        description: '{{ doc("snapshot_voting_power") }}'
+      - name: IPFS
+        description: '{{ doc("snapshot_ipfs") }}'
+      - name: VOTE_ID
+        description: '{{ doc("snapshot_vote_id") }}'

models/snapshot/silver/silver__snapshot.sql

@@ -1,40 +0,0 @@
-{{ config(
-    materialized = 'incremental',
-    unique_key = 'id',
-    incremental_strategy = 'delete+insert'
-) }}
-
-SELECT
-    id,
-    proposal_id,
-    voter,
-    vote_option,
-    voting_power,
-    vote_timestamp,
-    choices,
-    proposal_author,
-    proposal_title,
-    proposal_text,
-    space_id,
-    network,
-    delay,
-    quorum,
-    voting_period,
-    voting_type,
-    proposal_start_time,
-    proposal_end_time,
-    _inserted_timestamp
-FROM
-    {{ ref('bronze__snapshot') }}
-
-{% if is_incremental() %}
-WHERE
-    _inserted_timestamp >= (
-        SELECT
-            MAX(
-                _inserted_timestamp
-            ) :: DATE
-        FROM
-            {{ this }}
-    )
-{% endif %}

models/snapshot/silver/silver__snapshot.yml

@@ -1,61 +0,0 @@
-version: 2
-models:
-  - name: silver__snapshot
-    tests:
-      - dbt_utils.unique_combination_of_columns:
-          combination_of_columns:
-            - ID
-    columns:
-      - name: ID
-        tests:
-          - not_null  
-      - name: PROPOSAL_ID
-        tests:
-          - not_null
-      - name: VOTER
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_match_regex:
-              regex: 0[xX][0-9a-fA-F]+
-      - name: VOTE_OPTION
-        tests:
-          - not_null
-      - name: VOTING_POWER
-        tests: 
-          - not_null
-      - name: VOTE_TIMESTAMP 
-        tests: 
-          - not_null
-          - dbt_expectations.expect_column_values_to_be_in_type_list:
-              column_type_list:
-                - TIMESTAMP_LTZ
-                - TIMESTAMP_NTZ
-      - name: CHOICES
-        tests:
-          - not_null
-      - name: PROPOSAL_AUTHOR
-        tests:
-          - not_null
-          - dbt_expectations.expect_column_values_to_match_regex:
-              regex: 0[xX][0-9a-fA-F]+
-      - name: PROPOSAL_TITLE
-        tests:
-          - not_null
-      - name: PROPOSAL_TEXT
-        tests:
-          - not_null: 
-              enabled: False # Some proposals don't have a body 
-      - name: SPACE_ID
-        tests:
-          - not_null
-      - name: PROPOSAL_START_TIME
-        tests:
-          - not_null
-      - name: PROPOSAL_END_TIME
-        tests:
-          - not_null
-      - name: _INSERTED_TIMESTAMP
-        tests:
-          - dbt_expectations.expect_row_values_to_have_recent_data:
-              datepart: day
-              interval: 1

models/snapshot/silver/silver__snapshot_proposals.sql

@@ -0,0 +1,41 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'proposal_id',
+    incremental_strategy = 'delete+insert',
+    tags = ['snapshot']
+) }}
+
+SELECT
+    proposal_id,
+    ipfs,
+    choices,
+    LOWER(proposal_author) AS proposal_author,
+    proposal_title,
+    proposal_text,
+    delay,
+    quorum,
+    voting_period,
+    voting_type,
+    space_id,
+    p.network AS network_id,
+    LOWER(chain) AS network,
+    created_at,
+    proposal_start_time,
+    proposal_end_time,
+    p._inserted_timestamp
+FROM
+    {{ ref('bronze__snapshot_proposals') }}
+    p
+    LEFT JOIN {{ ref('bronze__defillama_chains') }}
+    d
+    ON d.chain_id :: STRING = p.network :: STRING
+
+{% if is_incremental() %}
+WHERE
+    p._inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/snapshot/silver/silver__snapshot_proposals.yml

@@ -0,0 +1,27 @@
+version: 2
+models:
+  - name: silver__snapshot_proposals
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - PROPOSAL_ID
+    columns:
+      - name: PROPOSAL_ID
+        tests:
+          - not_null
+      - name: CREATED_AT
+        tests:
+          - not_null
+      - name: PROPOSAL_START_TIME
+        tests:
+          - not_null
+      - name: PROPOSAL_END_TIME
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ

models/snapshot/silver/silver__snapshot_spaces.sql

@@ -0,0 +1,33 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'space_id',
+    incremental_strategy = 'delete+insert',
+    tags = ['snapshot']
+) }}
+
+SELECT
+    space_id,
+    SPACE,
+    about,
+    symbol,
+    network,
+    categories,
+    domain,
+    is_private,
+    is_verified,
+    admins,
+    members,
+    treasuries,
+    _inserted_timestamp
+FROM
+    {{ ref('bronze__snapshot_spaces') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/snapshot/silver/silver__snapshot_spaces.yml

@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: silver__snapshot_spaces
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - SPACE_ID
+    columns:
+      - name: SPACE_ID
+        tests:
+          - not_null
+      - name: SPACE
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null

models/snapshot/silver/silver__snapshot_users.sql

@@ -0,0 +1,27 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'address',
+    incremental_strategy = 'delete+insert',
+    tags = ['snapshot']
+) }}
+
+SELECT
+    address,
+    NAME,
+    about,
+    avatar,
+    ipfs,
+    created_at,
+    _inserted_timestamp
+FROM
+    {{ ref('bronze__snapshot_users') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/snapshot/silver/silver__snapshot_users.yml

@@ -0,0 +1,17 @@
+version: 2
+models:
+  - name: silver__snapshot_users
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - ADDRESS
+    columns:
+      - name: ADDRESS
+        tests:
+          - not_null
+      - name: CREATED_AT
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - not_null

models/snapshot/silver/silver__snapshot_votes.sql

@@ -0,0 +1,28 @@
+{{ config(
+    materialized = 'incremental',
+    unique_key = 'vote_id',
+    incremental_strategy = 'delete+insert',
+    tags = ['snapshot']
+) }}
+
+SELECT
+    vote_timestamp,
+    LOWER(voter) AS voter,
+    proposal_id,
+    vote_option,
+    voting_power,
+    ipfs,
+    id AS vote_id,
+    _inserted_timestamp
+FROM
+    {{ ref('bronze__snapshot_votes') }}
+
+{% if is_incremental() %}
+WHERE
+    _inserted_timestamp >= (
+        SELECT
+            MAX(_inserted_timestamp)
+        FROM
+            {{ this }}
+    )
+{% endif %}

models/snapshot/silver/silver__snapshot_votes.yml

@@ -0,0 +1,34 @@
+version: 2
+models:
+  - name: silver__snapshot_votes
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - VOTE_ID
+    columns:
+      - name: VOTE_TIMESTAMP
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_in_type_list:
+              column_type_list:
+                - TIMESTAMP_LTZ
+                - TIMESTAMP_NTZ
+      - name: VOTER
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: 0[xX][0-9a-fA-F]+
+      - name: PROPOSAL_ID
+        tests:
+          - not_null
+      - name: VOTE_OPTION
+        tests:
+          - not_null
+      - name: VOTE_ID
+        tests:
+          - not_null
+      - name: _INSERTED_TIMESTAMP
+        tests:
+          - dbt_expectations.expect_row_values_to_have_recent_data:
+              datepart: day
+              interval: 1

models/sources.yml

@@ -42,7 +42,6 @@ sources:
     schema: silver
     tables:
       - name: nft_transfers
-      - name: snapshot
   - name: bronze_streamline
     database: streamline
     schema: external

packages.yml

@@ -1,7 +1,9 @@
 packages:
   - package: calogica/dbt_expectations
-    version: 0.8.0
+    version: 0.8.2
   - package: dbt-labs/dbt_external_tables
-    version: 0.8.0
+    version: 0.8.2
   - package: dbt-labs/dbt_utils
-    version: 0.9.2
+    version: 1.0.0
+  - git: https://github.com/FlipsideCrypto/fsc-utils.git
+    revision: v1.4.1