inital commit

.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

README.md

@@ -1 +1,30 @@
-# algorand-models
+## Profile Set Up
+
+#### Use the following within profiles.yml 
+----
+
+```yml
+osmosis:
+  target: dev
+  outputs:
+    dev:
+      type: snowflake
+      account: <ACCOUNT>
+      role: <ROLE>
+      user: <USERNAME>
+      password: <PASSWORD>
+      region: <REGION>
+      database: ALGORAND_DEV
+      warehouse: <WAREHOUSE>
+      schema: silver
+      threads: 4
+      client_session_keep_alive: False
+      query_tag: <TAG>
+```
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

dbt_project.yml

@@ -0,0 +1,44 @@
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: "algorand_models"
+version: "1.0.0"
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: "algorand"
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+seed-paths: ["data"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target" # directory which will store compiled SQL files
+clean-targets: # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_modules"
+  - "dbt_packages"
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/ directory
+# as tables. These settings can be overridden in the individual model files
+# using the `{{ config(...) }}` macro.
+models:
+  +copy_grants: true
+  +persist_docs:
+    relation: true
+    columns: true
+
+vars:
+  "dbt_date:time_zone": GMT
+
+on-run-start:
+  - '{{create_sps()}}'
+  - '{{create_udfs()}}'

macros/create_sps.sql

@@ -0,0 +1,6 @@
+{% macro create_sps() %}
+    {% if target.database == 'ALGORAND' %}
+        CREATE schema IF NOT EXISTS _internal;
+{{ sp_create_prod_clone('_internal') }};
+    {% endif %}
+{% endmacro %}

macros/create_udfs.sql

@@ -0,0 +1,14 @@
+{% macro create_udfs() %}
+    {% set sql %}
+    {{ udf_bulk_get_asset_metadata() }};
+    {% endset %}
+    {% do run_query(sql) %}
+    {% set sql %}
+    {{ udf_bulk_get_balances() }};
+    {% endset %}
+    {% do run_query(sql) %}
+    {% set sql %}
+    {{ udf_bulk_get_validator_metadata() }};
+    {% endset %}
+    {% do run_query(sql) %}
+{% endmacro %}

macros/custom_naming_macros.sql

@@ -0,0 +1,17 @@
+{% macro generate_schema_name(
+        custom_schema_name = none,
+        node = none
+    ) -%}
+    {% set node_name = node.name %}
+    {% set split_name = node_name.split('__') %}
+    {{ split_name [0] | trim }}
+{%- endmacro %}
+
+{% macro generate_alias_name(
+        custom_alias_name = none,
+        node = none
+    ) -%}
+    {% set node_name = node.name %}
+    {% set split_name = node_name.split('__') %}
+    {{ split_name [1] | trim }}
+{%- endmacro %}

macros/run_sp_create_prod_clone.sql

@@ -0,0 +1,10 @@
+{% macro run_sp_create_prod_clone() %}
+    {% set clone_query %}
+    call osmosis._internal.create_prod_clone(
+        'algorand',
+        'algorand_dev',
+        'internal_dev'
+    );
+{% endset %}
+    {% do run_query(clone_query) %}
+{% endmacro %}

macros/sp_create_prod_clone.sql

@@ -0,0 +1,66 @@
+{% macro sp_create_prod_clone(target_schema) -%}
+
+create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string)
+returns boolean 
+language javascript
+execute as caller
+as
+$$
+    snowflake.execute({sqlText: `BEGIN TRANSACTION;`});
+    try {
+        snowflake.execute({sqlText: `DROP DATABASE IF EXISTS ${DESTINATION_DB_NAME}`});
+        snowflake.execute({sqlText: `CREATE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`});
+        snowflake.execute({sqlText: `DROP SCHEMA ${DESTINATION_DB_NAME}._INTERNAL`}); /* this only needs to be in prod */
+
+        var existing_schemas = snowflake.execute({sqlText: `SELECT table_schema
+            FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.TABLE_PRIVILEGES
+            WHERE grantor IS NOT NULL
+            GROUP BY 1;`});
+
+        while (existing_schemas.next()) {
+            var schema = existing_schemas.getColumnValue(1)
+            snowflake.execute({sqlText: `GRANT OWNERSHIP ON SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
+        }
+
+        var existing_tables = snowflake.execute({sqlText: `SELECT table_schema, table_name
+            FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.TABLE_PRIVILEGES
+            WHERE grantor IS NOT NULL
+            GROUP BY 1,2;`});
+
+        while (existing_tables.next()) {
+            var schema = existing_tables.getColumnValue(1)
+            var table_name = existing_tables.getColumnValue(2)
+            snowflake.execute({sqlText: `GRANT OWNERSHIP ON TABLE ${DESTINATION_DB_NAME}.${schema}.${table_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`});
+        }
+
+        var existing_functions = snowflake.execute({sqlText: `SELECT function_schema, function_name, argument_signature
+            FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.FUNCTIONS;`});
+
+        while (existing_functions.next()) {
+            var schema = existing_functions.getColumnValue(1)
+            var function_name = existing_functions.getColumnValue(2)
+            var argument_signature = existing_functions.getColumnValue(3)
+            snowflake.execute({sqlText: `GRANT OWNERSHIP ON FUNCTION ${DESTINATION_DB_NAME}.${schema}.${function_name}${argument_signature} to role ${ROLE_NAME};`});
+        }
+
+        var existing_procedures = snowflake.execute({sqlText: `SELECT procedure_schema, procedure_name, argument_signature
+            FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.PROCEDURES;`});
+
+        while (existing_procedures.next()) {
+            var schema = existing_procedures.getColumnValue(1)
+            var procedure_name = existing_procedures.getColumnValue(2)
+            var argument_signature = existing_procedures.getColumnValue(3)
+            snowflake.execute({sqlText: `GRANT OWNERSHIP ON PROCEDURE ${DESTINATION_DB_NAME}.${schema}.${procedure_name}${argument_signature} to role ${ROLE_NAME};`});
+        }
+
+        snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`})
+        snowflake.execute({sqlText: `COMMIT;`});
+    } catch (err) {
+        snowflake.execute({sqlText: `ROLLBACK;`});
+        throw(err);
+    }
+    
+    return true
+$$
+
+{%- endmacro %}

models/bronze/bonze__block.sql

@@ -0,0 +1,20 @@
+{{ config (
+  materialized = 'view'
+) }}
+
+SELECT
+  ROUND,
+  realtime,
+  rewardslevel,
+  header,
+  __HEVO__LOADED_AT,
+  DATEADD(
+    ms,
+    __HEVO__LOADED_AT,
+    '1970-01-01'
+  ) AS _inserted_timestamp
+FROM
+  {{ source(
+    'algorand',
+    'BLOCK_HEADER'
+  ) }}

models/sources.yml

@@ -0,0 +1,244 @@
+version: 2
+
+sources:
+  - name: shared
+    schema: silver
+    tables:
+      - name: prices
+      - name: prices_v2
+      - name: cmc_assets
+      - name: market_asset_metadata
+      - name: coingecko_prices
+      - name: hours
+      - name: udm_address_labels
+      - name: udm_address_labels_new
+      - name: udm_address_tags
+      - name: udm_balances
+      - name: udm_decimal_adjustments
+      - name: udm_velocity
+      - name: uniswapv2factory_event_paircreated
+      - name: flow_balances
+      - name: near_balances
+      - name: algorand_balances
+      - name: terra_balances
+  - name: ethereum
+    schema: silver
+    tables:
+      - name: ethereum_balances
+      - name: ethereum_contracts
+      - name: ethereum_contracts_backfill
+      - name: daily_ethereum_token_balances
+      - name: ethereum_address_labels
+      - name: ethereum_address_labels_updates
+      - name: ethereum_contract_decimal_adjustments
+      - name: ethereum_decoded_log_methods
+      - name: ethereum_events
+      - name: ethereum_events_emitted
+      - name: ethereum_proxy_contract_registry
+      - name: ethereum_reads
+      - name: ethereum_transactions
+      - name: ethereum_token_balances
+      - name: ethereum_token_contracts
+      - name: sha256_function_signatures
+      - name: nft_metadata
+  - name: flow
+    schema: redshift
+    tables:
+      - name: daily_flow_staked_balances
+      - name: udm_daily_balances_flow
+      - name: udm_events_flow
+
+  - name: uniswapv3_eth
+    schema: silver
+    tables:
+      - name: rinkeby_uniswapv3_liquidity_actions
+      - name: rinkeby_uniswapv3_pools
+      - name: rinkeby_uniswapv3_pool_stats
+      - name: rinkeby_uniswapv3_positions
+      - name: rinkeby_uniswapv3_swaps
+      - name: uniswapv3_liquidity_actions
+      - name: uniswapv3_pools
+      - name: uniswapv3_pool_stats
+      - name: uniswapv3_positions
+      - name: uniswapv3_swaps
+  - name: gold
+    schema: gold
+    tables:
+      - name: flow_delegator_addresses
+      - name: terra_vesting_schedule
+  - name: thorchain_midgard
+    schema: BRONZE_MIDGARD_2_6_9_20220405
+    tables:
+      - name: midgard_active_vault_events
+      - name: midgard_add_events
+      - name: midgard_asgard_fund_yggdrasil_events
+      - name: midgard_block_log
+      - name: midgard_block_pool_depths
+      - name: midgard_bond_events
+      - name: bond_events_pk_count
+      - name: midgard_constants
+      - name: midgard_errata_events
+      - name: midgard_fee_events
+      - name: fee_events_pk_count
+      - name: midgard_gas_events
+      - name: midgard_inactive_vault_events
+      - name: midgard_message_events
+      - name: midgard_new_node_events
+      - name: midgard_outbound_events
+      - name: midgard_pending_liquidity_events
+      - name: midgard_pool_balance_change_events
+      - name: midgard_pool_events
+      - name: midgard_refund_events
+      - name: midgard_reserve_events
+      - name: midgard_rewards_event_entries
+      - name: midgard_rewards_events
+      - name: midgard_set_ip_address_events
+      - name: midgard_set_mimir_events
+      - name: midgard_set_node_keys_events
+      - name: midgard_set_version_events
+      - name: midgard_slash_amounts
+      - name: midgard_stake_events
+      - name: midgard_swap_events
+      - name: midgard_switch_events
+      - name: switch_events_pk_count
+      - name: midgard_thorname_change_events
+      - name: midgard_transfer_events
+      - name: midgard_unstake_events
+      - name: midgard_update_node_account_status_events
+      - name: midgard_validator_request_leave_events
+  - name: terra
+    schema: silver
+    tables:
+      - name: udm_daily_balances_terra
+      - name: terra_validator_voting_power
+      - name: udm_custom_fields_terra_tax_rate # TODO: drop table as it is not being referenced in dbt anymore.
+  - name: bronze
+    schema: bronze
+    tables:
+      - name: prod_matic_sink_510901820
+      - name: prod_terra_sink_645110886
+      - name: prod_terra_api
+      - name: algorand_api
+      - name: prod_address_label_sink_291098491
+      - name: prod_ethereum_sink_407559501
+      - name: JIM_TEST_REST_PROXY_1507016047
+      - name: prod_nft_metadata_uploads_1828572827
+      - name: prod_data_science_uploads_1748940988
+  - name: crosschain
+    schema: silver_crosschain
+    tables:
+      - name: address_labels
+  - name: near
+    schema: redshift
+    tables:
+      - name: near_validators
+      - name: near_daily_reward_flow
+      - name: near_delegator_counts
+      - name: udm_events_near
+  - name: near_silver
+    schema: silver_near
+    tables:
+      - name: daily_balances
+  - name: algorand
+    schema: BRONZE_ALGORAND_2_8_0__HEVO_PARALLEL
+    tables:
+      - name: ACCOUNT
+      - name: ACCOUNT_APP
+      - name: ACCOUNT_ASSET
+      - name: APP
+      - name: ASSET
+      - name: BLOCK_HEADER
+      - name: METASTATE
+      - name: TXN
+      - name: TXN_PARTICIPATION
+  - name: prod
+    database: chainwalkers
+    schema: prod
+    tables:
+      - name: solana_blocks
+      - name: solana_txs
+      - name: ethereum_blocks
+      - name: ethereum_txs
+  - name: solana
+    database: solana
+    schema: core
+    tables:
+      - name: dim_labels
+      - name: dim_nft_metadata
+      - name: fact_blocks
+      - name: fact_events
+      - name: fact_gauges_votes
+      - name: fact_gov_actions
+      - name: fact_nft_mints
+      - name: fact_nft_sales
+      - name: fact_proposal_votes
+      - name: fact_staking_lp_actions
+      - name: fact_swaps
+      - name: fact_transactions
+      - name: fact_transfers
+      - name: fact_votes_agg_block
+  - name: ethereum_db
+    database: ethereum
+    schema: core
+    tables:
+      - name: dim_contracts
+      - name: dim_dates
+      - name: dim_labels
+      - name: fact_blocks
+      - name: fact_event_logs
+      - name: fact_hourly_token_prices
+      - name: fact_token_transfers
+      - name: fact_transactions
+      - name: ez_token_transfers
+      - name: dim_dex_liquidity_pools
+      - name: ez_dex_swaps
+      - name: ez_nft_sales
+      - name: ez_nft_mints
+      - name: ez_nft_transfers
+      - name: fact_traces
+  - name: ethereum_db_sushi
+    database: ethereum
+    schema: sushi
+    tables:
+      - name: ez_swaps
+      - name: dim_distributor_reward_schedule
+  - name: ethereum_db_external
+    database: ethereum
+    schema: silver
+    tables:
+      - name: nft_metadata_api
+  - name: algorand_db_external
+    schema: bronze
+    loader: S3
+    tables:
+      - name: algorand_indexer_tx
+        external:
+          location: "@{{ target.database }}.bronze.ANALYTICS_EXTERNAL_TABLES/ALGORAND_INDEXER_{{ target.name }}/TX"
+          file_format: "( type = json, strip_outer_array = TRUE )"
+          auto_refresh: false
+          partitions:
+            - name: _partition_by_date
+              data_type: date
+              expression: to_date(split_part(metadata$filename, '/', 3) || split_part(metadata$filename, '/', 4) || split_part(metadata$filename, '/', 5), 'YYYYMMDD')
+        columns:
+          - name: account_id
+            data_type: string
+            description: "account_id used in request"
+            expression: value:account_id::string
+          - name: tx_id
+            data_type: string
+            description: "txid used in request"
+            expression: value:tx_id::string
+          - name: metadata
+            data_type: variant
+            description: "raw indexer response"
+            expression: value:metadata::variant
+          - name: data
+            data_type: variant
+            description: "raw indexer response"
+            expression: value:data::variant
+  - name: Harmony_db
+    database: HARMONY
+    schema: PROD
+    tables:
+      - name: EZ_SUSHI_SWAPS

packages.yml

@@ -0,0 +1,5 @@
+packages:
+  - package: calogica/dbt_expectations
+    version: [">=0.4.0", "<0.9.0"]
+  - package: dbt-labs/dbt_external_tables
+    version: 0.8.0