From 9c07fdb96bc06959c7ca16e2cc9130a5540c9474 Mon Sep 17 00:00:00 2001 From: Desmond Hui Date: Fri, 25 Feb 2022 14:49:40 -0800 Subject: [PATCH] dbt proj files and sp setup --- analysis/.gitkeep | 0 data/.gitkeep | 0 dbt_project.yml | 44 +++++++++++++++++++++++ macros/create_sps.sql | 7 ++++ macros/create_tasks.sql | 8 +++++ macros/custom_naming_macros.sql | 17 +++++++++ macros/sp_create_prod_clone.sql | 45 ++++++++++++++++++++++++ macros/task_run_sp_create_prod_clone.sql | 7 ++++ models/sources.yml | 9 +++++ packages.yml | 3 ++ snapshots/.gitkeep | 0 tests/.gitkeep | 0 12 files changed, 140 insertions(+) create mode 100644 analysis/.gitkeep create mode 100644 data/.gitkeep create mode 100644 dbt_project.yml create mode 100644 macros/create_sps.sql create mode 100644 macros/create_tasks.sql create mode 100644 macros/custom_naming_macros.sql create mode 100644 macros/sp_create_prod_clone.sql create mode 100644 macros/task_run_sp_create_prod_clone.sql create mode 100644 models/sources.yml create mode 100644 packages.yml create mode 100644 snapshots/.gitkeep create mode 100644 tests/.gitkeep diff --git a/analysis/.gitkeep b/analysis/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 00000000..904dbbd9 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,44 @@ +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "solana_models" +version: "1.0.0" +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: "snowflake" + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +target-path: "target" # directory which will store compiled SQL files +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_modules" + +on-run-start: + - '{{create_sps()}}' + - '{{create_tasks()}}' + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ directory +# as tables. These settings can be overridden in the individual model files +# using the `{{ config(...) }}` macro. +models: + +copy_grants: true + sql_models: + solana: + materialized: incremental + +schema: bronze_solana + +vars: + "dbt_date:time_zone": America/Los_Angeles diff --git a/macros/create_sps.sql b/macros/create_sps.sql new file mode 100644 index 00000000..16e7e287 --- /dev/null +++ b/macros/create_sps.sql @@ -0,0 +1,7 @@ +{% macro create_sps() %} + CREATE SCHEMA IF NOT EXISTS _internal; + + {% if target.database == 'SOLANA' %} + {{ sp_create_prod_clone('_internal') }}; + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/create_tasks.sql b/macros/create_tasks.sql new file mode 100644 index 00000000..1f039d48 --- /dev/null +++ b/macros/create_tasks.sql @@ -0,0 +1,8 @@ +{% macro create_tasks() %} + CREATE SCHEMA IF NOT EXISTS _internal; + + {% if target.database == 'SOLANA' %} + {{ task_run_sp_create_prod_clone('_internal') }}; + {% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/custom_naming_macros.sql b/macros/custom_naming_macros.sql new file mode 100644 index 00000000..b8a30718 --- /dev/null +++ b/macros/custom_naming_macros.sql @@ -0,0 +1,17 @@ +{% macro generate_schema_name( + custom_schema_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name [0] | trim }} +{%- endmacro %} + +{% macro generate_alias_name( + custom_alias_name = none, + node = none + ) -%} + {% set node_name = node.name %} + {% set split_name = node_name.split('__') %} + {{ split_name [1] | trim }} +{%- endmacro %} diff --git a/macros/sp_create_prod_clone.sql b/macros/sp_create_prod_clone.sql new file mode 100644 index 00000000..fe1d2749 --- /dev/null +++ b/macros/sp_create_prod_clone.sql @@ -0,0 +1,45 @@ +{% macro sp_create_prod_clone(target_schema) -%} + +create or replace procedure {{ target_schema }}.create_prod_clone(source_db_name string, destination_db_name string, role_name string) +returns boolean +language javascript +execute as caller +as +$$ + snowflake.execute({sqlText: `BEGIN TRANSACTION;`}); + try { + snowflake.execute({sqlText: `DROP DATABASE IF EXISTS ${DESTINATION_DB_NAME}`}); + snowflake.execute({sqlText: `CREATE DATABASE ${DESTINATION_DB_NAME} CLONE ${SOURCE_DB_NAME}`}); + + var existing_schemas = snowflake.execute({sqlText: `SELECT table_schema + FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.TABLE_PRIVILEGES + WHERE grantor IS NOT NULL + GROUP BY 1;`}); + + while (existing_schemas.next()) { + var schema = existing_schemas.getColumnValue(1) + snowflake.execute({sqlText: `GRANT OWNERSHIP ON SCHEMA ${DESTINATION_DB_NAME}.${schema} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + } + + var existing_tables = snowflake.execute({sqlText: `SELECT table_schema, table_name + FROM ${DESTINATION_DB_NAME}.INFORMATION_SCHEMA.TABLE_PRIVILEGES + WHERE grantor IS NOT NULL + GROUP BY 1,2;`}); + + while (existing_tables.next()) { + var schema = existing_tables.getColumnValue(1) + var table_name = existing_tables.getColumnValue(2) + snowflake.execute({sqlText: `GRANT OWNERSHIP ON TABLE ${DESTINATION_DB_NAME}.${schema}.${table_name} TO ROLE ${ROLE_NAME} COPY CURRENT GRANTS;`}); + } + + snowflake.execute({sqlText: `GRANT OWNERSHIP ON DATABASE ${DESTINATION_DB_NAME} TO ROLE ${ROLE_NAME};`}) + snowflake.execute({sqlText: `COMMIT;`}); + } catch (err) { + snowflake.execute({sqlText: `ROLLBACK;`}); + throw(err); + } + + return true +$$ + +{%- endmacro %} \ No newline at end of file diff --git a/macros/task_run_sp_create_prod_clone.sql b/macros/task_run_sp_create_prod_clone.sql new file mode 100644 index 00000000..436d4800 --- /dev/null +++ b/macros/task_run_sp_create_prod_clone.sql @@ -0,0 +1,7 @@ +{% macro task_run_sp_create_prod_clone(target_schema) -%} + create or replace task {{target_schema}}.run_sp_create_prod_clone + warehouse = dbt_cloud + schedule = 'USING CRON 15 6 * * * UTC' + as + call {{ target_schema }}.create_prod_clone('solana', 'solana_dev', 'internal_dev') +{%- endmacro %} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml new file mode 100644 index 00000000..0fc36592 --- /dev/null +++ b/models/sources.yml @@ -0,0 +1,9 @@ +version: 2 + +sources: + - name: prod + database: chainwalkers + schema: prod + tables: + - name: solana_blocks + - name: solana_txs diff --git a/packages.yml b/packages.yml new file mode 100644 index 00000000..ee3a1287 --- /dev/null +++ b/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: calogica/dbt_expectations + version: [">=0.4.0", "<0.5.0"] \ No newline at end of file diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 00000000..e69de29b