From 40b2f5506d33b362bcb72894ad08f923c229eee4 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Sat, 4 Jan 2025 22:13:26 -0800 Subject: [PATCH] Revert This reverts commit a70e0905490df0ea1d849630377da320deacd4ee. --- .user.yml | 1 - cspell.config.yaml | 8 -- dbt_project.yml | 19 +++-- macros/admin/create_api_integration.sql | 20 ----- macros/admin/create_snowflake_user.sql | 8 -- ...amline_dev_api_integration_permissions.sql | 23 ------ .../admin/run_streamline_dev_permissions.sql | 73 ----------------- ...mline_prod_api_integration_permissions.sql | 23 ------ .../admin/run_streamline_prod_permissions.sql | 82 ------------------- profiles.yml | 6 +- 10 files changed, 13 insertions(+), 250 deletions(-) delete mode 100644 .user.yml delete mode 100644 cspell.config.yaml delete mode 100644 macros/admin/create_api_integration.sql delete mode 100644 macros/admin/create_snowflake_user.sql delete mode 100644 macros/admin/run_streamline_dev_api_integration_permissions.sql delete mode 100644 macros/admin/run_streamline_dev_permissions.sql delete mode 100644 macros/admin/run_streamline_prod_api_integration_permissions.sql delete mode 100644 macros/admin/run_streamline_prod_permissions.sql diff --git a/.user.yml b/.user.yml deleted file mode 100644 index d452b8a..0000000 --- a/.user.yml +++ /dev/null @@ -1 +0,0 @@ -id: 195a8066-defe-420d-8aca-6f39739693b1 diff --git a/cspell.config.yaml b/cspell.config.yaml deleted file mode 100644 index 0f50605..0000000 --- a/cspell.config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -version: "0.2" -ignorePaths: [] -dictionaryDefinitions: [] -dictionaries: [] -words: - - berachain -ignoreWords: [] -import: [] diff --git a/dbt_project.yml b/dbt_project.yml index 994a993..d206e49 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -51,7 +51,7 @@ query-comment: models: +copy_grants: true +on_schema_change: "append_new_columns" - + # In this example config, we tell dbt to build all models in the example/ directory # as tables. These settings can be overridden in the individual model files # using the `{{ config(...) }}` macro. @@ -75,24 +75,25 @@ vars: EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] if var("config")[target.name] else var("config")["dev"]["EXTERNAL_FUNCTION_URI"] }}' API_AWS_ROLE_ARN: '{{ var("config")[target.name]["API_AWS_ROLE_ARN"] if var("config")[target.name] else var("config")["dev"]["API_AWS_ROLE_ARN"] }}' ROLES: | - ["RESEARCHERS"] + ["INTERNAL_DEV"] config: # The keys correspond to dbt profiles and are case sensitive dev: - API_INTEGRATION: H_BERACHAIN_DEV_V2 - EXTERNAL_FUNCTION_URI: jhe0icz5z1.execute-api.us-west-2.amazonaws.com/stg/ - API_AWS_ROLE_ARN: arn:aws:iam::528775386234:role/berachain-api-stg-rolesnowflakeudfsAF733095-vBCdPygbG0MM + API_INTEGRATION: aws_berachain_api_stg_v2 + EXTERNAL_FUNCTION_URI: b387zu3um2.execute-api.us-east-1.amazonaws.com/stg/ + API_AWS_ROLE_ARN: arn:aws:iam::704693948482:role/berachain-api-stg-rolesnowflakeudfsAF733095-Y5WdOPq9Ipf3 ROLES: - - RESEARCHERS - - READ_ONLY + - AWS_LAMBDA_BERACHAIN_API + - INTERNAL_DEV prod: API_INTEGRATION: aws_berachain_api_prod_v2 EXTERNAL_FUNCTION_URI: ayji5axa1e.execute-api.us-east-1.amazonaws.com/prod/ API_AWS_ROLE_ARN: arn:aws:lambda:us-east-1:924682671219:function:berachain-api-prod-APIHandler-X72SH1yiwuQ4 ROLES: - - RESEARCHERS - - READ_ONLY + - AWS_LAMBDA_BERACHAIN_API + - INTERNAL_DEV + - DBT_CLOUD_BERACHAIN #### STREAMLINE 2.0 END #### \ No newline at end of file diff --git a/macros/admin/create_api_integration.sql b/macros/admin/create_api_integration.sql deleted file mode 100644 index 9b67a26..0000000 --- a/macros/admin/create_api_integration.sql +++ /dev/null @@ -1,20 +0,0 @@ -{% macro create_api_integration(project_name, snowflake_role_arn, endpoint_urls) %} - - {% set allowed_prefixes = [] %} - {% for url in endpoint_urls %} - {% do allowed_prefixes.append("'" ~ url ~ "'") %} - {% endfor %} - {% set allowed_prefixes = allowed_prefixes|join(", ") %} - {% set sql %} - CREATE OR REPLACE API INTEGRATION {{ project_name ~ "_v2" }} - API_PROVIDER = aws_api_gateway - API_AWS_ROLE_ARN = '{{ snowflake_role_arn }}' - API_ALLOWED_PREFIXES = ({{ allowed_prefixes }}) - ENABLED=true - {% endset %} - - {% do log(sql, info=true)%} - - {% do run_query(sql) %} - {% do log("API Integration " ~ integration_name ~ " successfully created", true) %} -{% endmacro %} \ No newline at end of file diff --git a/macros/admin/create_snowflake_user.sql b/macros/admin/create_snowflake_user.sql deleted file mode 100644 index 8ca77d0..0000000 --- a/macros/admin/create_snowflake_user.sql +++ /dev/null @@ -1,8 +0,0 @@ -{% macro create_snowflake_user(user_name, user_password) %} - {% set sql %} - CREATE USER IF NOT EXISTS {{ user_name }} PASSWORD = '{{ user_password }}' MUST_CHANGE_PASSWORD = FALSE; - {% endset %} - - {% do run_query(sql) %} - {% do log("User '" ~ user_name ~ "' created successfully", info=true) %} -{% endmacro %} diff --git a/macros/admin/run_streamline_dev_api_integration_permissions.sql b/macros/admin/run_streamline_dev_api_integration_permissions.sql deleted file mode 100644 index 211f7a8..0000000 --- a/macros/admin/run_streamline_dev_api_integration_permissions.sql +++ /dev/null @@ -1,23 +0,0 @@ -{% macro run_streamline_dev_api_integration_permissions(project_name) %} - --This will run after api integration is created - - {% set sql %} - - use role accountadmin; - use warehouse dbt; - - --Grant usage on api integration - grant usage on integration aws_{{ project_name }}_api_stg_v2 to role dbt_cloud_{{ project_name }}; - grant usage on integration aws_{{ project_name }}_api_stg_v2 to role aws_lambda_{{ project_name }}_api; - grant usage on integration aws_{{ project_name }}_api_stg_v2 to role internal_dev; - - --Grant usage on stage - grant usage on stage streamline.bronze.{{ project_name }}_serverless_stg to role internal_dev; - - {% endset %} - - {% do log(sql, info=true)%} - - {% do run_query(sql) %} - {% do log("Streamline DEV API Integration Permissions for " ~ project_name ~ " successfully ran", true) %} -{% endmacro %} \ No newline at end of file diff --git a/macros/admin/run_streamline_dev_permissions.sql b/macros/admin/run_streamline_dev_permissions.sql deleted file mode 100644 index 6366c99..0000000 --- a/macros/admin/run_streamline_dev_permissions.sql +++ /dev/null @@ -1,73 +0,0 @@ -{% macro run_streamline_dev_permissions(project_name) %} - --This will run before api integration is created - --aws_lambda_{{ project_name }}_api and dbt_cloud_{{ project_name }} users should both be created prior to running this macro - - {% set sql %} - - use role accountadmin; - use warehouse dbt; - create role IF NOT EXISTS dbt_cloud_{{ project_name }}; - create role IF NOT EXISTS aws_lambda_{{ project_name }}_api; - - grant role aws_lambda_{{ project_name }}_api to user aws_lambda_{{ project_name }}_api; - grant role internal_dev to role aws_lambda_{{ project_name }}_api; - grant usage on warehouse dbt_cloud to role aws_lambda_{{ project_name }}_api; - - --Create dbt_cloud_{{ project_name }} role and grant permissions - grant role dbt_cloud_{{ project_name }} to role ACCOUNTADMIN; - grant role internal_dev to role dbt_cloud_{{ project_name }}; - grant create integration on account to role dbt_cloud_{{ project_name }}; - grant usage on warehouse dbt_cloud to role dbt_cloud_{{ project_name }}; - - --Grants necessary for DB cloning - grant create database on account to role dbt_cloud_{{ project_name }}; - grant manage grants on account to role dbt_cloud_{{ project_name }}; - grant execute task on account to role dbt_cloud_{{ project_name }}; - - - --Create Database and Grant Perms - use role internal_dev; - create database if not exists {{ project_name }}_dev; - grant usage on database {{ project_name }}_dev to role aws_lambda_{{ project_name }}_api; - grant usage on database {{ project_name }}_dev to role dbt_cloud_{{ project_name }}; - grant usage on database {{ project_name }}_dev to role internal_dev; - grant usage on database {{ project_name }}_dev to role datascience; - - --Create Streamline Schema and Grant Permissions - create schema if not exists {{ project_name }}_dev.bronze; - create schema if not exists {{ project_name }}_dev._internal; - create schema if not exists {{ project_name }}_dev.silver; - create schema if not exists {{ project_name }}_dev.streamline; - create schema if not exists streamline.{{ project_name }}_dev; - - - use role accountadmin; - - --Grant schema usage first - grant usage on schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - - --Then grant object permissions - grant select on all views in schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - grant select on all tables in schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - grant usage on all functions in schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - grant select on future views in schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - grant select on future tables in schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - grant usage on future functions in schema {{ project_name }}_dev.streamline to role aws_lambda_{{ project_name }}_api; - - --Permissions for Streamline external tables - grant usage on schema streamline.{{ project_name }}_dev to role aws_lambda_{{ project_name }}_api; - grant select on all tables in schema streamline.{{ project_name }}_dev to role aws_lambda_{{ project_name }}_api; - grant select on all views in schema streamline.{{ project_name }}_dev to role aws_lambda_{{ project_name }}_api; - grant select on future tables in schema streamline.{{ project_name }}_dev to role aws_lambda_{{ project_name }}_api; - grant select on future views in schema streamline.{{ project_name }}_dev to role aws_lambda_{{ project_name }}_api; - - grant usage on schema streamline.{{ project_name }}_dev to role streamline_snowflake; - grant create external table on schema streamline.{{ project_name }}_dev to role streamline_snowflake; - grant create stage on schema streamline.{{ project_name }}_dev to role streamline_snowflake; - - {% endset %} - - {% do log(sql, info=true)%} - {% do run_query(sql) %} - {% do log("Streamline DEV Permissions for " ~ project_name ~ " successfully ran", true) %} -{% endmacro %} \ No newline at end of file diff --git a/macros/admin/run_streamline_prod_api_integration_permissions.sql b/macros/admin/run_streamline_prod_api_integration_permissions.sql deleted file mode 100644 index a0ce1b5..0000000 --- a/macros/admin/run_streamline_prod_api_integration_permissions.sql +++ /dev/null @@ -1,23 +0,0 @@ -{% macro run_streamline_prod_api_integration_permissions(project_name) %} - --This will run after api integration is created - - {% set sql %} - - use role accountadmin; - use warehouse dbt; - - --Grant usage on api integration - grant usage on integration aws_{{ project_name }}_api_prod_v2 to role dbt_cloud_{{ project_name }}; - grant usage on integration aws_{{ project_name }}_api_prod_v2 to role aws_lambda_{{ project_name }}_api; - grant usage on integration aws_{{ project_name }}_api_prod_v2 to role internal_dev; - - --Grant usage on stage - grant usage on stage streamline.bronze.{{ project_name }}_serverless_prod to role dbt_cloud_{{ project_name }}; - - {% endset %} - - {% do log(sql, info=true)%} - - {% do run_query(sql) %} - {% do log("Streamline PROD API Integration Permissions for " ~ project_name ~ " successfully ran", true) %} -{% endmacro %} \ No newline at end of file diff --git a/macros/admin/run_streamline_prod_permissions.sql b/macros/admin/run_streamline_prod_permissions.sql deleted file mode 100644 index 78f306b..0000000 --- a/macros/admin/run_streamline_prod_permissions.sql +++ /dev/null @@ -1,82 +0,0 @@ -{% macro run_streamline_prod_permissions(project_name) %} - --This will run after api integration is created - - {% set sql %} - - use role dbt_cloud_{{ project_name }}; - use warehouse dbt; - - --Create Database and Grant Perms - create database if not exists {{ project_name }}; - grant usage on database {{ project_name }} to role aws_lambda_{{ project_name }}_api; - grant usage on database {{ project_name }} to role dbt_cloud_{{ project_name }}; - grant usage on database {{ project_name }} to role internal_dev; - grant usage on database {{ project_name }} to role datascience; - - --Create Streamline Schema and Grant Permissions - create schema if not exists {{ project_name }}.bronze; - create schema if not exists {{ project_name }}._internal; - create schema if not exists {{ project_name }}.silver; - create schema if not exists {{ project_name }}.streamline; - - use role accountadmin; - - grant select on all views in schema {{ project_name }}.streamline to role aws_lambda_{{ project_name }}_api; - grant select on all tables in schema {{ project_name }}.streamline to role aws_lambda_{{ project_name }}_api; - grant usage on all functions in schema {{ project_name }}.streamline to role aws_lambda_{{ project_name }}_api; - grant select on future views in schema {{ project_name }}.streamline to role aws_lambda_{{ project_name }}_api; - grant select on future tables in schema {{ project_name }}.streamline to role aws_lambda_{{ project_name }}_api; - grant usage on future functions in schema {{ project_name }}.streamline to role aws_lambda_{{ project_name }}_api; - - --Internal_DEV Permissions for PROD - grant usage on all schemas in database {{ project_name }} to role internal_dev; - grant usage on future schemas in database {{ project_name }} to role internal_dev; - grant usage on all functions in database {{ project_name }} to role internal_dev; - grant usage on future functions in database {{ project_name }} to role internal_dev; - grant select on all tables in database {{ project_name }} to role internal_dev; - grant select on future tables in database {{ project_name }} to role internal_dev; - grant select on all views in database {{ project_name }} to role internal_dev; - grant select on future views in database {{ project_name }} to role internal_dev; - - grant role dbt_cloud_{{ project_name }} to user dbt_cloud_{{ project_name }}; - - --Permissions for Streamline external tables - use role streamline_snowflake; - create schema if not exists streamline.{{ project_name }}; - - use role accountadmin; - - grant usage on schema streamline.{{ project_name }} to role dbt_cloud_{{ project_name }}; - grant select on all tables in schema streamline.{{ project_name }} to role dbt_cloud_{{ project_name }}; - grant select on all views in schema streamline.{{ project_name }} to role dbt_cloud_{{ project_name }}; - grant select on future tables in schema streamline.{{ project_name }} to role dbt_cloud_{{ project_name }}; - grant select on future views in schema streamline.{{ project_name }} to role dbt_cloud_{{ project_name }}; - grant usage on schema streamline.{{ project_name }}_dev to role dbt_cloud_{{ project_name }}; - grant select on all tables in schema streamline.{{ project_name }}_dev to role dbt_cloud_{{ project_name }}; - grant select on all views in schema streamline.{{ project_name }}_dev to role dbt_cloud_{{ project_name }}; - grant select on future tables in schema streamline.{{ project_name }}_dev to role dbt_cloud_{{ project_name }}; - grant select on future views in schema streamline.{{ project_name }}_dev to role dbt_cloud_{{ project_name }}; - - grant usage on schema streamline.{{ project_name }} to role aws_lambda_{{ project_name }}_api; - grant select on all tables in schema streamline.{{ project_name }} to role aws_lambda_{{ project_name }}_api; - grant select on all views in schema streamline.{{ project_name }} to role aws_lambda_{{ project_name }}_api; - grant select on future tables in schema streamline.{{ project_name }} to role aws_lambda_{{ project_name }}_api; - grant select on future views in schema streamline.{{ project_name }} to role aws_lambda_{{ project_name }}_api; - - grant usage on schema streamline.{{ project_name }} to role internal_dev; - grant select on all tables in schema streamline.{{ project_name }} to role internal_dev; - grant select on all views in schema streamline.{{ project_name }} to role internal_dev; - grant select on future tables in schema streamline.{{ project_name }} to role internal_dev; - grant select on future views in schema streamline.{{ project_name }} to role internal_dev; - - grant usage on schema streamline.{{ project_name }} to role streamline_snowflake; - grant create external table on schema streamline.{{ project_name }} to role streamline_snowflake; - grant create stage on schema streamline.{{ project_name }} to role streamline_snowflake; - - {% endset %} - - {% do log(sql, info=true)%} - - {% do run_query(sql) %} - {% do log("Streamline PROD Permissions for " ~ project_name ~ " successfully ran", true) %} -{% endmacro %} \ No newline at end of file diff --git a/profiles.yml b/profiles.yml index b516c0f..c787be4 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,5 +1,5 @@ berachain: - target: dev + target: prod outputs: dev: type: snowflake @@ -8,7 +8,7 @@ berachain: user: "{{ env_var('USER') }}" password: "{{ env_var('PASSWORD') }}" region: "{{ env_var('REGION') }}" - database: "H_BERACHAIN_DEV" + database: "{{ env_var('DATABASE') }}" warehouse: "{{ env_var('WAREHOUSE') }}" schema: SILVER threads: 4 @@ -19,7 +19,7 @@ berachain: account: "{{ env_var('ACCOUNT') }}" role: "{{ env_var('ROLE') }}" user: "{{ env_var('USER') }}" - password: "H_BERACHAIN" + password: "{{ env_var('PASSWORD') }}" region: "{{ env_var('REGION') }}" database: "{{ env_var('DATABASE') }}" warehouse: "{{ env_var('WAREHOUSE') }}"