From 573fd8cd26422b9a185e631f58d26da156c1846a Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 26 Jan 2024 12:56:58 +0700 Subject: [PATCH 1/3] update 1.7 --- .github/workflows/dbt_alter_gha_task.yml | 12 ++++++------ .github/workflows/dbt_docs_update.yml | 4 ++-- .github/workflows/dbt_run_adhoc.yml | 4 ++-- .github/workflows/dbt_run_atlas.yml | 2 +- .github/workflows/dbt_run_deployment.yml | 2 +- .github/workflows/dbt_run_dev_refresh.yml | 2 +- .github/workflows/dbt_run_full_observability.yml | 4 ++-- .github/workflows/dbt_run_observability.yml | 4 ++-- .github/workflows/dbt_run_scheduled.yml | 2 +- .github/workflows/dbt_run_streamline_history.yml | 2 +- ...run_streamline_realtime_blocks_transactions.yml | 2 +- .../dbt_run_streamline_realtime_tx_receipts.yml | 2 +- .../dbt_run_streamline_traces_history.yml | 4 ++-- .github/workflows/dbt_test_daily.yml | 4 ++-- .github/workflows/dbt_test_tasks.yml | 6 +++--- .github/workflows/dbt_test_weekly.yml | 4 ++-- package-lock.yml | 14 ++++++++++++++ requirements.txt | 2 +- 18 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 package-lock.yml diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml index da996d8..d3094a6 100644 --- a/.github/workflows/dbt_alter_gha_task.yml +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -11,14 +11,14 @@ on: description: Name of the workflow to perform the action on, no .yml extension required: true task_action: - type: choice + type: choice description: Action to perform required: true options: - SUSPEND - RESUME default: SUSPEND - + env: DBT_PROFILES_DIR: ./ @@ -36,11 +36,11 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@AN-4374/upgrade-dbt-1.7 with: workflow_name: | - ${{ inputs.workflow_name }} + ${{ inputs.workflow_name }} task_action: | - ${{ inputs.task_action }} + ${{ inputs.task_action }} environment: workflow_prod - secrets: inherit \ No newline at end of file + secrets: inherit diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 51f2d88..b6b463d 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -23,5 +23,5 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main - secrets: inherit + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 0c0f963..96b5f32 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -53,9 +53,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_atlas.yml b/.github/workflows/dbt_run_atlas.yml index d4949d6..f8fbea0 100644 --- a/.github/workflows/dbt_run_atlas.yml +++ b/.github/workflows/dbt_run_atlas.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:atlas diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index 01ceb6b..dc5a8d5 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -38,7 +38,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: | ${{ inputs.dbt_command }} diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index e83e55c..f3232df 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -26,7 +26,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_full_observability.yml b/.github/workflows/dbt_run_full_observability.yml index 631aaaa..80177d6 100644 --- a/.github/workflows/dbt_run_full_observability.yml +++ b/.github/workflows/dbt_run_full_observability.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_observability.yml b/.github/workflows/dbt_run_observability.yml index 3c9827c..8cfd82b 100644 --- a/.github/workflows/dbt_run_observability.yml +++ b/.github/workflows/dbt_run_observability.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 6af6314..c5bc87e 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:core diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 07ac885..552af64 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -22,7 +22,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/silver/streamline/realtime/ --full-refresh diff --git a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml index a4689ee..7e28767 100644 --- a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml +++ b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml index 44b8a26..db0625d 100644 --- a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml +++ b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_traces_history.yml b/.github/workflows/dbt_run_streamline_traces_history.yml index 65f8547..b725ff3 100644 --- a/.github/workflows/dbt_run_streamline_traces_history.yml +++ b/.github/workflows/dbt_run_streamline_traces_history.yml @@ -4,7 +4,7 @@ run-name: dbt_run_streamline_traces_history on: workflow_dispatch: schedule: - # Run every 2 hours + # Run every 2 hours - cron: "0 */2 * * *" env: @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql diff --git a/.github/workflows/dbt_test_daily.yml b/.github/workflows/dbt_test_daily.yml index ca470a6..e3b06fa 100644 --- a/.github/workflows/dbt_test_daily.yml +++ b/.github/workflows/dbt_test_daily.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml index 976440b..594148b 100644 --- a/.github/workflows/dbt_test_tasks.yml +++ b/.github/workflows/dbt_test_tasks.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: branches: - "main" - + env: DBT_PROFILES_DIR: ./ @@ -23,5 +23,5 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main - secrets: inherit \ No newline at end of file + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7 + secrets: inherit diff --git a/.github/workflows/dbt_test_weekly.yml b/.github/workflows/dbt_test_weekly.yml index 9c73f86..ce504a4 100644 --- a/.github/workflows/dbt_test_weekly.yml +++ b/.github/workflows/dbt_test_weekly.yml @@ -32,9 +32,9 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" - name: install dependencies run: | diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..36b5126 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,14 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.5 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: c84d623aa09ce2acb9451e6aedf5fa70c19b0b95 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.1 +- package: calogica/dbt_date + version: 0.7.2 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: 97973763ca6ea590de018e4df5195006f2827e27 diff --git a/requirements.txt b/requirements.txt index ec44b06..c61d3e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -dbt-snowflake>=1.4,<1.5 \ No newline at end of file +dbt-snowflake>=1.7,<1.8 From b49f9abb7e6cdca555d4638fcd5e023f0fabe63a Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:27:56 +0700 Subject: [PATCH 2/3] profile --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 26deae9..8968aea 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -5,7 +5,7 @@ name: "aurora_models" version: "1.0.0" config-version: 2 -require-dbt-version: ">=1.4.0" +require-dbt-version: ">=1.7.0" # This setting configures which "profile" dbt uses for this project. profile: "aurora" From 5e32a86f34f1705693e720f0d9f8ed3c1a5a905d Mon Sep 17 00:00:00 2001 From: WHYTEWYLL <49686519+WHYTEWYLL@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:31:03 +0700 Subject: [PATCH 3/3] @ fix --- .github/workflows/dbt_run_atlas.yml | 2 +- .github/workflows/dbt_run_dev_refresh.yml | 2 +- .github/workflows/dbt_run_scheduled.yml | 2 +- .github/workflows/dbt_run_streamline_history.yml | 2 +- .../dbt_run_streamline_realtime_blocks_transactions.yml | 2 +- .github/workflows/dbt_run_streamline_realtime_tx_receipts.yml | 2 +- .github/workflows/dbt_run_streamline_traces_history.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dbt_run_atlas.yml b/.github/workflows/dbt_run_atlas.yml index f8fbea0..cbbc3f2 100644 --- a/.github/workflows/dbt_run_atlas.yml +++ b/.github/workflows/dbt_run_atlas.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:atlas diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index f3232df..cce0cc9 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -26,7 +26,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run-operation run_sp_create_prod_clone diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index c5bc87e..79393c9 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run -s tag:core diff --git a/.github/workflows/dbt_run_streamline_history.yml b/.github/workflows/dbt_run_streamline_history.yml index 552af64..d42c955 100644 --- a/.github/workflows/dbt_run_streamline_history.yml +++ b/.github/workflows/dbt_run_streamline_history.yml @@ -22,7 +22,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_RUN_HISTORY":True}' -m 1+models/silver/streamline/realtime/ --full-refresh diff --git a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml index 7e28767..3c77a92 100644 --- a/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml +++ b/.github/workflows/dbt_run_streamline_realtime_blocks_transactions.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__blocks_realtime.sql 1+models/silver/streamline/realtime/streamline__transactions_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml index db0625d..8d360ee 100644 --- a/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml +++ b/.github/workflows/dbt_run_streamline_realtime_tx_receipts.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/realtime/streamline__tx_receipts_realtime.sql diff --git a/.github/workflows/dbt_run_streamline_traces_history.yml b/.github/workflows/dbt_run_streamline_traces_history.yml index b725ff3..d42c2ee 100644 --- a/.github/workflows/dbt_run_streamline_traces_history.yml +++ b/.github/workflows/dbt_run_streamline_traces_history.yml @@ -25,7 +25,7 @@ concurrency: jobs: called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@@AN-4374/upgrade-dbt-1.7 + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@AN-4374/upgrade-dbt-1.7 with: dbt_command: > dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/silver/streamline/history/streamline__traces_history.sql