From 0f30a4b0a265ea3b0ebdc0d49bab6a1118927e8b Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:20:31 -0600 Subject: [PATCH] add tasks (#296) --- .github/workflows/dbt_alter_gha_task.yml | 46 +++++++++++++++++++ .../dbt_run_streamline_asset_metadata.yml | 5 +- .../dbt_run_streamline_prices_history.yml | 5 +- .../dbt_run_streamline_prices_realtime.yml | 5 +- .github/workflows/dbt_test_tasks.yml | 27 +++++++++++ data/github_actions__workflows.csv | 5 ++ dbt_project.yml | 2 - macros/create_udfs.sql | 5 -- .../github_actions__current_task_status.sql | 6 +++ .../github_actions__current_task_status.yml | 16 +++++++ .../github_actions__task_history.sql | 5 ++ .../github_actions__task_performance.sql | 5 ++ .../github_actions__task_schedule.sql | 5 ++ .../github_actions/github_actions__tasks.sql | 5 ++ 14 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/dbt_alter_gha_task.yml create mode 100644 .github/workflows/dbt_test_tasks.yml create mode 100644 data/github_actions__workflows.csv create mode 100644 models/silver/github_actions/github_actions__current_task_status.sql create mode 100644 models/silver/github_actions/github_actions__current_task_status.yml create mode 100644 models/silver/github_actions/github_actions__task_history.sql create mode 100644 models/silver/github_actions/github_actions__task_performance.sql create mode 100644 models/silver/github_actions/github_actions__task_schedule.sql create mode 100644 models/silver/github_actions/github_actions__tasks.sql diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..da996d8 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,46 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_streamline_asset_metadata.yml b/.github/workflows/dbt_run_streamline_asset_metadata.yml index 6b58505..64c3ef9 100644 --- a/.github/workflows/dbt_run_streamline_asset_metadata.yml +++ b/.github/workflows/dbt_run_streamline_asset_metadata.yml @@ -3,9 +3,8 @@ run-name: dbt_run_streamline_asset_metadata on: workflow_dispatch: - schedule: - # Runs at minute 5 every hour (see https://crontab.guru) - - cron: '5 * * * *' + branches: + - "main" env: DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" diff --git a/.github/workflows/dbt_run_streamline_prices_history.yml b/.github/workflows/dbt_run_streamline_prices_history.yml index 765d2ef..6e1df5e 100644 --- a/.github/workflows/dbt_run_streamline_prices_history.yml +++ b/.github/workflows/dbt_run_streamline_prices_history.yml @@ -3,9 +3,8 @@ run-name: dbt_run_streamline_prices_history on: workflow_dispatch: - schedule: - # Runs once daily (see https://crontab.guru) - - cron: '45 * */1 * *' + branches: + - "main" env: DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" diff --git a/.github/workflows/dbt_run_streamline_prices_realtime.yml b/.github/workflows/dbt_run_streamline_prices_realtime.yml index e495ca2..1121b9f 100644 --- a/.github/workflows/dbt_run_streamline_prices_realtime.yml +++ b/.github/workflows/dbt_run_streamline_prices_realtime.yml @@ -3,9 +3,8 @@ run-name: dbt_run_streamline_prices_realtime on: workflow_dispatch: - schedule: - # Runs at minute 25 every hour (see https://crontab.guru) - - cron: '25 * * * *' + branches: + - "main" env: DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 0000000..976440b --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 0000000..df23244 --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,5 @@ +workflow_name,workflow_schedule +dbt_run_streamline_prices_history,"45 * */1 * *" +dbt_run_streamline_prices_realtime,"25 * * * *" +dbt_run_streamline_asset_metadata,"5 * * * *" +dbt_test_tasks,"15 * * * *" \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 0e337e9..ad7e684 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -27,8 +27,6 @@ clean-targets: # directories to be removed by `dbt clean` on-run-start: - "{{ create_sps() }}" - '{{ create_udfs() }}' - - '{{ sp_create_bulk_get_coin_gecko_asset_metadata() }}' - - '{{ sp_create_bulk_get_coin_market_cap_asset_metadata() }}' - '{{ sp_refresh_external_table_by_recent_date() }}' diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 66d890d..bbd3a94 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -2,11 +2,6 @@ {% if var("UPDATE_UDFS_AND_SPS", False) %} {% set sql %} CREATE schema if NOT EXISTS streamline; - {{ udf_bulk_get_coin_gecko_asset_metadata() }}; - {{ () }}; - {{ udf_bulk_get_coin_market_cap_hourly_ohlc() }}; - {{ udf_bulk_get_coin_gecko_ohlc() }}; - {{ udf_bulk_get_coin_gecko_asset_market_data_historical() }}; {{ create_udf_hex_to_int( schema = "public" ) }} diff --git a/models/silver/github_actions/github_actions__current_task_status.sql b/models/silver/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/silver/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/silver/github_actions/github_actions__current_task_status.yml b/models/silver/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..7923144 --- /dev/null +++ b/models/silver/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,16 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 2 + config: + severity: warn \ No newline at end of file diff --git a/models/silver/github_actions/github_actions__task_history.sql b/models/silver/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/silver/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/silver/github_actions/github_actions__task_performance.sql b/models/silver/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/silver/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/silver/github_actions/github_actions__task_schedule.sql b/models/silver/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/silver/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/silver/github_actions/github_actions__tasks.sql b/models/silver/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/silver/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file