diff --git a/.github/workflows/dbt_alter_gha_task.yml b/.github/workflows/dbt_alter_gha_task.yml new file mode 100644 index 0000000..da996d8 --- /dev/null +++ b/.github/workflows/dbt_alter_gha_task.yml @@ -0,0 +1,46 @@ +name: dbt_alter_gha_task +run-name: dbt_alter_gha_task + +on: + workflow_dispatch: + branches: + - "main" + inputs: + workflow_name: + type: string + description: Name of the workflow to perform the action on, no .yml extension + required: true + task_action: + type: choice + description: Action to perform + required: true + options: + - SUSPEND + - RESUME + default: SUSPEND + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_alter_gha_tasks.yml@main + with: + workflow_name: | + ${{ inputs.workflow_name }} + task_action: | + ${{ inputs.task_action }} + environment: workflow_prod + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/dbt_run_daily.yml b/.github/workflows/dbt_run_daily.yml index 7e6f729..2134636 100644 --- a/.github/workflows/dbt_run_daily.yml +++ b/.github/workflows/dbt_run_daily.yml @@ -28,7 +28,7 @@ jobs: with: dbt_command: | dbt run-operation stage_external_sources --vars "ext_full_refresh: true" - dbt run -s tag:balances tag:daily + dbt run -s "osmosis_models,tag:balances" "osmosis_models,tag:daily" dbt run-operation stage_external_sources --vars "ext_full_refresh: true" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} diff --git a/.github/workflows/dbt_run_incremental.yml b/.github/workflows/dbt_run_incremental.yml index 1584bca..f4786ff 100644 --- a/.github/workflows/dbt_run_incremental.yml +++ b/.github/workflows/dbt_run_incremental.yml @@ -3,8 +3,8 @@ run-name: dbt_run_incremental on: workflow_dispatch: - schedule: - - cron: '25,55 * * * *' + # schedule: + # - cron: '25,55 * * * *' env: USE_VARS: "${{ vars.USE_VARS }}" @@ -28,7 +28,7 @@ jobs: with: dbt_command: | dbt run-operation stage_external_sources --vars "ext_full_refresh: true" - dbt run -s ./models --exclude models/silver/_observability models/streamline tag:balances tag:daily + dbt run -s "osmosis_models,./models" --exclude models/silver/_observability models/streamline tag:balances tag:daily dbt run-operation stage_external_sources --vars "ext_full_refresh: true" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} diff --git a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml index da97db7..de5afd7 100644 --- a/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml +++ b/.github/workflows/dbt_run_streamline_blocks_txcount_realtime.yml @@ -3,9 +3,9 @@ run-name: dbt_run_streamline_blocks_txcount_realtime on: workflow_dispatch: - schedule: - # Runs "at minute 25 and 55, every hour" (see https://crontab.guru) - - cron: '0,30 * * * *' + # schedule: + # # Runs "at minute 25 and 55, every hour" (see https://crontab.guru) + # - cron: '0,30 * * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_run_streamline_transactions_realtime.yml b/.github/workflows/dbt_run_streamline_transactions_realtime.yml index 6c980d5..1e955c0 100644 --- a/.github/workflows/dbt_run_streamline_transactions_realtime.yml +++ b/.github/workflows/dbt_run_streamline_transactions_realtime.yml @@ -3,9 +3,9 @@ run-name: dbt_run_streamline_transactions_realtime on: workflow_dispatch: - schedule: - # Runs "at minute 8 and 38, every hour" (see https://crontab.guru) - - cron: '8,38 * * * *' + # schedule: + # # Runs "at minute 8 and 38, every hour" (see https://crontab.guru) + # - cron: '8,38 * * * *' env: DBT_PROFILES_DIR: ./ diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index 0926f27..d9b5d21 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -27,7 +27,7 @@ jobs: uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main with: dbt_command: | - dbt test -m ./models + dbt test -m "osmosis_models,./models" environment: workflow_prod warehouse: ${{ vars.WAREHOUSE }} secrets: inherit diff --git a/.github/workflows/dbt_test_tasks.yml b/.github/workflows/dbt_test_tasks.yml new file mode 100644 index 0000000..976440b --- /dev/null +++ b/.github/workflows/dbt_test_tasks.yml @@ -0,0 +1,27 @@ +name: dbt_test_tasks +run-name: dbt_test_tasks + +on: + workflow_dispatch: + branches: + - "main" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main + secrets: inherit \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv new file mode 100644 index 0000000..15f323d --- /dev/null +++ b/data/github_actions__workflows.csv @@ -0,0 +1,5 @@ +workflow_name,workflow_schedule +dbt_run_incremental,"25,55 * * * *" +dbt_run_streamline_blocks_txcount_realtime,"0,30 * * * *" +dbt_run_streamline_transactions_realtime,"8,38 * * * *" +dbt_test_tasks,"0,30 * * * *" diff --git a/dbt_project.yml b/dbt_project.yml index 00c6580..766cef5 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -48,6 +48,7 @@ vars: STREAMLINE_RUN_HISTORY: False UPDATE_UDFS_AND_SPS: False UPDATE_SNOWFLAKE_TAGS: True + START_GHA_TASKS: False tests: +store_failures: true # all tests diff --git a/models/github_actions/github_actions__current_task_status.sql b/models/github_actions/github_actions__current_task_status.sql new file mode 100644 index 0000000..577a226 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.sql @@ -0,0 +1,6 @@ +{{ config( + materialized = 'view', + tags = ['gha_tasks'] +) }} + +{{ fsc_utils.gha_task_current_status_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__current_task_status.yml b/models/github_actions/github_actions__current_task_status.yml new file mode 100644 index 0000000..e80ad12 --- /dev/null +++ b/models/github_actions/github_actions__current_task_status.yml @@ -0,0 +1,17 @@ +version: 2 +models: + - name: github_actions__current_task_status + columns: + - name: PIPELINE_ACTIVE + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - TRUE + - name: SUCCESSES + tests: + - dbt_expectations.expect_column_values_to_be_in_set: + value_set: + - 204 + config: + severity: warn + warn_if: ">0" \ No newline at end of file diff --git a/models/github_actions/github_actions__task_history.sql b/models/github_actions/github_actions__task_history.sql new file mode 100644 index 0000000..9c35ce7 --- /dev/null +++ b/models/github_actions/github_actions__task_history.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_history_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_performance.sql b/models/github_actions/github_actions__task_performance.sql new file mode 100644 index 0000000..117ded5 --- /dev/null +++ b/models/github_actions/github_actions__task_performance.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_performance_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__task_schedule.sql b/models/github_actions/github_actions__task_schedule.sql new file mode 100644 index 0000000..ff95a44 --- /dev/null +++ b/models/github_actions/github_actions__task_schedule.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_task_schedule_view() }} \ No newline at end of file diff --git a/models/github_actions/github_actions__tasks.sql b/models/github_actions/github_actions__tasks.sql new file mode 100644 index 0000000..feab82a --- /dev/null +++ b/models/github_actions/github_actions__tasks.sql @@ -0,0 +1,5 @@ +{{ config( + materialized = 'view' +) }} + +{{ fsc_utils.gha_tasks_view() }} \ No newline at end of file diff --git a/models/sources.yml b/models/sources.yml index 75b5324..5f3fb27 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -55,4 +55,9 @@ sources: - name: bronze schema: bronze tables: - - name: chainwalkers_txs \ No newline at end of file + - name: chainwalkers_txs + - name: github_actions + database: osmosis + schema: github_actions + tables: + - name: workflows \ No newline at end of file diff --git a/packages.yml b/packages.yml index 4ad5d9d..8a5f089 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_external_tables version: 0.8.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.3.0" + revision: "v1.11.0" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ec44b06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dbt-snowflake>=1.4,<1.5 \ No newline at end of file