From 04172fff95db1b97582c954cc8d34ffdb8583b0a Mon Sep 17 00:00:00 2001 From: Eric Laurello Date: Thu, 2 Nov 2023 15:04:54 -0400 Subject: [PATCH] ch1 backfill turn on --- .../dbt_run_streamline_blocks_ch1.yml | 35 +++++++++++++++++++ .../dbt_run_streamline_transactions_ch1.yml | 34 ++++++++++++++++++ .../streamline__complete_transactions_ch1.sql | 2 +- packages.yml | 2 +- 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/dbt_run_streamline_blocks_ch1.yml create mode 100644 .github/workflows/dbt_run_streamline_transactions_ch1.yml diff --git a/.github/workflows/dbt_run_streamline_blocks_ch1.yml b/.github/workflows/dbt_run_streamline_blocks_ch1.yml new file mode 100644 index 0000000..c53a5c4 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_blocks_ch1.yml @@ -0,0 +1,35 @@ +name: dbt_run_streamline_blocks +run-name: dbt_run_streamline_blocks + +on: + workflow_dispatch: + schedule: + # “At minute 0 and 30.” + - cron: '0,30 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit + diff --git a/.github/workflows/dbt_run_streamline_transactions_ch1.yml b/.github/workflows/dbt_run_streamline_transactions_ch1.yml new file mode 100644 index 0000000..1afd34c --- /dev/null +++ b/.github/workflows/dbt_run_streamline_transactions_ch1.yml @@ -0,0 +1,34 @@ +name: dbt_run_streamline_transactions +run-name: dbt_run_streamline_transactions + +on: + workflow_dispatch: + schedule: + # “At minute 8 and 38 every hour” + - cron: '8,38 * * * *' + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + called_workflow_template: + uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main + with: + dbt_command: | + dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m 1+models/streamline/genesis_backfill/cosmos_hub_1/streamline__transactions_genesis_backfill_ch1.sql + environment: workflow_prod + warehouse: ${{ vars.WAREHOUSE }} + secrets: inherit \ No newline at end of file diff --git a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql index 48acf2a..41b94b9 100644 --- a/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql +++ b/models/streamline/genesis_backfill/complete/cosmos_hub_1/streamline__complete_transactions_ch1.sql @@ -37,7 +37,7 @@ max_date AS ( "tx_search_ch1" ) }} JOIN meta b - ON b.file_name = metadata $ filename + ON b.file_name = metadata$filename {% if is_incremental() %} WHERE diff --git a/packages.yml b/packages.yml index 14e0503..6c4d9d9 100644 --- a/packages.yml +++ b/packages.yml @@ -4,6 +4,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.3.0" + revision: "v1.9.3" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] \ No newline at end of file