From 21b6ad618d3ff5860fd8654bcecdc4e606f06920 Mon Sep 17 00:00:00 2001 From: drethereum <71602799+drethereum@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:10:29 -0700 Subject: [PATCH] removed temp job (#216) --- .../dbt_run_streamline_temp_decoder_2.yml | 45 ------------------- models/silver/core/silver__decoded_logs2.sql | 24 +++------- 2 files changed, 5 insertions(+), 64 deletions(-) delete mode 100644 .github/workflows/dbt_run_streamline_temp_decoder_2.yml diff --git a/.github/workflows/dbt_run_streamline_temp_decoder_2.yml b/.github/workflows/dbt_run_streamline_temp_decoder_2.yml deleted file mode 100644 index 1c274d9f..00000000 --- a/.github/workflows/dbt_run_streamline_temp_decoder_2.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: dbt_run_streamline_temp_decoder2 -run-name: dbt_run_streamline_temp_decoder2 - -on: - workflow_dispatch: - schedule: - # Runs "every 2 hours" (see https://crontab.guru) - - cron: '5 */2 * * *' - -env: - DBT_PROFILES_DIR: ./ - - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - run_dbt_jobs: - runs-on: ubuntu-latest - environment: - name: workflow_prod_backfill - - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - name: Run DBT Jobs - run: | - dbt run -m "optimism_models,tag:decoded_logs2" \ No newline at end of file diff --git a/models/silver/core/silver__decoded_logs2.sql b/models/silver/core/silver__decoded_logs2.sql index 891ff8af..8ea6cce9 100644 --- a/models/silver/core/silver__decoded_logs2.sql +++ b/models/silver/core/silver__decoded_logs2.sql @@ -6,13 +6,10 @@ incremental_predicates = ["dynamic_range", "block_number"], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION", merge_exclude_columns = ["inserted_timestamp"], - tags = ['decoded_logs2'] + full_refresh = false, + tags = ['decoded_logs','reorg'] ) }} --- revert after backfill --- full_refresh = false, --- tags = ['decoded_logs','reorg'] - WITH base_data AS ( SELECT @@ -35,25 +32,14 @@ WITH base_data AS ( FROM {% if is_incremental() %} -{{ ref('bronze__fr_decoded_logs') }} --revert to bronze__decoded_logs after backfill +{{ ref('bronze__decoded_logs') }} WHERE - --TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( - --SELECT - --MAX(_inserted_timestamp) - --FROM - --{{ this }} - --) - _partition_by_block_number BETWEEN ( + TO_TIMESTAMP_NTZ(_inserted_timestamp) >= ( SELECT - ROUND(MAX(block_number), -4) + MAX(_inserted_timestamp) FROM {{ this }} ) - AND ( - SELECT - ROUND(MAX(block_number), -4) + 1000000 - FROM - {{ this }}) AND DATA NOT ILIKE '%Event topic is not present in given ABI%' {% else %} {{ ref('bronze__fr_decoded_logs') }}