revert to normal job (#307)

This commit is contained in:
drethereum 2023-11-15 15:42:08 -07:00 committed by GitHub
parent 650e1df7c7
commit 5e2e5c07cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 64 deletions

View File

@ -1,46 +0,0 @@
name: dbt_run_streamline_temp_decoder2
run-name: dbt_run_streamline_temp_decoder2
on:
workflow_dispatch:
schedule:
# Runs "every 2 hours" (see https://crontab.guru)
- cron: '5 */2 * * *'
env:
DBT_PROFILES_DIR: ./
ACCOUNT: "${{ vars.ACCOUNT }}"
ROLE: "${{ vars.ROLE }}"
USER: "${{ vars.USER }}"
PASSWORD: "${{ secrets.PASSWORD }}"
REGION: "${{ vars.REGION }}"
DATABASE: "${{ vars.DATABASE }}"
WAREHOUSE: "${{ vars.WAREHOUSE }}"
SCHEMA: "${{ vars.SCHEMA }}"
concurrency:
group: ${{ github.workflow }}
jobs:
run_dbt_jobs:
runs-on: ubuntu-latest
environment:
name: workflow_prod_2xl
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "pip"
- name: install dependencies
run: |
pip install -r requirements.txt
dbt deps
- name: Run DBT Jobs
run: |
dbt run -m "polygon_models,tag:decoded_logs2"

View File

@ -6,12 +6,10 @@
incremental_predicates = ["dynamic_range", "block_number"],
merge_exclude_columns = ["inserted_timestamp"],
post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION",
tags = ['decoded_logs2']
full_refresh = false,
tags = ['decoded_logs','reorg']
) }}
-- revert after backfill
-- full_refresh = false,
-- tags = ['decoded_logs','reorg']
WITH base_data AS (
@ -35,25 +33,14 @@ WITH base_data AS (
FROM
{% if is_incremental() %}
{{ ref('bronze__fr_decoded_logs') }} --revert to bronze__decoded_logs after backfill
{{ ref('bronze__decoded_logs') }}
WHERE
--TO_TIMESTAMP_NTZ(_inserted_timestamp) >= (
--SELECT
--MAX(_inserted_timestamp)
--FROM
--{{ this }}
--)
_partition_by_block_number BETWEEN (
TO_TIMESTAMP_NTZ(_inserted_timestamp) >= (
SELECT
ROUND(MAX(block_number), -4)
MAX(_inserted_timestamp)
FROM
{{ this }}
)
AND (
SELECT
ROUND(MAX(block_number), -4) + 1000000
FROM
{{ this }})
{% else %}
{{ ref('bronze__fr_decoded_logs') }}
WHERE