diff --git a/.github/workflows/dbt_run_scheduled_curated.yml b/.github/workflows/dbt_run_scheduled_curated.yml new file mode 100644 index 0000000..c9474c9 --- /dev/null +++ b/.github/workflows/dbt_run_scheduled_curated.yml @@ -0,0 +1,45 @@ +name: dbt_run_scheduled_curated +run-name: dbt_run_scheduled_curated + +on: + workflow_dispatch: + schedule: + # Runs at minutes 15 and 45 (see https://crontab.guru) + - cron: '15,45 * * * *' + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + dbt run -m "polygon_models,tag:curated" \ No newline at end of file diff --git a/models/silver/NFT/silver__complete_nft_sales.sql b/models/silver/NFT/silver__complete_nft_sales.sql index 6af35b7..8ed0802 100644 --- a/models/silver/NFT/silver__complete_nft_sales.sql +++ b/models/silver/NFT/silver__complete_nft_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = ['block_number','platform_name','platform_exchange_version'], cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH nft_base_models AS ( diff --git a/models/silver/NFT/silver__nft_transfers.sql b/models/silver/NFT/silver__nft_transfers.sql index b6ca608..a895d17 100644 --- a/models/silver/NFT/silver__nft_transfers.sql +++ b/models/silver/NFT/silver__nft_transfers.sql @@ -4,7 +4,7 @@ unique_key = 'block_number', cluster_by = ['block_timestamp::DATE', '_inserted_timestamp::DATE'], post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(contract_address, tx_hash)", - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH base AS ( diff --git a/models/silver/NFT/silver__seaport_1_1_sales.sql b/models/silver/NFT/silver__seaport_1_1_sales.sql index d2ae19f..dae28fb 100644 --- a/models/silver/NFT/silver__seaport_1_1_sales.sql +++ b/models/silver/NFT/silver__seaport_1_1_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH seaport_fees_wallet AS ( diff --git a/models/silver/NFT/silver__seaport_1_4_sales.sql b/models/silver/NFT/silver__seaport_1_4_sales.sql index 67be4d6..2d9eb93 100644 --- a/models/silver/NFT/silver__seaport_1_4_sales.sql +++ b/models/silver/NFT/silver__seaport_1_4_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH seaport_fees_wallet AS ( diff --git a/models/silver/NFT/silver__seaport_1_5_sales.sql b/models/silver/NFT/silver__seaport_1_5_sales.sql index cc4ef63..13f1a5a 100644 --- a/models/silver/NFT/silver__seaport_1_5_sales.sql +++ b/models/silver/NFT/silver__seaport_1_5_sales.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH seaport_fees_wallet AS ( diff --git a/models/silver/dex/balancer/silver_dex__balancer_pools.sql b/models/silver/dex/balancer/silver_dex__balancer_pools.sql index 612a8a1..15ac235 100644 --- a/models/silver/dex/balancer/silver_dex__balancer_pools.sql +++ b/models/silver/dex/balancer/silver_dex__balancer_pools.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', full_refresh = false, - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pools_registered AS ( diff --git a/models/silver/dex/balancer/silver_dex__balancer_swaps.sql b/models/silver/dex/balancer/silver_dex__balancer_swaps.sql index 9d2c8e7..dbaf602 100644 --- a/models/silver/dex/balancer/silver_dex__balancer_swaps.sql +++ b/models/silver/dex/balancer/silver_dex__balancer_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pool_name AS ( diff --git a/models/silver/dex/curve/silver_dex__curve_pools.sql b/models/silver/dex/curve/silver_dex__curve_pools.sql index 7d97b0f..2e65e6a 100644 --- a/models/silver/dex/curve/silver_dex__curve_pools.sql +++ b/models/silver/dex/curve/silver_dex__curve_pools.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', full_refresh = false, - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH contract_deployments AS ( diff --git a/models/silver/dex/curve/silver_dex__curve_swaps.sql b/models/silver/dex/curve/silver_dex__curve_swaps.sql index 7c13b8e..f6dc37a 100644 --- a/models/silver/dex/curve/silver_dex__curve_swaps.sql +++ b/models/silver/dex/curve/silver_dex__curve_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pool_meta AS ( diff --git a/models/silver/dex/dodo/silver_dex__dodo_v1_pools.sql b/models/silver/dex/dodo/silver_dex__dodo_v1_pools.sql index 4192d03..4ddbd3e 100644 --- a/models/silver/dex/dodo/silver_dex__dodo_v1_pools.sql +++ b/models/silver/dex/dodo/silver_dex__dodo_v1_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_events AS ( diff --git a/models/silver/dex/dodo/silver_dex__dodo_v1_swaps.sql b/models/silver/dex/dodo/silver_dex__dodo_v1_swaps.sql index 5b15659..3198be6 100644 --- a/models/silver/dex/dodo/silver_dex__dodo_v1_swaps.sql +++ b/models/silver/dex/dodo/silver_dex__dodo_v1_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/dodo/silver_dex__dodo_v2_pools.sql b/models/silver/dex/dodo/silver_dex__dodo_v2_pools.sql index c3fc6e0..80a5721 100644 --- a/models/silver/dex/dodo/silver_dex__dodo_v2_pools.sql +++ b/models/silver/dex/dodo/silver_dex__dodo_v2_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pools AS ( diff --git a/models/silver/dex/dodo/silver_dex__dodo_v2_swaps.sql b/models/silver/dex/dodo/silver_dex__dodo_v2_swaps.sql index 7e5b54a..2830288 100644 --- a/models/silver/dex/dodo/silver_dex__dodo_v2_swaps.sql +++ b/models/silver/dex/dodo/silver_dex__dodo_v2_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/fraxswap/silver_dex__fraxswap_pools.sql b/models/silver/dex/fraxswap/silver_dex__fraxswap_pools.sql index ad66fd9..1007e15 100644 --- a/models/silver/dex/fraxswap/silver_dex__fraxswap_pools.sql +++ b/models/silver/dex/fraxswap/silver_dex__fraxswap_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/fraxswap/silver_dex__fraxswap_swaps.sql b/models/silver/dex/fraxswap/silver_dex__fraxswap_swaps.sql index 7f0de31..ac2ebe3 100644 --- a/models/silver/dex/fraxswap/silver_dex__fraxswap_swaps.sql +++ b/models/silver/dex/fraxswap/silver_dex__fraxswap_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/hashflow/silver_dex__hashflow_pools.sql b/models/silver/dex/hashflow/silver_dex__hashflow_pools.sql index df3c2d5..d9676fd 100644 --- a/models/silver/dex/hashflow/silver_dex__hashflow_pools.sql +++ b/models/silver/dex/hashflow/silver_dex__hashflow_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH contract_deployments AS ( diff --git a/models/silver/dex/hashflow/silver_dex__hashflow_swaps.sql b/models/silver/dex/hashflow/silver_dex__hashflow_swaps.sql index a246a72..193b658 100644 --- a/models/silver/dex/hashflow/silver_dex__hashflow_swaps.sql +++ b/models/silver/dex/hashflow/silver_dex__hashflow_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql index 2832073..991d64f 100644 --- a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql +++ b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql index 68c781d..cdc7f53 100644 --- a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql +++ b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_dynamic_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql index 5eee9b3..81e08de 100644 --- a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql +++ b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql index 65052d0..ec3637a 100644 --- a/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql +++ b/models/silver/dex/kyberswap/silver_dex__kyberswap_v1_static_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql b/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql index 85e7c57..b6f8e81 100644 --- a/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql +++ b/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql b/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql index 7c70219..299ff15 100644 --- a/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql +++ b/models/silver/dex/kyberswap/silver_dex__kyberswap_v2_elastic_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/quickswap/silver_dex__quickswap_v2_pools.sql b/models/silver/dex/quickswap/silver_dex__quickswap_v2_pools.sql index fa7a63d..092082b 100644 --- a/models/silver/dex/quickswap/silver_dex__quickswap_v2_pools.sql +++ b/models/silver/dex/quickswap/silver_dex__quickswap_v2_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/quickswap/silver_dex__quickswap_v2_swaps.sql b/models/silver/dex/quickswap/silver_dex__quickswap_v2_swaps.sql index 2b6ba63..5713930 100644 --- a/models/silver/dex/quickswap/silver_dex__quickswap_v2_swaps.sql +++ b/models/silver/dex/quickswap/silver_dex__quickswap_v2_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/quickswap/silver_dex__quickswap_v3_pools.sql b/models/silver/dex/quickswap/silver_dex__quickswap_v3_pools.sql index 8928828..bc1940f 100644 --- a/models/silver/dex/quickswap/silver_dex__quickswap_v3_pools.sql +++ b/models/silver/dex/quickswap/silver_dex__quickswap_v3_pools.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['_inserted_timestamp::DATE'], - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/quickswap/silver_dex__quickswap_v3_swaps.sql b/models/silver/dex/quickswap/silver_dex__quickswap_v3_swaps.sql index ca93631..d570953 100644 --- a/models/silver/dex/quickswap/silver_dex__quickswap_v3_swaps.sql +++ b/models/silver/dex/quickswap/silver_dex__quickswap_v3_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH base_swaps AS ( diff --git a/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql b/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql index ffcd32c..2338563 100644 --- a/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql +++ b/models/silver/dex/silver_dex__complete_dex_liquidity_pools.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH contracts AS ( diff --git a/models/silver/dex/silver_dex__complete_dex_swaps.sql b/models/silver/dex/silver_dex__complete_dex_swaps.sql index 284f395..03a04aa 100644 --- a/models/silver/dex/silver_dex__complete_dex_swaps.sql +++ b/models/silver/dex/silver_dex__complete_dex_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = ['block_number','platform','version'], cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH contracts AS ( diff --git a/models/silver/dex/sushi/silver_dex__sushi_pools.sql b/models/silver/dex/sushi/silver_dex__sushi_pools.sql index 9dd36fc..1a69158 100644 --- a/models/silver/dex/sushi/silver_dex__sushi_pools.sql +++ b/models/silver/dex/sushi/silver_dex__sushi_pools.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'block_number', - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH pool_creation AS ( diff --git a/models/silver/dex/sushi/silver_dex__sushi_swaps.sql b/models/silver/dex/sushi/silver_dex__sushi_swaps.sql index b145569..cb41e8f 100644 --- a/models/silver/dex/sushi/silver_dex__sushi_swaps.sql +++ b/models/silver/dex/sushi/silver_dex__sushi_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH pools AS ( diff --git a/models/silver/dex/uniswap/silver_dex__univ3_pools.sql b/models/silver/dex/uniswap/silver_dex__univ3_pools.sql index b50908c..412a694 100644 --- a/models/silver/dex/uniswap/silver_dex__univ3_pools.sql +++ b/models/silver/dex/uniswap/silver_dex__univ3_pools.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'created_block', cluster_by = ['_inserted_timestamp::DATE'], - tags = ['non_realtime'] + tags = ['curated'] ) }} WITH created_pools AS ( diff --git a/models/silver/dex/uniswap/silver_dex__univ3_swaps.sql b/models/silver/dex/uniswap/silver_dex__univ3_swaps.sql index e38dc2d..46b91e0 100644 --- a/models/silver/dex/uniswap/silver_dex__univ3_swaps.sql +++ b/models/silver/dex/uniswap/silver_dex__univ3_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH base_swaps AS ( diff --git a/models/silver/dex/woofi/silver_dex__woofi_swaps.sql b/models/silver/dex/woofi/silver_dex__woofi_swaps.sql index e067138..06a403e 100644 --- a/models/silver/dex/woofi/silver_dex__woofi_swaps.sql +++ b/models/silver/dex/woofi/silver_dex__woofi_swaps.sql @@ -3,7 +3,7 @@ incremental_strategy = 'delete+insert', unique_key = 'block_number', cluster_by = ['block_timestamp::DATE'], - tags = ['non_realtime','reorg'] + tags = ['curated','reorg'] ) }} WITH router_swaps_base AS (