mirror of
https://github.com/FlipsideCrypto/crosschain-models.git
synced 2026-02-06 13:26:52 +00:00
updates for fsc-evm migrations (#437)
* updates for fsc-evm migrations * update workflows for slack notify and sources
This commit is contained in:
parent
b341f476e3
commit
398df8f163
28
.github/workflows/dbt_docs_update.yml
vendored
28
.github/workflows/dbt_docs_update.yml
vendored
@ -6,8 +6,8 @@ on:
|
||||
- "main"
|
||||
|
||||
env:
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
DBT_PROFILES_DIR: ${{ vars.DBT_PROFILES_DIR }}
|
||||
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
@ -19,7 +19,9 @@ env:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
|
||||
|
||||
|
||||
jobs:
|
||||
scheduled_run:
|
||||
name: docs_update
|
||||
@ -30,18 +32,17 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: actions/setup-python@v1
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10.x"
|
||||
|
||||
python-version: "3.10"
|
||||
cache: "pip"
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install dbt-snowflake==1.3.0
|
||||
pip install dbt-snowflake==1.7.0
|
||||
dbt deps
|
||||
|
||||
- name: checkout docs branch
|
||||
run: |
|
||||
git checkout -B docs origin/main
|
||||
git checkout -b docs origin/main
|
||||
|
||||
- name: generate dbt docs
|
||||
run: dbt docs generate -t prod
|
||||
@ -66,4 +67,11 @@ jobs:
|
||||
git commit -am "Auto-update docs"
|
||||
- name: push changes to docs
|
||||
run: |
|
||||
git push -f --set-upstream origin docs
|
||||
git push -f --set-upstream origin docs
|
||||
|
||||
notify-failure:
|
||||
needs: [scheduled_run]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -43,3 +43,10 @@ jobs:
|
||||
dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'TOKEN', frequency: ['contributors']}"
|
||||
dbt run -m "crosschain_models,tag:github_activity"
|
||||
dbt test -s github_data_null
|
||||
|
||||
notify-failure:
|
||||
needs: [dbt]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -43,3 +43,10 @@ jobs:
|
||||
dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'TOKEN', frequency: ['core']}"
|
||||
dbt run -m "crosschain_models,tag:github_activity"
|
||||
dbt test -s github_data_null
|
||||
|
||||
notify-failure:
|
||||
needs: [dbt]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -42,3 +42,10 @@ jobs:
|
||||
run: |
|
||||
dbt run-operation get_github_repo_data --args "{GITHUB_TOKEN: 'TOKEN', frequency: ['last_year']}"
|
||||
dbt run -m "crosschain_models,tag:github_activity"
|
||||
|
||||
notify-failure:
|
||||
needs: [dbt]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
9
.github/workflows/dbt_run_adhoc.yml
vendored
9
.github/workflows/dbt_run_adhoc.yml
vendored
@ -67,4 +67,11 @@ jobs:
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
${{ inputs.dbt_command }}
|
||||
${{ inputs.dbt_command }}
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
68
.github/workflows/dbt_run_deployment.yml
vendored
68
.github/workflows/dbt_run_deployment.yml
vendored
@ -1,68 +0,0 @@
|
||||
name: dbt_run_deployment
|
||||
run-name: dbt_run_deployment
|
||||
on:
|
||||
workflow_dispatch:
|
||||
branches:
|
||||
- "main"
|
||||
inputs:
|
||||
warehouse:
|
||||
type: choice
|
||||
description: Snowflake warehouse
|
||||
required: true
|
||||
options:
|
||||
- DBT
|
||||
- DBT_CLOUD
|
||||
- DBT_EMERGENCY
|
||||
default: DBT
|
||||
dbt_command:
|
||||
type: string
|
||||
description: "DBT Run Command"
|
||||
required: true
|
||||
env:
|
||||
USE_VARS: "${{ vars.USE_VARS }}"
|
||||
DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"
|
||||
DBT_VERSION: "${{ vars.DBT_VERSION }}"
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ inputs.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
|
||||
jobs:
|
||||
dbt:
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: workflow_prod
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
cache: "pip"
|
||||
|
||||
- name: install dependencies
|
||||
run: |
|
||||
pip install -r requirements.txt
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
${{ inputs.dbt_command }}
|
||||
- name: Run datashare model
|
||||
run: |
|
||||
cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi;
|
||||
dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}"
|
||||
- name: Store logs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: dbt-logs
|
||||
path: |
|
||||
logs
|
||||
target
|
||||
|
||||
14
.github/workflows/dbt_run_dev_refresh.yml
vendored
14
.github/workflows/dbt_run_dev_refresh.yml
vendored
@ -42,6 +42,13 @@ jobs:
|
||||
run: |
|
||||
dbt run-operation run_sp_create_prod_clone
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs_refresh]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
run_dbt_jobs_udfs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: run_dbt_jobs_refresh
|
||||
@ -64,3 +71,10 @@ jobs:
|
||||
run: |
|
||||
dbt run-operation fsc_utils.create_udf_bulk_rest_api_v2 --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
|
||||
dbt run -s livequery_models.deploy.core._live --vars '{"UPDATE_UDFS_AND_SPS":True}' -t dev
|
||||
|
||||
notify-failure-2:
|
||||
needs: [run_dbt_jobs_udfs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
9
.github/workflows/dbt_run_heal_models.yml
vendored
9
.github/workflows/dbt_run_heal_models.yml
vendored
@ -44,4 +44,11 @@ jobs:
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run -m "crosschain_models,tag:heal" --vars '{"HEAL_MODEL":True}'
|
||||
dbt run -m "crosschain_models,tag:heal" --vars '{"HEAL_MODEL":True}'
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -43,3 +43,10 @@ jobs:
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run -m "crosschain_models,tag:daily"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -42,4 +42,11 @@ jobs:
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run -m "crosschain_models,tag:hourly"
|
||||
dbt run -m "crosschain_models,tag:hourly"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -43,3 +43,10 @@ jobs:
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run -m "crosschain_models,tag:monthly"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -42,3 +42,10 @@ jobs:
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run -m "crosschain_models,tag:prices"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -43,4 +43,11 @@ jobs:
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run -m "crosschain_models,tag:weekly"
|
||||
dbt run -m "crosschain_models,tag:weekly_full_refresh" --full-refresh
|
||||
dbt run -m "crosschain_models,tag:weekly_full_refresh" --full-refresh
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -44,4 +44,11 @@ jobs:
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "crosschain_models,tag:streamline_cmc_asset_metadata" "crosschain_models,tag:streamline_cg_asset_metadata"
|
||||
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "crosschain_models,tag:streamline_cmc_asset_metadata" "crosschain_models,tag:streamline_cg_asset_metadata"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -44,4 +44,11 @@ jobs:
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "crosschain_models,tag:streamline_cg_prices_complete" "crosschain_models,tag:streamline_cg_prices_history"
|
||||
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "crosschain_models,tag:streamline_cg_prices_complete" "crosschain_models,tag:streamline_cg_prices_history"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
@ -44,4 +44,11 @@ jobs:
|
||||
dbt deps
|
||||
- name: Run DBT Jobs
|
||||
run: |
|
||||
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "crosschain_models,tag:streamline_cmc_prices_complete" "crosschain_models,tag:streamline_cmc_prices_realtime" "crosschain_models,tag:streamline_cg_prices_realtime"
|
||||
dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "crosschain_models,tag:streamline_cmc_prices_complete" "crosschain_models,tag:streamline_cmc_prices_realtime" "crosschain_models,tag:streamline_cg_prices_realtime"
|
||||
|
||||
notify-failure:
|
||||
needs: [run_dbt_jobs]
|
||||
if: failure()
|
||||
uses: ./.github/workflows/slack_notify.yml
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
29
.github/workflows/dbt_test_tasks.yml
vendored
29
.github/workflows/dbt_test_tasks.yml
vendored
@ -1,29 +0,0 @@
|
||||
name: dbt_test_tasks
|
||||
run-name: dbt_test_tasks
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
branches:
|
||||
- "main"
|
||||
|
||||
env:
|
||||
DBT_PROFILES_DIR: ./
|
||||
|
||||
ACCOUNT: "${{ vars.ACCOUNT }}"
|
||||
ROLE: "${{ vars.ROLE }}"
|
||||
USER: "${{ vars.USER }}"
|
||||
PASSWORD: "${{ secrets.PASSWORD }}"
|
||||
REGION: "${{ vars.REGION }}"
|
||||
DATABASE: "${{ vars.DATABASE }}"
|
||||
WAREHOUSE: "${{ vars.WAREHOUSE }}"
|
||||
SCHEMA: "${{ vars.SCHEMA }}"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
|
||||
|
||||
|
||||
jobs:
|
||||
called_workflow_template:
|
||||
uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@main
|
||||
secrets: inherit
|
||||
27
.github/workflows/slack_notify.yml
vendored
Normal file
27
.github/workflows/slack_notify.yml
vendored
Normal file
@ -0,0 +1,27 @@
|
||||
name: Slack Notification
|
||||
on:
|
||||
workflow_call:
|
||||
secrets:
|
||||
SLACK_WEBHOOK_URL:
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
environment: workflow_prod
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install requests
|
||||
|
||||
- name: Send Slack notification
|
||||
run: python macros/python/slack_alert.py
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
74
macros/python/slack_alert.py
Normal file
74
macros/python/slack_alert.py
Normal file
@ -0,0 +1,74 @@
|
||||
import requests
|
||||
import os
|
||||
import sys
|
||||
|
||||
def create_message():
|
||||
"""Creates a simple failure notification message with repo, workflow name, and URL"""
|
||||
|
||||
# Get GitHub environment variables
|
||||
repository = os.environ.get('GITHUB_REPOSITORY', 'Unknown repository')
|
||||
repo_name = repository.split('/')[-1] if '/' in repository else repository
|
||||
workflow_name = os.environ.get('GITHUB_WORKFLOW', 'Unknown workflow')
|
||||
run_id = os.environ.get('GITHUB_RUN_ID', '')
|
||||
server_url = os.environ.get('GITHUB_SERVER_URL', 'https://github.com')
|
||||
|
||||
# Build the workflow URL
|
||||
workflow_url = f"{server_url}/{repository}/actions/runs/{run_id}"
|
||||
|
||||
message_body = {
|
||||
"text": f"Failure in {repo_name}",
|
||||
"attachments": [
|
||||
{
|
||||
"color": "#f44336", # Red color for failures
|
||||
"fields": [
|
||||
{
|
||||
"title": "Repository",
|
||||
"value": repository,
|
||||
"short": True
|
||||
},
|
||||
{
|
||||
"title": "Workflow",
|
||||
"value": workflow_name,
|
||||
"short": True
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
{
|
||||
"type": "button",
|
||||
"text": "View Workflow Run",
|
||||
"style": "primary",
|
||||
"url": workflow_url
|
||||
}
|
||||
],
|
||||
"footer": "GitHub Actions"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
return message_body
|
||||
|
||||
def send_alert(webhook_url):
|
||||
"""Sends a failure notification to Slack"""
|
||||
|
||||
message = create_message()
|
||||
|
||||
try:
|
||||
response = requests.post(webhook_url, json=message)
|
||||
|
||||
if response.status_code == 200:
|
||||
print("Successfully sent Slack notification")
|
||||
else:
|
||||
print(f"Failed to send Slack notification: {response.status_code} {response.text}")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error sending Slack notification: {str(e)}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
webhook_url = os.environ.get("SLACK_WEBHOOK_URL")
|
||||
|
||||
if not webhook_url:
|
||||
print("ERROR: SLACK_WEBHOOK_URL environment variable is required")
|
||||
sys.exit(1)
|
||||
|
||||
send_alert(webhook_url)
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -52,7 +52,7 @@ base_labels AS (
|
||||
'CREATE',
|
||||
'CREATE2'
|
||||
)
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
AND to_address IS NOT NULL
|
||||
AND to_address NOT IN (
|
||||
SELECT
|
||||
|
||||
@ -20,7 +20,7 @@ WITH tokens AS (
|
||||
WHERE
|
||||
utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) :: FLOAT IS NOT NULL
|
||||
AND topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
),
|
||||
nfts AS (
|
||||
SELECT
|
||||
@ -35,7 +35,7 @@ nfts AS (
|
||||
WHERE
|
||||
utils.udf_hex_to_int(SUBSTR(DATA, 3, 64)) :: FLOAT IS NULL
|
||||
AND topics [0] :: STRING = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
)
|
||||
SELECT
|
||||
DISTINCT CURRENT_DATE :: TIMESTAMP AS system_created_at,
|
||||
|
||||
@ -1,148 +0,0 @@
|
||||
{{ config(
|
||||
materialized = 'incremental',
|
||||
merge_exclude_columns = ["inserted_timestamp"],
|
||||
unique_key = ['blockchain','test_timestamp'],
|
||||
full_refresh = false,
|
||||
tags = ['observability']
|
||||
) }}
|
||||
|
||||
WITH base AS (
|
||||
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'arbitrum' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'arbitrum_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'bsc' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'bsc_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'ethereum' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'ethereum_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'polygon' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'polygon_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'base' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'base_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'gnosis' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'gnosis_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'avalanche' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'avalanche_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'optimism' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'optimism_silver',
|
||||
'traces'
|
||||
) }}
|
||||
UNION
|
||||
SELECT
|
||||
MAX(block_timestamp) AS max_timestamp,
|
||||
'blast' AS chain,
|
||||
DATEDIFF(
|
||||
'minutes',
|
||||
max_timestamp,
|
||||
CURRENT_TIMESTAMP
|
||||
) AS delay
|
||||
FROM
|
||||
{{ source(
|
||||
'blast_silver',
|
||||
'traces'
|
||||
) }}
|
||||
)
|
||||
SELECT
|
||||
max_timestamp,
|
||||
chain AS blockchain,
|
||||
delay,
|
||||
'traces' AS test_table,
|
||||
CURRENT_TIMESTAMP AS test_timestamp,
|
||||
sysdate() as inserted_timestamp,
|
||||
sysdate() as modified_timestamp,
|
||||
{{ dbt_utils.generate_surrogate_key(['blockchain','test_timestamp']) }} AS data_delay_id,
|
||||
'{{ invocation_id }}' as _invocation_id
|
||||
FROM
|
||||
base
|
||||
@ -4,55 +4,6 @@
|
||||
) }}
|
||||
|
||||
{% set models = [
|
||||
('avalanche', source('avalanche_observ', 'blocks_completeness')),
|
||||
('avalanche', source('avalanche_observ', 'logs_completeness')),
|
||||
('avalanche', source('avalanche_observ', 'receipts_completeness')),
|
||||
('avalanche', source('avalanche_observ', 'traces_completeness')),
|
||||
('avalanche', source('avalanche_observ', 'transactions_completeness')),
|
||||
('ethereum', source('ethereum_observ', 'blocks_completeness')),
|
||||
('ethereum', source('ethereum_observ', 'logs_completeness')),
|
||||
('ethereum', source('ethereum_observ', 'receipts_completeness')),
|
||||
('ethereum', source('ethereum_observ', 'traces_completeness')),
|
||||
('ethereum', source('ethereum_observ', 'transactions_completeness')),
|
||||
('arbitrum', source('arbitrum_observ', 'blocks_completeness')),
|
||||
('arbitrum', source('arbitrum_observ', 'logs_completeness')),
|
||||
('arbitrum', source('arbitrum_observ', 'receipts_completeness')),
|
||||
('arbitrum', source('arbitrum_observ', 'traces_completeness')),
|
||||
('arbitrum', source('arbitrum_observ', 'transactions_completeness')),
|
||||
('bsc', source('bsc_observ', 'blocks_completeness')),
|
||||
('bsc', source('bsc_observ', 'logs_completeness')),
|
||||
('bsc', source('bsc_observ', 'receipts_completeness')),
|
||||
('bsc', source('bsc_observ', 'traces_completeness')),
|
||||
('bsc', source('bsc_observ', 'transactions_completeness')),
|
||||
('kaia', source('kaia_observ', 'blocks_completeness')),
|
||||
('kaia', source('kaia_observ', 'logs_completeness')),
|
||||
('kaia', source('kaia_observ', 'receipts_completeness')),
|
||||
('kaia', source('kaia_observ', 'traces_completeness')),
|
||||
('polygon', source('polygon_observ', 'blocks_completeness')),
|
||||
('polygon', source('polygon_observ', 'logs_completeness')),
|
||||
('polygon', source('polygon_observ', 'receipts_completeness')),
|
||||
('polygon', source('polygon_observ', 'traces_completeness')),
|
||||
('polygon', source('polygon_observ', 'transactions_completeness')),
|
||||
('optimism', source('optimism_observ', 'blocks_completeness')),
|
||||
('optimism', source('optimism_observ', 'logs_completeness')),
|
||||
('optimism', source('optimism_observ', 'receipts_completeness')),
|
||||
('optimism', source('optimism_observ', 'traces_completeness')),
|
||||
('optimism', source('optimism_observ', 'transactions_completeness')),
|
||||
('base', source('base_observ', 'blocks_completeness')),
|
||||
('base', source('base_observ', 'logs_completeness')),
|
||||
('base', source('base_observ', 'receipts_completeness')),
|
||||
('base', source('base_observ', 'traces_completeness')),
|
||||
('base', source('base_observ', 'transactions_completeness')),
|
||||
('gnosis', source('gnosis_observ', 'blocks_completeness')),
|
||||
('gnosis', source('gnosis_observ', 'logs_completeness')),
|
||||
('gnosis', source('gnosis_observ', 'receipts_completeness')),
|
||||
('gnosis', source('gnosis_observ', 'traces_completeness')),
|
||||
('gnosis', source('gnosis_observ', 'transactions_completeness')),
|
||||
('blast', source('blast_observ', 'blocks_completeness')),
|
||||
('blast', source('blast_observ', 'logs_completeness')),
|
||||
('blast', source('blast_observ', 'receipts_completeness')),
|
||||
('blast', source('blast_observ', 'traces_completeness')),
|
||||
('blast', source('blast_observ', 'transactions_completeness')),
|
||||
('cosmos', source('cosmos_observ', 'blocks_completeness')),
|
||||
('cosmos', source('cosmos_observ', 'transactions_completeness')),
|
||||
('osmosis', source('osmosis_observ', 'blocks_completeness')),
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -46,7 +46,7 @@ WITH t1 AS (
|
||||
WHERE
|
||||
origin_function_signature = '0x2e7ba6ef' -- Claim function
|
||||
AND event_name = 'Transfer'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
),
|
||||
t2 AS (
|
||||
SELECT
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -26,7 +26,7 @@ WITH pre_final AS (
|
||||
WHERE
|
||||
contract_address = LOWER('0x02101dfB77FDE026414827Fdc604ddAF224F0921')
|
||||
AND event_name = 'AccountCreated'
|
||||
AND tx_status = 'SUCCESS'
|
||||
AND tx_succeeded
|
||||
|
||||
{% if is_incremental() %}
|
||||
AND modified_timestamp > (
|
||||
|
||||
@ -495,15 +495,6 @@ sources:
|
||||
- name: events
|
||||
- name: decoded_metadata
|
||||
- name: mint_types
|
||||
- name: avalanche_observ
|
||||
database: avalanche
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: cosmos_observ
|
||||
database: cosmos
|
||||
schema: silver_observability
|
||||
@ -516,77 +507,6 @@ sources:
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: transactions_completeness
|
||||
- name: ethereum_observ
|
||||
database: ethereum
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: polygon_observ
|
||||
database: polygon
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: bsc_observ
|
||||
database: bsc
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: kaia_observ
|
||||
database: kaia
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: arbitrum_observ
|
||||
database: arbitrum
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: optimism_observ
|
||||
database: optimism
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: blast_observ
|
||||
database: blast
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: base_observ
|
||||
database: base
|
||||
schema: silver_observability
|
||||
tables:
|
||||
- name: blocks_completeness
|
||||
- name: logs_completeness
|
||||
- name: receipts_completeness
|
||||
- name: traces_completeness
|
||||
- name: transactions_completeness
|
||||
- name: base_silver
|
||||
database: base
|
||||
schema: silver
|
||||
|
||||
Loading…
Reference in New Issue
Block a user