diff --git a/.github/workflows/dbt.yml b/.github/workflows/dbt.yml new file mode 100644 index 0000000..418c1ff --- /dev/null +++ b/.github/workflows/dbt.yml @@ -0,0 +1,59 @@ +name: dbt +on: + workflow_call: + inputs: + warehouse: + required: false + type: string + environment: + required: true + type: string + command: + required: true + type: string + python_version: + required: false + type: string + default: "3.10" + +env: + DBT_PROFILES_DIR: ./ + + ACCOUNT: ${{ vars.ACCOUNT }} + ROLE: ${{ vars.ROLE }} + USER: ${{ vars.USER }} + PASSWORD: ${{ secrets.PASSWORD }} + REGION: ${{ vars.REGION }} + DATABASE: ${{ vars.DATABASE }} + WAREHOUSE: ${{ inputs.warehouse }} + SCHEMA: ${{ vars.SCHEMA }} + TARGET: ${{ vars.TARGET }} + +jobs: + dbt: + runs-on: ubuntu-latest + environment: + name: ${{ inputs.environment }} + env: + WAREHOUSE: ${{ inputs.warehouse || vars.WAREHOUSE }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + cache: "pip" + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + - name: Run DBT Jobs + run: | + ${{ inputs.command }} + - name: Store logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: logs-${{ inputs.environment }}-${{ github.run_number }}-${{ github.run_attempt }} + path: | + logs + target diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml new file mode 100644 index 0000000..ec1f55f --- /dev/null +++ b/.github/workflows/dbt_integration_test.yml @@ -0,0 +1,19 @@ +name: integration test +run-name: ${{ github.event.inputs.branch }} + +on: + workflow_dispatch: + + +concurrency: ${{ github.workflow }} + +jobs: + test: + name: livequery_integration_test + uses: ./.github/workflows/dbt.yml + secrets: inherit + + with: + warehouse: DBT_CLOUD + environment: ${{ github.ref == 'refs/heads/main' && 'prod' || 'stg' }} + command: dbt test -s 'livequery_models.deploy.core.*' diff --git a/.github/workflows/dbt_run_adhoc_with_alert.yml b/.github/workflows/dbt_run_adhoc_with_alert.yml new file mode 100644 index 0000000..590a2b7 --- /dev/null +++ b/.github/workflows/dbt_run_adhoc_with_alert.yml @@ -0,0 +1,88 @@ +name: dbt_run_adhoc_with_alert +run-name: ${{ inputs.dbt_command }} + +on: + workflow_dispatch: + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + - DBT_EMERGENCY + default: DBT + dbt_command: + type: string + description: 'DBT Run Command' + required: true + +env: + SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + DBT_VERSION: "${{ vars.DBT_VERSION }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + SCHEMA: "${{ vars.SCHEMA }}" + WAREHOUSE: "${{ inputs.warehouse }}" + ENVIRONMENT: "${{ inputs.environment }}" + DBT_COMMAND: "${{ inputs.dbt_command }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_${{ inputs.environment }} + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + id: dbt_run_command + run: | + start_time=$(date +%s) + ${{ inputs.dbt_command }} + end_time=$(date +%s) + elapsed_time=$(expr $end_time - $start_time) + echo "ELAPSED_TIME=$elapsed_time" >> $GITHUB_ENV + continue-on-error: true + + - name: Log run status + run: echo "DBT_RUN_STATUS=${{ steps.dbt_run_command.outcome }}" >> $GITHUB_ENV + + - name: Send Notification + run: | + python python/dbt_slack_notification.py + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: logs diff --git a/.github/workflows/dbt_run_history.yml b/.github/workflows/dbt_run_history.yml deleted file mode 100644 index 1e186a7..0000000 --- a/.github/workflows/dbt_run_history.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: dbt_run_history -run-name: dbt_run_history - -on: - workflow_dispatch: - schedule: - # Runs every 2 hours - # - cron: "0 */2 * * *" - # Runs every hour - - cron: "0 * * * *" - -env: - USE_VARS: "${{ vars.USE_VARS }}" - DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" - DBT_VERSION: "${{ vars.DBT_VERSION }}" - ACCOUNT: "${{ vars.ACCOUNT }}" - ROLE: "${{ vars.ROLE }}" - USER: "${{ vars.USER }}" - PASSWORD: "${{ secrets.PASSWORD }}" - REGION: "${{ vars.REGION }}" - DATABASE: "${{ vars.DATABASE }}" - WAREHOUSE: "${{ vars.WAREHOUSE }}" - SCHEMA: "${{ vars.SCHEMA }}" - -concurrency: - group: ${{ github.workflow }} - -jobs: - dbt: - runs-on: ubuntu-latest - environment: - name: workflow_prod - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: install dependencies - run: | - pip install -r requirements.txt - dbt deps - - - name: Run DBT Jobs - run: | - dbt run -s \ - 2+streamline__get_transactions_history_mainnet_18 \ - 2+streamline__get_transactions_history_mainnet_19 \ - 2+streamline__get_transaction_results_history_mainnet_14 \ - 2+streamline__get_transaction_results_history_mainnet_15 \ - 2+streamline__get_transaction_results_history_mainnet_16 \ - 2+streamline__get_batch_transaction_results_history_mainnet_17 \ - 2+streamline__get_batch_transaction_results_history_mainnet_18 \ - 2+streamline__get_batch_transaction_results_history_mainnet_19 \ - 2+streamline__get_batch_transaction_results_history_mainnet_22 \ - --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index f09da4a..90b6ce4 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -7,7 +7,9 @@ on: # 4x/hour schedule = Every 15 minutes (see https://crontab.guru) # - cron: "*/15 * * * *" # 3x/hour schedule = Every 20 minutes (see https://crontab.guru) - - cron: "*/20 * * * *" + # - cron: "*/20 * * * *" + # 2x/hour schedule = Every 30 minutes (see https://crontab.guru) + - cron: "*/30 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" @@ -46,7 +48,7 @@ jobs: run: > dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; dbt seed; - dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" - name: Store logs uses: actions/upload-artifact@v3 diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index 7ba4681..b9b6b42 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -1,5 +1,5 @@ -name: dbt_run_scheduled -run-name: dbt_run_scheduled +name: dbt_run_scheduled_non_core +run-name: dbt_run_scheduled_non_core on: workflow_dispatch: @@ -44,7 +44,7 @@ jobs: run: > dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; dbt seed; - dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + dbt run -s tag:scheduled_non_core - name: Store logs uses: actions/upload-artifact@v3 diff --git a/.github/workflows/dbt_run_streamline_blocks_testnet.yml b/.github/workflows/dbt_run_streamline_blocks_testnet.yml new file mode 100644 index 0000000..2c97938 --- /dev/null +++ b/.github/workflows/dbt_run_streamline_blocks_testnet.yml @@ -0,0 +1,46 @@ +name: dbt_run_streamline_blocks_testnet +run-name: dbt_run_streamline_blocks_testnet + +on: + workflow_dispatch: + schedule: + # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) + - cron: "7,27,47 10,20 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Realtime + run: | + dbt run -s 2+streamline__get_testnet_blocks_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_streamline_collections_testnet.yml b/.github/workflows/dbt_run_streamline_collections_testnet.yml new file mode 100644 index 0000000..bc166fd --- /dev/null +++ b/.github/workflows/dbt_run_streamline_collections_testnet.yml @@ -0,0 +1,46 @@ +name: dbt_run_streamline_collections_testnet +run-name: dbt_run_streamline_collections_testnet + +on: + workflow_dispatch: + schedule: + # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) + - cron: "11,31,51 10,20 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Realtime + run: | + dbt run -s 2+streamline__get_testnet_collections_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_streamline_transaction_results_testnet.yml b/.github/workflows/dbt_run_streamline_transaction_results_testnet.yml new file mode 100644 index 0000000..95db09a --- /dev/null +++ b/.github/workflows/dbt_run_streamline_transaction_results_testnet.yml @@ -0,0 +1,46 @@ +name: dbt_run_streamline_transaction_results_testnet +run-name: dbt_run_streamline_transaction_results_testnet + +on: + workflow_dispatch: + schedule: + # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) + - cron: "15,35,55 10,20 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Realtime + run: | + dbt run -s 1+streamline__get_testnet_transaction_results_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_streamline_transactions_testnet.yml b/.github/workflows/dbt_run_streamline_transactions_testnet.yml new file mode 100644 index 0000000..37ab93e --- /dev/null +++ b/.github/workflows/dbt_run_streamline_transactions_testnet.yml @@ -0,0 +1,46 @@ +name: dbt_run_streamline_transactions_testnet +run-name: dbt_run_streamline_transactions_testnet + +on: + workflow_dispatch: + schedule: + # 1x/hour schedule = At hour 8, 21, 36, 51 every day (see https://crontab.guru) + - cron: "15,35,55 10,20 * * *" + +env: + USE_VARS: "${{ vars.USE_VARS }}" + DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" + ACCOUNT: "${{ vars.ACCOUNT }}" + ROLE: "${{ vars.ROLE }}" + USER: "${{ vars.USER }}" + PASSWORD: "${{ secrets.PASSWORD }}" + REGION: "${{ vars.REGION }}" + DATABASE: "${{ vars.DATABASE }}" + WAREHOUSE: "${{ vars.WAREHOUSE }}" + SCHEMA: "${{ vars.SCHEMA }}" + +concurrency: + group: ${{ github.workflow }} + +jobs: + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Realtime + run: | + dbt run -s 1+streamline__get_testnet_transactions_realtime --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/dbt_project.yml b/dbt_project.yml index e03185c..96a07c4 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: "flow_models" -version: "1.7.0" +version: "1.8.0" config-version: 2 require-dbt-version: ">=1.7.0" @@ -64,8 +64,9 @@ vars: STREAMLINE_RUN_HISTORY: False DROP_UDFS_AND_SPS: False REST_API_PREFIX_PROD: quxfxtl934.execute-api.us-east-1.amazonaws.com/prod/ - REST_API_PREFIX_DEV: ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/ + REST_API_PREFIX_DEV: sicl8dvvv9.execute-api.us-east-1.amazonaws.com/dev/ STREAMLINE_START_BLOCK: 55114467 + LOAD_BACKFILL_VERSION: CANDIDATE_07 dispatch: - macro_namespace: dbt @@ -73,6 +74,6 @@ dispatch: - flow-models - dbt_snowflake_query_tags - dbt -# query-comment: -# comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' -# append: true # Snowflake removes prefixed comments. +query-comment: + comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' + append: true # Snowflake removes prefixed comments. diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql index 43cdf67..2644b0f 100644 --- a/macros/create_udfs.sql +++ b/macros/create_udfs.sql @@ -1,12 +1,12 @@ {% macro create_udfs() %} {% if var("UPDATE_UDFS_AND_SPS") %} - {% if target.database != "FLOW_COMMUNITY_DEV" %} {% set sql %} {{ create_udtf_get_base_table( schema = "streamline" ) }} {{ create_udf_get_chainhead() }} + {{ create_udf_get_chainhead_testnet() }} {{ create_udf_bulk_grpc() }} {{ run_create_udf_array_disjunctive_union() }} @@ -15,6 +15,5 @@ {% endset %} {% do run_query(sql) %} {{- fsc_utils.create_udfs() -}} - {% endif %} {% endif %} {% endmacro %} diff --git a/macros/streamline/api_integrations.sql b/macros/streamline/api_integrations.sql index 5022425..376eb7e 100644 --- a/macros/streamline/api_integrations.sql +++ b/macros/streamline/api_integrations.sql @@ -12,14 +12,23 @@ CREATE api integration IF NOT EXISTS aws_flow_api_prod_us_east_2 api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/flow-api-prod-rolesnowflakeudfsAF733095-F6SPYWFGQX9Z' api_allowed_prefixes = ( 'https://78rpbojpue.execute-api.us-east-2.amazonaws.com/prod/' ) enabled = TRUE; + {% endset %} {% do run_query(sql) %} + {% set sql %} + CREATE api integration IF NOT EXISTS aws_flow_evm_api_prod api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::924682671219:role/flow-api-prod-rolesnowflakeudfsAF733095-RmrgKIWbzoFL' api_allowed_prefixes = ( + 'https://rajpkbgko9.execute-api.us-east-1.amazonaws.com/prod/' + ) enabled = TRUE; + + {% endset %} + {% do run_query(sql) %} + {% elif target.name == "dev" %} {{ log("Generating api integration for target:" ~ target.name, info=True) }} {% set sql %} - CREATE api integration IF NOT EXISTS aws_flow_api_dev_2 api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/flow-api-dev-rolesnowflakeudfsAF733095-1IP9GV997U5RM' api_allowed_prefixes = ( - 'https://ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/' + CREATE api integration IF NOT EXISTS aws_flow_api_dev_2 api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::490041342817:role/flow-api-dev-rolesnowflakeudfsAF733095-i1JsMNTpSzX0' api_allowed_prefixes = ( + 'https://sicl8dvvv9.execute-api.us-east-1.amazonaws.com/dev/' ) enabled = TRUE; {% endset %} {% do run_query(sql) %} @@ -30,6 +39,14 @@ ) enabled = TRUE; {% endset %} {% do run_query(sql) %} + + {% set sql %} + CREATE api integration IF NOT EXISTS aws_flow_evm_api_dev api_provider = aws_api_gateway api_aws_role_arn = 'arn:aws:iam::704693948482:role/flow-api-stg-rolesnowflakeudfsAF733095-tPEdygwPC6IV' api_allowed_prefixes = ( + 'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/' + ) enabled = TRUE; + {% endset %} + {% do run_query(sql) %} + {% elif target.name == "sbx" %} {{ log("Generating api integration for target:" ~ target.name, info=True) }} {% set sql %} diff --git a/macros/streamline/get_base_table_udft.sql b/macros/streamline/get_base_table_udft.sql index 911f251..b54b3d3 100644 --- a/macros/streamline/get_base_table_udft.sql +++ b/macros/streamline/get_base_table_udft.sql @@ -10,7 +10,7 @@ $$ seq4() ) as id from - table(generator(rowcount => 100000000)) -- July 2023 Flow Chain head is at 57M + table(generator(rowcount => 1000000000)) ) select id as height diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index 84d77ae..88116ec 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -6,13 +6,28 @@ {% if target.name == "prod" %} aws_flow_api_prod AS 'https://quxfxtl934.execute-api.us-east-1.amazonaws.com/prod/get_chainhead' {% elif target.name == "dev" %} - aws_flow_api_dev_2 AS 'https://ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/get_chainhead' + aws_flow_api_dev_2 AS 'https://sicl8dvvv9.execute-api.us-east-1.amazonaws.com/dev/get_chainhead' {% elif target.name == "sbx" %} {{ log("Creating sbx get_chainhead", info=True) }} aws_flow_api_sbx AS 'https://bc5ejedoq8.execute-api.us-east-1.amazonaws.com/sbx/get_chainhead' {%- endif %}; {% endmacro %} +{% macro create_udf_get_chainhead_testnet() %} + {{ log("Creating udf get_chainhead_testnet for target:" ~ target.name ~ ", schema: " ~ target.schema, info=True) }} + {{ log("role:" ~ target.role ~ ", user:" ~ target.user, info=True) }} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_get_chainhead_testnet() returns variant api_integration = + {% if target.name == "prod" %} + aws_flow_api_prod_us_east_2 AS 'https://78rpbojpue.execute-api.us-east-2.amazonaws.com/prod/get_chainhead_testnet' + {% elif target.name == "dev" %} + aws_flow_api_dev_2 AS 'https://sicl8dvvv9.execute-api.us-east-1.amazonaws.com/dev/get_chainhead_testnet' + {% elif target.name == "sbx" %} + {{ log("Creating sbx get_chainhead_testnet", info=True) }} + aws_flow_api_sbx AS 'https://bc5ejedoq8.execute-api.us-east-1.amazonaws.com/sbx/get_chainhead_testnet' + {%- endif %}; +{% endmacro %} + {% macro create_udf_bulk_grpc() %} {{ log("Creating udf udf_bulk_grpc for target:" ~ target.name ~ ", schema: " ~ target.schema, info=True) }} {{ log("role:" ~ target.role ~ ", user:" ~ target.user, info=True) }} @@ -21,7 +36,7 @@ {% if target.name == "prod" %} aws_flow_api_prod AS 'https://quxfxtl934.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_grpc' {% elif target.name == "dev" %} - aws_flow_api_dev_2 AS 'https://ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/udf_bulk_grpc' + aws_flow_api_dev_2 AS 'https://sicl8dvvv9.execute-api.us-east-1.amazonaws.com/dev/udf_bulk_grpc' {% elif target.name == "sbx" %} {{ log("Creating sbx udf_bulk_grpc", info=True) }} aws_flow_api_sbx AS 'https://bc5ejedoq8.execute-api.us-east-1.amazonaws.com/sbx/udf_bulk_grpc' @@ -58,10 +73,32 @@ {% if target.name == "prod" %} aws_flow_api_prod AS 'https://quxfxtl934.execute-api.us-east-1.amazonaws.com/prod/udf_api' {% elif target.name == "dev" %} - aws_flow_api_dev_2 AS 'https://ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/udf_api' + aws_flow_api_dev_2 AS 'https://sicl8dvvv9.execute-api.us-east-1.amazonaws.com/dev/udf_api' {% elif target.name == "sbx" %} {{ log("Creating sbx udf_api", info=True) }} aws_flow_api_sbx AS 'https://bc5ejedoq8.execute-api.us-east-1.amazonaws.com/sbx/udf_api' {%- endif %}; {% endmacro %} +{% macro create_udf_bulk_rest_api_v2() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_rest_api_v2( + json OBJECT + ) returns ARRAY api_integration = + {% if target.name == "prod" %} + aws_flow_evm_api_prod AS 'https://rajpkbgko9.execute-api.us-east-1.amazonaws.com/prod/udf_bulk_rest_api' + {% else %} + aws_flow_evm_api_dev AS 'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/udf_bulk_rest_api' + {%- endif %}; +{% endmacro %} + +{% macro create_udf_bulk_decode_logs() %} + CREATE + OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_decode_logs( + json OBJECT + ) returns ARRAY api_integration = {% if target.name == "prod" %} + aws_flow_evm_api_prod AS 'https://rajpkbgko9.execute-api.us-east-1.amazonaws.com/prod/bulk_decode_logs' + {% else %} + aws_flow_evm_api_dev AS'https://pfv9lhg3kg.execute-api.us-east-1.amazonaws.com/stg/bulk_decode_logs' + {%- endif %}; +{% endmacro %} \ No newline at end of file diff --git a/models/bronze/bronze__labels.sql b/models/bronze/bronze__labels.sql index db0e93a..6cb414f 100644 --- a/models/bronze/bronze__labels.sql +++ b/models/bronze/bronze__labels.sql @@ -19,7 +19,7 @@ WITH labels AS ( labels_combined_id FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'labels_combined' ) }} WHERE diff --git a/models/bronze/bronze__prices.sql b/models/bronze/bronze__prices.sql deleted file mode 100644 index 3a0b9e6..0000000 --- a/models/bronze/bronze__prices.sql +++ /dev/null @@ -1,41 +0,0 @@ -{{ config( - materialized = 'view', - enabled = False -) }} - -WITH token_prices AS ( - - SELECT - * - FROM - {{ source( - 'silver', - 'prices_v2' - ) }} - WHERE - asset_id IN ( - '4558', - -- Flow - '6993', - -- Revv - '8075', - -- Rally - '12182', - -- Blocto Token - '15139', - -- Starly - '15194', - -- Sportium - 'flow', - 'rally-2', - 'revv', - 'sportium', - 'starly', - 'blocto-token' - ) - AND provider IS NOT NULL -) -SELECT - * -FROM - token_prices diff --git a/models/bronze/bronze__prices_v2.sql b/models/bronze/bronze__prices_v2.sql deleted file mode 100644 index a6cb781..0000000 --- a/models/bronze/bronze__prices_v2.sql +++ /dev/null @@ -1,82 +0,0 @@ -{{ config( - materialized = 'view', - tags = ['scheduled'] -) }} - -WITH coingecko AS ( - - SELECT - 'coingecko' AS provider, - id :: STRING AS id, - recorded_hour, - OPEN, - high, - low, - CLOSE, - _inserted_timestamp - FROM - {{ source( - 'crosschain_silver', - 'hourly_prices_coin_gecko' - ) }} -), -coinmarketcap AS ( - SELECT - 'coinmarketcap' AS provider, - id :: STRING AS id, - recorded_hour, - OPEN, - high, - low, - CLOSE, - _inserted_timestamp - FROM - {{ source( - 'crosschain_silver', - 'hourly_prices_coin_market_cap' - ) }} -), -token_prices AS ( - SELECT - * - FROM - coingecko - UNION ALL - SELECT - * - FROM - coinmarketcap -) -SELECT - * -FROM - token_prices -WHERE - -- numeric ids are cmc, alpha are coingecko - id IN ( - '4558', - -- Flow - '6993', - -- Revv - '8075', - -- Rally - '12182', - -- Blocto Token - '15139', - -- Starly - '15194', - -- Sportium - 'flow', - 'rally-2', - 'revv', - 'sportium', - 'starly', - 'blocto-token' - ) - AND provider IS NOT NULL -- tokens on increment that are not on either proider: - -- my - -- ozone - -- sdm - -- stFLOVATAR - -- thul - -- ce tokens diff --git a/models/bronze/prices/bronze__complete_native_asset_metadata.sql b/models/bronze/prices/bronze__complete_native_asset_metadata.sql new file mode 100644 index 0000000..835d103 --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_asset_metadata.sql @@ -0,0 +1,26 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_asset_metadata' + ) }} +WHERE + blockchain = 'flow' + AND symbol = 'FLOW' diff --git a/models/bronze/prices/bronze__complete_native_prices.sql b/models/bronze/prices/bronze__complete_native_prices.sql new file mode 100644 index 0000000..c330bc1 --- /dev/null +++ b/models/bronze/prices/bronze__complete_native_prices.sql @@ -0,0 +1,29 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_native_prices' + ) }} +WHERE + blockchain = 'flow' + AND symbol = 'FLOW' diff --git a/models/bronze/prices/bronze__complete_provider_asset_metadata.sql b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..1b4035e --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_asset_metadata.sql @@ -0,0 +1,25 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_asset_metadata' + ) }} +WHERE + platform = 'flow' \ No newline at end of file diff --git a/models/bronze/prices/bronze__complete_provider_prices.sql b/models/bronze/prices/bronze__complete_provider_prices.sql new file mode 100644 index 0000000..82d0627 --- /dev/null +++ b/models/bronze/prices/bronze__complete_provider_prices.sql @@ -0,0 +1,24 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_provider_prices' + ) }} +-- prices for all ids diff --git a/models/bronze/prices/bronze__complete_token_asset_metadata.sql b/models/bronze/prices/bronze__complete_token_asset_metadata.sql new file mode 100644 index 0000000..14b1aba --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_asset_metadata.sql @@ -0,0 +1,28 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_asset_metadata' + ) }} +WHERE + blockchain = 'flow' diff --git a/models/bronze/prices/bronze__complete_token_prices.sql b/models/bronze/prices/bronze__complete_token_prices.sql new file mode 100644 index 0000000..a3262c6 --- /dev/null +++ b/models/bronze/prices/bronze__complete_token_prices.sql @@ -0,0 +1,31 @@ +{{ config ( + materialized = 'view' +) }} + +SELECT + HOUR, + token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ source( + 'silver_crosschain', + 'complete_token_prices' + ) }} +WHERE + blockchain = 'flow' diff --git a/models/bronze/streamline/history/README.md b/models/bronze/streamline/history/README.md new file mode 100644 index 0000000..7f0f50e --- /dev/null +++ b/models/bronze/streamline/history/README.md @@ -0,0 +1,54 @@ +# Backfill + +Bronze backfill models have been parametrized to load one network version at a time, as each set of NVs and method responses is a separate bucket and external table. + +Run either an individual model type (blocks, collections, transactions, transaction_results) or all 4 at once with `tag:streamline_load`. + +```shell +dbt run -s 1+tag:streamline_load --vars '{"LOAD_BACKFILL": True, "LOAD_BACKFILL_VERSION": ""}' +``` + +## Valid Network Versions + - CANDIDATE_07 + - CANDIDATE_08 + - CANDIDATE_09 + - MAINNET_01 + - MAINNET_02 + - MAINNET_03 + - MAINNET_04 + - MAINNET_05 + - MAINNET_06 + - MAINNET_07 + - MAINNET_08 + - MAINNET_09 + - MAINNET_10 + - MAINNET_11 + - MAINNET_12 + - MAINNET_13 + - MAINNET_14 + - MAINNET_15 + - MAINNET_16 + - MAINNET_17 + - MAINNET_18 + - MAINNET_19 + - MAINNET_20 + - MAINNET_21 + - MAINNET_22 + +## View Types +Views with the word `complete` in the name are used in the complete history models at `models/silver/streamline/core/complete`. These use a macro to scan multiple external tables in one call, and feed the streamline backfill process. + +The views `bronze__streamline__history` query just one network version based on the `LOAD_BACKFILL_VERSION` argument passed at runtime. No default is set for this variable so execution fails if it is forgottten. + +## Running Streamline Backfill +If a a network version requires more backfill due to missing blocks or transactions (at present, there are 5800 missing transaction results), run the following command as the workflow dbt_run_history has been deleted. +```shell +dbt run -s 2+streamline__get__history_ --vars '{"STREAMLINE_INVOKE_STREAMS": True}' +``` + +i.e. +```shell +dbt run -s \ +2+streamline__get_transaction_results_history_mainnet_22 \ +--vars '{"STREAMLINE_INVOKE_STREAMS": True}' +``` diff --git a/models/bronze/streamline/history/bronze__streamline_blocks_history.sql b/models/bronze/streamline/history/bronze__streamline_blocks_history.sql new file mode 100644 index 0000000..3ed5294 --- /dev/null +++ b/models/bronze/streamline/history/bronze__streamline_blocks_history.sql @@ -0,0 +1,36 @@ +{{ config ( + materialized = 'ephemeral' +) }} + +{% set history_model = "BLOCKS_" ~ var('LOAD_BACKFILL_VERSION') %} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", history_model ) }}' + ) + ) A +) +SELECT + block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source("bronze_streamline", history_model ) }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id diff --git a/models/bronze/streamline/history/bronze__streamline_collections_history.sql b/models/bronze/streamline/history/bronze__streamline_collections_history.sql new file mode 100644 index 0000000..bbef965 --- /dev/null +++ b/models/bronze/streamline/history/bronze__streamline_collections_history.sql @@ -0,0 +1,37 @@ +{{ config ( + materialized = 'ephemeral' +) }} + +{% set history_model = "COLLECTIONS_" ~ var('LOAD_BACKFILL_VERSION') %} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", history_model ) }}' + ) + ) A +) +SELECT + block_number, + id, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source("bronze_streamline", history_model ) }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql b/models/bronze/streamline/history/bronze__streamline_complete_blocks_history.sql similarity index 100% rename from models/silver/streamline/bronze/core/history/bronze__streamline_blocks_history.sql rename to models/bronze/streamline/history/bronze__streamline_complete_blocks_history.sql diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql b/models/bronze/streamline/history/bronze__streamline_complete_collections_history.sql similarity index 100% rename from models/silver/streamline/bronze/core/history/bronze__streamline_collections_history.sql rename to models/bronze/streamline/history/bronze__streamline_complete_collections_history.sql diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql b/models/bronze/streamline/history/bronze__streamline_complete_transaction_results_history.sql similarity index 100% rename from models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql rename to models/bronze/streamline/history/bronze__streamline_complete_transaction_results_history.sql diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql b/models/bronze/streamline/history/bronze__streamline_complete_transactions_history.sql similarity index 100% rename from models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql rename to models/bronze/streamline/history/bronze__streamline_complete_transactions_history.sql diff --git a/models/bronze/streamline/history/bronze__streamline_transaction_results_history.sql b/models/bronze/streamline/history/bronze__streamline_transaction_results_history.sql new file mode 100644 index 0000000..97a0e77 --- /dev/null +++ b/models/bronze/streamline/history/bronze__streamline_transaction_results_history.sql @@ -0,0 +1,37 @@ +{{ config ( + materialized = 'ephemeral' +) }} + +{% set history_model = "TRANSACTION_RESULTS_" ~ var('LOAD_BACKFILL_VERSION') %} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", history_model ) }}' + ) + ) A +) +SELECT + block_number, + id, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source("bronze_streamline", history_model ) }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id diff --git a/models/bronze/streamline/history/bronze__streamline_transactions_history.sql b/models/bronze/streamline/history/bronze__streamline_transactions_history.sql new file mode 100644 index 0000000..5084d26 --- /dev/null +++ b/models/bronze/streamline/history/bronze__streamline_transactions_history.sql @@ -0,0 +1,37 @@ +{{ config ( + materialized = 'ephemeral' +) }} + +{% set history_model = "TRANSACTIONS_" ~ var('LOAD_BACKFILL_VERSION') %} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", history_model ) }}' + ) + ) A +) +SELECT + block_number, + id, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source("bronze_streamline", history_model ) }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id diff --git a/models/bronze/streamline/realtime/README.md b/models/bronze/streamline/realtime/README.md new file mode 100644 index 0000000..e958584 --- /dev/null +++ b/models/bronze/streamline/realtime/README.md @@ -0,0 +1,3 @@ +# Streamline Realtime Models + +Flow migrated to Streamline with Mainnet-23. These views point to the external tables from there forward. diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_blocks.sql b/models/bronze/streamline/realtime/bronze__streamline_blocks.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_blocks.sql rename to models/bronze/streamline/realtime/bronze__streamline_blocks.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_collections.sql b/models/bronze/streamline/realtime/bronze__streamline_collections.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_collections.sql rename to models/bronze/streamline/realtime/bronze__streamline_collections.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_blocks.sql b/models/bronze/streamline/realtime/bronze__streamline_fr_blocks.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_blocks.sql rename to models/bronze/streamline/realtime/bronze__streamline_fr_blocks.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_collections.sql b/models/bronze/streamline/realtime/bronze__streamline_fr_collections.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_collections.sql rename to models/bronze/streamline/realtime/bronze__streamline_fr_collections.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transaction_results.sql b/models/bronze/streamline/realtime/bronze__streamline_fr_transaction_results.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transaction_results.sql rename to models/bronze/streamline/realtime/bronze__streamline_fr_transaction_results.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transactions.sql b/models/bronze/streamline/realtime/bronze__streamline_fr_transactions.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_fr_transactions.sql rename to models/bronze/streamline/realtime/bronze__streamline_fr_transactions.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_transaction_results.sql b/models/bronze/streamline/realtime/bronze__streamline_transaction_results.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_transaction_results.sql rename to models/bronze/streamline/realtime/bronze__streamline_transaction_results.sql diff --git a/models/silver/streamline/bronze/core/realtime/bronze__streamline_transactions.sql b/models/bronze/streamline/realtime/bronze__streamline_transactions.sql similarity index 100% rename from models/silver/streamline/bronze/core/realtime/bronze__streamline_transactions.sql rename to models/bronze/streamline/realtime/bronze__streamline_transactions.sql diff --git a/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_blocks.sql b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_blocks.sql new file mode 100644 index 0000000..3a15aca --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_blocks.sql @@ -0,0 +1,34 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + SELECT + registered_on AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_files( + table_name => '{{ source( "bronze_streamline", "testnet_blocks") }}' + ) + ) A +) +SELECT + block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE +FROM + {{ source("bronze_streamline", "testnet_blocks") }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id +WHERE + b._partition_by_block_id = s._partition_by_block_id diff --git a/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_collections.sql b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_collections.sql new file mode 100644 index 0000000..e192cab --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_collections.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-2:] | join('_') %} +{{ streamline_external_table_FR_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_transaction_results.sql b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_transaction_results.sql new file mode 100644 index 0000000..7fbacfa --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_transaction_results.sql @@ -0,0 +1,13 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-3:] | join('_') %} +{{ streamline_external_table_FR_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} + + diff --git a/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_transactions.sql b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_transactions.sql new file mode 100644 index 0000000..e192cab --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_fr_testnet_transactions.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-2:] | join('_') %} +{{ streamline_external_table_FR_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/testnet/bronze__streamline_testnet_blocks.sql b/models/bronze/streamline/testnet/bronze__streamline_testnet_blocks.sql new file mode 100644 index 0000000..24c26fa --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_testnet_blocks.sql @@ -0,0 +1,35 @@ +{{ config ( + materialized = 'view' +) }} + +WITH meta AS ( + SELECT + last_modified AS _inserted_timestamp, + file_name, + CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER) AS _partition_by_block_id + FROM + TABLE( + information_schema.external_table_file_registration_history( + start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + table_name => '{{ source( "bronze_streamline", "testnet_blocks") }}') + ) A + ) + SELECT + block_number, + DATA, + _inserted_timestamp, + MD5( + CAST( + COALESCE(CAST(block_number AS text), '' :: STRING) AS text + ) + ) AS _fsc_id, + s._partition_by_block_id, + s.value AS VALUE + FROM + {{ source("bronze_streamline","testnet_blocks") }} s + JOIN meta b + ON b.file_name = metadata$filename + AND b._partition_by_block_id = s._partition_by_block_id + WHERE + b._partition_by_block_id = s._partition_by_block_id + diff --git a/models/bronze/streamline/testnet/bronze__streamline_testnet_collections.sql b/models/bronze/streamline/testnet/bronze__streamline_testnet_collections.sql new file mode 100644 index 0000000..545031e --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_testnet_collections.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-2:] | join('_') %} +{{ streamline_external_table_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/testnet/bronze__streamline_testnet_transaction_results.sql b/models/bronze/streamline/testnet/bronze__streamline_testnet_transaction_results.sql new file mode 100644 index 0000000..bd31738 --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_testnet_transaction_results.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-3:] | join('_') %} +{{ streamline_external_table_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/bronze/streamline/testnet/bronze__streamline_testnet_transactions.sql b/models/bronze/streamline/testnet/bronze__streamline_testnet_transactions.sql new file mode 100644 index 0000000..545031e --- /dev/null +++ b/models/bronze/streamline/testnet/bronze__streamline_testnet_transactions.sql @@ -0,0 +1,11 @@ +{{ config ( + materialized = 'view' +) }} + +{% set model = this.identifier.split("_")[-2:] | join('_') %} +{{ streamline_external_table_query( + model, + partition_function = "CAST(SPLIT_PART(SPLIT_PART(file_name, '/', 3), '_', 1) AS INTEGER)", + partition_name = "_partition_by_block_id", + unique_key = "id" +) }} diff --git a/models/descriptions/amount_fee_usd.md b/models/descriptions/amount_fee_usd.md new file mode 100644 index 0000000..b165a9c --- /dev/null +++ b/models/descriptions/amount_fee_usd.md @@ -0,0 +1,5 @@ +{% docs amount_fee_usd %} + +Amount denominated in USD, where pricing data is available. + +{% enddocs %} diff --git a/models/descriptions/amount_usd.md b/models/descriptions/amount_usd.md new file mode 100644 index 0000000..020d640 --- /dev/null +++ b/models/descriptions/amount_usd.md @@ -0,0 +1,5 @@ +{% docs amount_usd %} + +Amount denominated in USD, where pricing data is available. + +{% enddocs %} diff --git a/models/descriptions/arguments.md b/models/descriptions/arguments.md index 7ec1652..3f78ed8 100644 --- a/models/descriptions/arguments.md +++ b/models/descriptions/arguments.md @@ -1,5 +1,5 @@ {% docs arguments %} -The arguments included in the transaction body. +The arguments passed into the Cadence script when the transaction was submitted. {% enddocs %} diff --git a/models/descriptions/prices/prices.md b/models/descriptions/prices/prices.md new file mode 100644 index 0000000..7c7f6a1 --- /dev/null +++ b/models/descriptions/prices/prices.md @@ -0,0 +1,137 @@ +{% docs prices_dim_asset_metadata_table_doc %} + +A comprehensive dimensional table holding asset metadata and other relevant details pertaining to each id, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this dimensional model. If you are interested in using a curated data set instead, please utilize ez_asset_metadata. + +{% enddocs %} + +{% docs prices_ez_asset_metadata_table_doc %} + +A convenience table holding prioritized asset metadata and other relevant details pertaining to each token_address and native asset. This data set is highly curated and contains metadata for one unique asset per blockchain. + +{% enddocs %} + +{% docs prices_fact_prices_ohlc_hourly_table_doc %} + +A comprehensive fact table holding id and provider specific open, high, low, close hourly prices, from multiple providers. This data set includes raw, non-transformed data coming directly from the provider APIs and rows are not intended to be unique. As a result, there may be data quality issues persisting in the APIs that flow through to this fact based model. If you are interested in using a curated data set instead, please utilize ez_prices_hourly. + +{% enddocs %} + +{% docs prices_ez_prices_hourly_table_doc %} + +A convenience table for determining token prices by address and blockchain, and native asset prices by symbol and blockchain. This data set is highly curated and contains metadata for one price per hour per unique asset and blockchain. + +{% enddocs %} + +{% docs prices_provider %} + +The provider or source of the data. + +{% enddocs %} + +{% docs prices_asset_id %} + +The unique identifier representing the asset. + +{% enddocs %} + +{% docs prices_name %} + +The name of asset. + +{% enddocs %} + +{% docs prices_symbol %} + +The symbol of asset. + +{% enddocs %} + +{% docs prices_token_address %} + +The specific address representing the asset on a specific platform. This will be NULL if referring to a native asset. + +{% enddocs %} + +{% docs prices_blockchain %} + +The Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_blockchain_id %} + +The unique identifier of the Blockchain, Network, or Platform for this asset. + +{% enddocs %} + +{% docs prices_decimals %} + +The number of decimals for the asset. May be NULL. + +{% enddocs %} + +{% docs prices_is_native %} + +A flag indicating assets native to the respective blockchain. + +{% enddocs %} + +{% docs prices_is_deprecated %} + +A flag indicating if the asset is deprecated or no longer supported by the provider. + +{% enddocs %} + +{% docs prices_id_deprecation %} + +Deprecating soon! Please use the `asset_id` column instead. + +{% enddocs %} + +{% docs prices_decimals_deprecation %} + +Deprecating soon! Please use the decimals column in `ez_asset_metadata` or join in `dim_contracts` instead. + +{% enddocs %} + +{% docs prices_hour %} + +Hour that the price was recorded at. + +{% enddocs %} + +{% docs prices_price %} + +Closing price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_is_imputed %} + +A flag indicating if the price was imputed, or derived, from the last arriving record. This is generally used for tokens with low-liquidity or inconsistent reporting. + +{% enddocs %} + +{% docs prices_open %} + +Opening price of the recorded hour in USD. + +{% enddocs %} + +{% docs prices_high %} + +Highest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_low %} + +Lowest price of the recorded hour in USD + +{% enddocs %} + +{% docs prices_close %} + +Closing price of the recorded hour in USD + +{% enddocs %} \ No newline at end of file diff --git a/models/descriptions/token_symbol.md b/models/descriptions/token_symbol.md new file mode 100644 index 0000000..023d5c8 --- /dev/null +++ b/models/descriptions/token_symbol.md @@ -0,0 +1,5 @@ +{% docs token_symbol %} + +Abbreviated symbol for the asset. + +{% enddocs %} diff --git a/models/gold/core/core__dim_contract_labels.sql b/models/gold/core/core__dim_contract_labels.sql index 6fe0c36..2f9c4be 100644 --- a/models/gold/core/core__dim_contract_labels.sql +++ b/models/gold/core/core__dim_contract_labels.sql @@ -3,61 +3,16 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS event_contract_id, - event_contract, - contract_name, - account_address, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__contract_labels') }} -), -streamline AS ( - SELECT - event_contract_id, - event_contract, - contract_name, - account_address, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__contract_labels_s') }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT + event_contract_id, event_contract, contract_name, account_address, + inserted_timestamp, + modified_timestamp, COALESCE ( event_contract_id, {{ dbt_utils.generate_surrogate_key(['event_contract']) }} - ) AS dim_contract_labels_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + ) AS dim_contract_labels_id FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY event_contract - ORDER BY - _inserted_timestamp DESC - ) = 1 + {{ ref('silver__contract_labels_s') }} diff --git a/models/gold/core/core__ez_token_transfers.sql b/models/gold/core/core__ez_token_transfers.sql index cdcb304..dc46fda 100644 --- a/models/gold/core/core__ez_token_transfers.sql +++ b/models/gold/core/core__ez_token_transfers.sql @@ -3,68 +3,6 @@ tags = ['ez', 'scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS token_transfers_id, - block_height, - block_timestamp, - tx_id, - sender, - recipient, - token_contract, - amount, - tx_succeeded, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__token_transfers') }} - WHERE - token_contract NOT IN ( - 'A.c38aea683c0c4d38.ZelosAccountingToken', - 'A.f1b97c06745f37ad.SwapPair' - ) - AND block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - token_transfers_id, - block_height, - block_timestamp, - tx_id, - sender, - recipient, - token_contract, - amount, - tx_succeeded, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__token_transfers_s') }} - WHERE - token_contract NOT IN ( - 'A.c38aea683c0c4d38.ZelosAccountingToken', - 'A.f1b97c06745f37ad.SwapPair' - ) - AND block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - streamline - UNION ALL - SELECT - * - FROM - chainwalkers -) SELECT block_height, block_timestamp, @@ -80,13 +18,12 @@ SELECT ['tx_id','sender', 'recipient','token_contract', 'amount'] ) }} ) AS ez_token_transfers_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__token_transfers_s') }} +WHERE + token_contract NOT IN ( + 'A.c38aea683c0c4d38.ZelosAccountingToken', + 'A.f1b97c06745f37ad.SwapPair' + ) diff --git a/models/gold/core/core__ez_token_transfers.yml b/models/gold/core/core__ez_token_transfers.yml index 3e8a324..09a3db6 100644 --- a/models/gold/core/core__ez_token_transfers.yml +++ b/models/gold/core/core__ez_token_transfers.yml @@ -15,7 +15,6 @@ models: - amount config: severity: warn - error_if: ">10" columns: - name: TX_ID diff --git a/models/gold/core/core__fact_blocks.sql b/models/gold/core/core__fact_blocks.sql index b349778..f9c355e 100644 --- a/models/gold/core/core__fact_blocks.sql +++ b/models/gold/core/core__fact_blocks.sql @@ -3,67 +3,12 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS blocks_id, - block_height, - block_timestamp, - network, - network_version, - chain_id, - tx_count, - id, - parent_id, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__blocks') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - blocks_id, - block_height, - block_timestamp, - 'mainnet' AS network, - network_version, - 'flow' AS chain_id, - tx_count, - id, - parent_id, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__streamline_blocks') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT - blocks_id, - block_height, + block_height :: INT AS block_height, block_timestamp, - network, + 'mainnet' AS network, network_version, - chain_id, + 'flow' AS chain_id, tx_count, id, parent_id, @@ -71,13 +16,8 @@ SELECT blocks_id, {{ dbt_utils.generate_surrogate_key(['block_height']) }} ) AS fact_blocks_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + _inserted_timestamp, + inserted_timestamp, + modified_timestamp FROM - FINAL + {{ ref('silver__streamline_blocks') }} diff --git a/models/gold/core/core__fact_events.sql b/models/gold/core/core__fact_events.sql index 4cf1610..ef5bbd1 100644 --- a/models/gold/core/core__fact_events.sql +++ b/models/gold/core/core__fact_events.sql @@ -3,64 +3,10 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS streamline_event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__events_final') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - streamline_event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__streamline_events') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT tx_id, block_timestamp, - block_height, + block_height :: INT AS block_height, tx_succeeded, event_index, event_contract, @@ -70,13 +16,8 @@ SELECT streamline_event_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS fact_events_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + _inserted_timestamp, + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__streamline_events') }} diff --git a/models/gold/core/core__fact_transactions.sql b/models/gold/core/core__fact_transactions.sql index a40d38e..e37b757 100644 --- a/models/gold/core/core__fact_transactions.sql +++ b/models/gold/core/core__fact_transactions.sql @@ -3,103 +3,35 @@ tags = ['scheduled'] ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS streamline_transaction_id, - tx_id, - block_timestamp, - block_height, - chain_id, - tx_index, - proposer, - payer, - authorizers, - count_authorizers, - gas_limit, - transaction_result, - tx_succeeded, - error_msg, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__transactions') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - streamline_transaction_id, - tx_id, - block_timestamp, - block_height, - 'flow' AS chain_id, - NULL AS tx_index, - proposer, - payer, - authorizers, - count_authorizers, - gas_limit, - OBJECT_CONSTRUCT( - 'error', - error_message, - 'events', - events, - 'status', - status - ) AS transaction_result, - tx_succeeded, - error_message AS error_msg, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__streamline_transactions_final') }} - WHERE - NOT pending_result_response - AND block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT tx_id, block_timestamp, - block_height, - chain_id, - tx_index, + block_height :: INT AS block_height, + 'flow' AS chain_id, proposer, payer, authorizers, count_authorizers, gas_limit, - transaction_result, + script, + arguments, + OBJECT_CONSTRUCT( + 'error', + error_message, + 'events', + events, + 'status', + status + ) AS transaction_result, tx_succeeded, - error_msg, + error_message AS error_msg, COALESCE ( streamline_transaction_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS fact_transactions_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__streamline_transactions_final') }} +WHERE + NOT pending_result_response diff --git a/models/gold/core/core__fact_transactions.yml b/models/gold/core/core__fact_transactions.yml index 5ef1c41..e7865a7 100644 --- a/models/gold/core/core__fact_transactions.yml +++ b/models/gold/core/core__fact_transactions.yml @@ -85,6 +85,12 @@ models: column_type_list: - NUMBER + - name: SCRIPT + description: "{{ doc('script') }}" + + - name: ARGUMENTS + description: "{{ doc('arguments') }}" + - name: TRANSACTION_RESULT description: "{{ doc('transaction_result') }}" tests: @@ -92,6 +98,7 @@ models: - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - VARIANT + - OBJECT - name: TX_SUCCEEDED description: "{{ doc('tx_succeeded') }}" diff --git a/models/gold/defi/defi__dim_swap_pool_labels.sql b/models/gold/defi/defi__dim_swap_pool_labels.sql index 0f46764..b4f2ae0 100644 --- a/models/gold/defi/defi__dim_swap_pool_labels.sql +++ b/models/gold/defi/defi__dim_swap_pool_labels.sql @@ -3,40 +3,8 @@ tag = ['scheduled'] ) }} -WITH pairs_cw AS ( +WITH pairs_s AS ( - SELECT - tx_id, - NULL AS labels_pools_metapier_id, - swap_contract, - deployment_timestamp, - token0_contract, - token1_contract, - pool_id, - vault_address, - NULL AS inserted_timestamp, - _inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__labels_pools') }} -), -metapier_cw AS ( - SELECT - tx_id, - NULL AS labels_pools_metapier_id, - swap_contract, - deployment_timestamp, - token0_contract, - token1_contract, - pool_id, - vault_address, - NULL AS inserted_timestamp, - _inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__labels_pools_metapier') }} -), -pairs_s AS ( SELECT tx_id, labels_pools_id AS labels_pools_metapier_id, @@ -47,7 +15,6 @@ pairs_s AS ( pool_id, vault_address, inserted_timestamp, - _inserted_timestamp, modified_timestamp FROM {{ ref('silver__labels_pools_s') }} @@ -63,27 +30,16 @@ metapier_s AS ( pool_id, vault_address, inserted_timestamp, - _inserted_timestamp, modified_timestamp FROM {{ ref('silver__labels_pools_metapier_s') }} ), FINAL AS ( - SELECT - * - FROM - pairs_cw - UNION - SELECT - * - FROM - metapier_cw - UNION SELECT * FROM pairs_s - UNION + UNION ALL SELECT * FROM @@ -100,13 +56,7 @@ SELECT labels_pools_metapier_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS dim_swap_pool_labels_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + FINAL diff --git a/models/gold/defi/defi__ez_bridge_activity.sql b/models/gold/defi/defi__ez_bridge_activity.sql new file mode 100644 index 0000000..c1dd383 --- /dev/null +++ b/models/gold/defi/defi__ez_bridge_activity.sql @@ -0,0 +1,56 @@ +{{ config( + materialized = 'view', + tags = ['ez', 'bridge', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BRIDGE' }} } +) }} + +WITH prices AS ( + + SELECT + hour, + token_address, + symbol, + price + FROM + {{ ref('silver__complete_token_prices') }} + UNION ALL + SELECT + hour, + 'A.1654653399040a61.FlowToken' AS token_address, + symbol, + price + FROM + {{ ref('silver__complete_native_prices') }} +) +SELECT + tx_id, + block_timestamp, + block_height, + bridge_address, + b.token_address, + p.symbol AS token_symbol, + gross_amount AS amount, + amount_fee, + gross_amount * p.price AS amount_usd, + amount_fee * p.price AS amount_fee_usd, + source_address, + destination_address, + source_chain, + destination_chain, + platform, + bridge_complete_id AS ez_bridge_activity_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__bridge_complete') }} b + LEFT JOIN prices p + ON LOWER( + b.token_address + ) = LOWER( + p.token_address + ) + AND DATE_TRUNC( + 'hour', + b.block_timestamp + ) = p.hour + diff --git a/models/silver_cw/transfers/silver__bridge_blocto.yml b/models/gold/defi/defi__ez_bridge_activity.yml similarity index 63% rename from models/silver_cw/transfers/silver__bridge_blocto.yml rename to models/gold/defi/defi__ez_bridge_activity.yml index d42121e..970ff60 100644 --- a/models/silver_cw/transfers/silver__bridge_blocto.yml +++ b/models/gold/defi/defi__ez_bridge_activity.yml @@ -1,32 +1,29 @@ version: 2 models: - - name: silver__bridge_blocto + - name: defi__ez_bridge_activity description: |- - This table parses transactions where tokens are bridged to or from the Flow network via Blocto Teleport. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id + This table parses transactions where tokens are bridged to or from the Flow network using Blocto teleport or the Celer bridge. columns: - - name: tx_id + - name: TX_ID description: "{{ doc('tx_id') }}" tests: - not_null - - name: block_timestamp + - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day interval: 3 + where: block_height >= 55114467 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ - - name: block_height + - name: BLOCK_HEIGHT description: "{{ doc('block_height') }}" tests: - not_null @@ -35,8 +32,8 @@ models: - NUMBER - FLOAT - - name: teleport_contract - description: "{{ doc('teleport_contract') }}" + - name: BRIDGE_ADDRESS + description: "{{ doc('bridge_contract') }}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: @@ -44,7 +41,7 @@ models: - STRING - VARCHAR - - name: token_contract + - name: TOKEN_ADDRESS description: "{{ doc('token_contract') }}" tests: - not_null @@ -53,8 +50,11 @@ models: - STRING - VARCHAR - - name: gross_amount - description: "{{ doc('gross_amount') }}" + - name: TOKEN_SYMBOL + description: "{{ doc('token_symbol') }}" + + - name: AMOUNT + description: "{{ doc('amount') }}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: @@ -63,45 +63,39 @@ models: - DOUBLE - FLOAT - - name: amount_fee + - name: AMOUNT_FEE description: "{{ doc('amount_fee') }}" tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT + - not_null: + where: platform = 'blocto' - - name: net_amount - description: "{{ doc('net_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT + - name: AMOUNT_USD + description: "{{ doc('amount_usd') }}" + + - name: AMOUNT_FEE_USD + description: "{{ doc('amount_fee_usd') }}" - - name: flow_wallet_address + - name: SOURCE_ADDRESS description: "{{ doc('flow_wallet_address') }}" tests: - - not_null + - not_null: + where: source_chain = 'flow' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - - name: teleport_direction - description: "{{ doc('teleport_direction') }}" + - name: DESTINATION_ADDRESS + description: "{{ doc('flow_wallet_address') }}" tests: - - not_null + - not_null: + where: destination_chain = 'flow' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - - name: blockchain + - name: SOURCE_CHAIN description: "{{ doc('blockchain') }}" tests: - not_null @@ -110,7 +104,16 @@ models: - STRING - VARCHAR - - name: bridge + - name: DESTINATION_CHAIN + description: "{{ doc('blockchain') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: PLATFORM description: "{{ doc('bridge') }}" tests: - not_null @@ -119,13 +122,11 @@ models: - STRING - VARCHAR - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" + - name: FACT_BRIDGE_TRANSACTIONS_ID + description: "{{ doc('pk_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/defi/defi__ez_bridge_transactions.sql b/models/gold/defi/defi__ez_bridge_transactions.sql index 32b5523..03098c6 100644 --- a/models/gold/defi/defi__ez_bridge_transactions.sql +++ b/models/gold/defi/defi__ez_bridge_transactions.sql @@ -4,84 +4,20 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BRIDGE' }} } ) }} -WITH blocto_cw AS ( - - SELECT - * - FROM - {{ ref('silver__bridge_blocto') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -celer_cw AS ( - SELECT - * - FROM - {{ ref('silver__bridge_celer') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), +WITH blocto_s AS ( SELECT * FROM {{ ref('silver__bridge_blocto_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} ), celer_s AS ( SELECT * FROM {{ ref('silver__bridge_celer_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} ), combo AS ( - SELECT - NULL AS bridge_id, - tx_id, - block_timestamp, - block_height, - teleport_contract AS bridge_contract, - token_contract, - gross_amount AS amount, - flow_wallet_address, - blockchain, - teleport_direction AS direction, - bridge, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - blocto_cw - UNION ALL - SELECT - NULL AS bridge_id, - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - blockchain, - direction, - bridge, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - celer_cw - UNION ALL SELECT bridge_blocto_id AS bridge_id, tx_id, @@ -94,7 +30,6 @@ combo AS ( blockchain, teleport_direction AS direction, bridge, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM @@ -112,7 +47,6 @@ combo AS ( blockchain, direction, bridge, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM @@ -133,13 +67,7 @@ SELECT bridge_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS ez_bridge_transactions_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM combo \ No newline at end of file diff --git a/models/gold/defi/defi__ez_bridge_transactions.yml b/models/gold/defi/defi__ez_bridge_transactions.yml index ce23d79..e153d3b 100644 --- a/models/gold/defi/defi__ez_bridge_transactions.yml +++ b/models/gold/defi/defi__ez_bridge_transactions.yml @@ -3,8 +3,8 @@ version: 2 models: - name: defi__ez_bridge_transactions description: |- - This table parses transactions where tokens are bridged to or from the Flow network using Blocto teleport or the Celer bridge. - + Deprecating Soon! This table will be deprecated on September 1st, please migrate work `defi__ez_bridge_activity` instead. + tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/models/gold/defi/defi__ez_dex_swaps.sql b/models/gold/defi/defi__ez_dex_swaps.sql new file mode 100644 index 0000000..77a1d57 --- /dev/null +++ b/models/gold/defi/defi__ez_dex_swaps.sql @@ -0,0 +1,69 @@ +{{ config ( + materialized = 'view', + tags = ['ez', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } +) }} +WITH prices AS ( + + SELECT + hour, + token_address, + symbol, + price + FROM + {{ ref('silver__complete_token_prices') }} + UNION ALL + SELECT + hour, + 'A.1654653399040a61.FlowToken' AS token_address, + symbol, + price + FROM + {{ ref('silver__complete_native_prices') }} +) +SELECT + tx_id, + block_timestamp, + block_height, + swap_contract AS contract_address, + swap_index, + trader, + platform, + token_out_source AS origin_from_address, + token_out_contract AS token_out, + po.symbol AS token_out_symbol, + token_out_amount AS amount_out, + token_out_amount * po.price AS amount_out_usd, + token_in_destination AS origin_to_address, + token_in_contract AS token_in, + pi.symbol AS token_in_symbol, + token_in_amount AS amount_in, + token_in_amount * pi.price AS amount_in_usd, + swaps_final_id AS ez_dex_swaps_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__swaps_final') }} s + LEFT JOIN prices po + ON LOWER( + s.token_out_contract + ) = LOWER( + po.token_address + ) + AND DATE_TRUNC( + 'hour', + s.block_timestamp + ) = po.hour + LEFT JOIN prices pi + ON LOWER( + s.token_in_contract + ) = LOWER( + pi.token_address + ) + AND DATE_TRUNC( + 'hour', + s.block_timestamp + ) = pi.hour + +WHERE + token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__ez_dex_swaps.yml b/models/gold/defi/defi__ez_dex_swaps.yml new file mode 100644 index 0000000..06623b1 --- /dev/null +++ b/models/gold/defi/defi__ez_dex_swaps.yml @@ -0,0 +1,88 @@ +version: 2 + +models: + - name: defi__ez_dex_swaps + description: |- + This table records asset swaps on the Flow blockchain, with price and label information. + + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: BLOCK_HEIGHT + description: "{{ doc('block_height') }}" + tests: + - not_null + + - name: CONTRACT_ADDRESS + description: "{{ doc('swap_contract') }}" + tests: + - not_null + + - name: SWAP_INDEX + description: "{{ doc('swap_index') }}" + tests: + - not_null + + - name: TRADER + description: "{{ doc('trader') }}" + tests: + - not_null: + severity: warn + + - name: PLATFORM + description: "{{ doc('platform') }}" + + - name: ORIGIN_FROM_ADDRESS + description: "{{ doc('token_out_source') }}" + + - name: TOKEN_OUT + description: "{{ doc('token_out_contract') }}" + tests: + - not_null + + - name: TOKEN_OUT_SYMBOL + description: "{{ doc('symbol') }}" + + - name: AMOUNT_OUT + description: "{{ doc('token_out_amount') }}" + tests: + - not_null + + - name: AMOUNT_OUT_USD + description: "{{ doc('amount_usd') }}" + + - name: ORIGIN_TO_ADDRESS + description: "{{ doc('token_in_destination') }}" + + - name: TOKEN_IN + description: "{{ doc('token_in_contract') }}" + tests: + - not_null + + - name: TOKEN_IN_SYMBOL + description: "{{ doc('symbol') }}" + + - name: AMOUNT_IN + description: "{{ doc('token_in_amount') }}" + tests: + - not_null + + - name: AMOUNT_IN_USD + description: "{{ doc('amount_usd') }}" + + - name: EZ_DEX_SWAPS_ID + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/defi/defi__ez_swaps.sql b/models/gold/defi/defi__ez_swaps.sql index a682a37..96a1d78 100644 --- a/models/gold/defi/defi__ez_swaps.sql +++ b/models/gold/defi/defi__ez_swaps.sql @@ -4,67 +4,8 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS swaps_id, - tx_id, - block_timestamp, - block_height, - swap_contract, - swap_index, - trader, - token_out_source, - token_out_contract, - token_out_amount, - token_in_destination, - token_in_contract, - token_in_amount, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__swaps') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - swaps_final_id AS swaps_id, - tx_id, - block_timestamp, - block_height, - swap_contract, - swap_index, - trader, - token_out_source, - token_out_contract, - token_out_amount, - token_in_destination, - token_in_contract, - token_in_amount, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__swaps_final') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT + swaps_final_id AS swaps_id, tx_id, block_timestamp, block_height, @@ -81,9 +22,9 @@ SELECT swaps_id, {{ dbt_utils.generate_surrogate_key(['tx_id', 'swap_index']) }} ) AS ez_swaps_id, - COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, - COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL + {{ ref('silver__swaps_final') }} WHERE token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__ez_swaps.yml b/models/gold/defi/defi__ez_swaps.yml index fd4f0ca..1f83bdd 100644 --- a/models/gold/defi/defi__ez_swaps.yml +++ b/models/gold/defi/defi__ez_swaps.yml @@ -3,7 +3,7 @@ version: 2 models: - name: defi__ez_swaps description: |- - This table records asset swaps on the Flow blockchain. + Deprecating Soon! Please migrate work to the new `defi.ez_dex_swaps` table by September 1st. columns: - name: TX_ID @@ -34,7 +34,8 @@ models: - name: TRADER description: "{{ doc('trader') }}" tests: - - not_null + - not_null: + severity: warn - name: TOKEN_OUT_SOURCE description: "{{ doc('token_out_source') }}" diff --git a/models/gold/defi/defi__fact_bridge_activity.sql b/models/gold/defi/defi__fact_bridge_activity.sql new file mode 100644 index 0000000..3ba03b9 --- /dev/null +++ b/models/gold/defi/defi__fact_bridge_activity.sql @@ -0,0 +1,24 @@ +{{ config( + materialized = 'view', + tags = ['ez', 'bridge', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'BRIDGE' }} } +) }} + +SELECT + tx_id, + block_timestamp, + block_height, + bridge_address, + token_address, + gross_amount AS amount, + amount_fee, + source_address, + destination_address, + source_chain, + destination_chain, + platform, + bridge_complete_id AS fact_bridge_activity_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__bridge_complete') }} diff --git a/models/silver_cw/transfers/silver__bridge_celer.yml b/models/gold/defi/defi__fact_bridge_activity.yml similarity index 68% rename from models/silver_cw/transfers/silver__bridge_celer.yml rename to models/gold/defi/defi__fact_bridge_activity.yml index 363ea98..ca8e69a 100644 --- a/models/silver_cw/transfers/silver__bridge_celer.yml +++ b/models/gold/defi/defi__fact_bridge_activity.yml @@ -1,32 +1,29 @@ version: 2 models: - - name: silver__bridge_celer + - name: defi__fact_bridge_activity description: |- - This table parses transactions where tokens are bridged to or from the Flow network using the Celer cBridge. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id + This table parses transactions where tokens are bridged to or from the Flow network using Blocto teleport or the Celer bridge. columns: - - name: tx_id + - name: TX_ID description: "{{ doc('tx_id') }}" tests: - not_null - - name: block_timestamp + - name: BLOCK_TIMESTAMP description: "{{ doc('block_timestamp') }}" tests: - not_null - dbt_expectations.expect_row_values_to_have_recent_data: datepart: day interval: 3 + where: block_height >= 55114467 - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ - - name: block_height + - name: BLOCK_HEIGHT description: "{{ doc('block_height') }}" tests: - not_null @@ -35,7 +32,7 @@ models: - NUMBER - FLOAT - - name: bridge_contract + - name: BRIDGE_ADDRESS description: "{{ doc('bridge_contract') }}" tests: - not_null @@ -44,7 +41,7 @@ models: - STRING - VARCHAR - - name: token_contract + - name: TOKEN_ADDRESS description: "{{ doc('token_contract') }}" tests: - not_null @@ -53,7 +50,7 @@ models: - STRING - VARCHAR - - name: amount + - name: AMOUNT description: "{{ doc('amount') }}" tests: - not_null @@ -63,33 +60,33 @@ models: - DOUBLE - FLOAT - - name: flow_wallet_address + - name: AMOUNT_FEE + description: "{{ doc('amount_fee') }}" + tests: + - not_null: + where: platform = 'blocto' + + - name: SOURCE_ADDRESS description: "{{ doc('flow_wallet_address') }}" tests: - - not_null + - not_null: + where: source_chain = 'flow' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - - name: counterparty - description: "{{ doc('counterparty') }}" + - name: DESTINATION_ADDRESS + description: "{{ doc('flow_wallet_address') }}" tests: - - not_null + - not_null: + where: destination_chain = 'flow' - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - - name: chain_id - description: "{{ doc('chain_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: blockchain + - name: SOURCE_CHAIN description: "{{ doc('blockchain') }}" tests: - not_null @@ -98,8 +95,8 @@ models: - STRING - VARCHAR - - name: direction - description: "{{ doc('direction') }}" + - name: DESTINATION_CHAIN + description: "{{ doc('blockchain') }}" tests: - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: @@ -107,7 +104,7 @@ models: - STRING - VARCHAR - - name: bridge + - name: PLATFORM description: "{{ doc('bridge') }}" tests: - not_null @@ -116,13 +113,11 @@ models: - STRING - VARCHAR - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" + - name: FACT_BRIDGE_TRANSACTIONS_ID + description: "{{ doc('pk_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/defi/defi__fact_dex_swaps.sql b/models/gold/defi/defi__fact_dex_swaps.sql new file mode 100644 index 0000000..51f4ffd --- /dev/null +++ b/models/gold/defi/defi__fact_dex_swaps.sql @@ -0,0 +1,27 @@ +{{ config ( + materialized = 'view', + tags = ['ez', 'scheduled'], + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } +) }} + +SELECT + tx_id, + block_timestamp, + block_height, + swap_contract AS contract_address, + swap_index, + trader, + platform, + token_out_source AS origin_from_address, + token_out_contract AS token_out, + token_out_amount AS amount_out, + token_in_destination AS origin_to_address, + token_in_contract AS token_in, + token_in_amount AS amount_in, + swaps_final_id AS fact_dex_swaps_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__swaps_final') }} +WHERE + token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__fact_dex_swaps.yml b/models/gold/defi/defi__fact_dex_swaps.yml new file mode 100644 index 0000000..9397201 --- /dev/null +++ b/models/gold/defi/defi__fact_dex_swaps.yml @@ -0,0 +1,76 @@ +version: 2 + +models: + - name: defi__fact_dex_swaps + description: |- + This table records asset swaps on the Flow blockchain. + + columns: + - name: TX_ID + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: BLOCK_TIMESTAMP + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + + - name: BLOCK_HEIGHT + description: "{{ doc('block_height') }}" + tests: + - not_null + + - name: CONTRACT_ADDRESS + description: "{{ doc('swap_contract') }}" + tests: + - not_null + + - name: SWAP_INDEX + description: "{{ doc('swap_index') }}" + tests: + - not_null + + - name: TRADER + description: "{{ doc('trader') }}" + tests: + - not_null: + severity: warn + + - name: PLATFORM + description: "{{ doc('platform') }}" + + - name: ORIGIN_FROM_ADDRESS + description: "{{ doc('token_out_source') }}" + + - name: AMOUNT_OUT + description: "{{ doc('token_out_amount') }}" + tests: + - not_null + + - name: TOKEN_OUT + description: "{{ doc('token_out_contract') }}" + tests: + - not_null + + - name: ORIGIN_TO_ADDRESS + description: "{{ doc('token_in_destination') }}" + + - name: AMOUNT_IN + description: "{{ doc('token_in_amount') }}" + tests: + - not_null + + - name: TOKEN_IN + description: "{{ doc('token_in_contract') }}" + tests: + - not_null + + - name: FACT_DEX_SWAPS_ID + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/gov/gov__ez_staking_actions.sql b/models/gold/gov/gov__ez_staking_actions.sql index 19b49d4..68e1c5e 100644 --- a/models/gold/gov/gov__ez_staking_actions.sql +++ b/models/gold/gov/gov__ez_staking_actions.sql @@ -1,65 +1,9 @@ {{ config( materialized = 'view', tags = ['ez', 'scheduled'], - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }}} + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'STAKING' }} } ) }} -WITH chainwalkers AS ( - - SELECT - * - FROM - {{ ref('silver__staking_actions') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - * - FROM - {{ ref('silver__staking_actions_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - NULL AS staking_actions_id, - tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - delegator, - action, - amount, - node_id, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - chainwalkers - UNION ALL - SELECT - staking_actions_id, - tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - delegator, - action, - amount, - node_id, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - streamline -) SELECT tx_id, event_index, @@ -74,13 +18,7 @@ SELECT staking_actions_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS ez_staking_actions_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL \ No newline at end of file + {{ ref('silver__staking_actions_s') }} diff --git a/models/gold/nft/nft__dim_allday_metadata.sql b/models/gold/nft/nft__dim_allday_metadata.sql index d85cbfc..3ebe075 100644 --- a/models/gold/nft/nft__dim_allday_metadata.sql +++ b/models/gold/nft/nft__dim_allday_metadata.sql @@ -6,32 +6,6 @@ WITH allday AS ( - SELECT - NULL AS nft_unique_id, - nft_id, - nft_collection, - nflallday_id, - serial_number, - moment_tier, - total_circulation, - moment_description, - player, - team, - season, - week, - classification, - play_type, - moment_date, - series, - set_name, - video_urls, - moment_stats_full, - _inserted_timestamp, - _inserted_timestamp AS inserted_timestamp, - _inserted_timestamp AS modified_timestamp - FROM - {{ ref('silver__nft_allday_metadata') }} - UNION SELECT nft_allday_metadata_s_id AS nft_unique_id, nft_id, @@ -52,7 +26,6 @@ WITH allday AS ( set_name, video_urls, moment_stats_full, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM @@ -78,7 +51,6 @@ WITH allday AS ( set_name, video_urls, moment_stats_full, - _inserted_timestamp, _inserted_timestamp AS inserted_timestamp, _inserted_timestamp AS modified_timestamp FROM @@ -107,13 +79,7 @@ SELECT nft_unique_id, {{ dbt_utils.generate_surrogate_key(['nft_id']) }} ) AS dim_allday_metadata_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM allday diff --git a/models/gold/nft/nft__dim_allday_metadata.yml b/models/gold/nft/nft__dim_allday_metadata.yml index 8f5dafa..270e086 100644 --- a/models/gold/nft/nft__dim_allday_metadata.yml +++ b/models/gold/nft/nft__dim_allday_metadata.yml @@ -5,99 +5,56 @@ models: description: |- Data for NFL AllDay Moments, including player, team, stats and more. This is produced via API and may differ in structure from metadata available on-chain in the `dim_moment_metadata` table. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - nft_id - severity: error - error_if: ">10" - warn_if: "<10" - columns: - name: NFT_ID description: "{{ doc('nft_id') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" - - unique: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null + - unique - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - severity: error - error_if: ">10" - warn_if: "<10" + - name: NFT_COLLECTION description: "{{ doc('nft_collection') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - severity: error - error_if: ">10" - warn_if: "<10" - name: NFLALLDAY_ID description: "{{ doc('nflallday_id') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR - severity: error - error_if: ">10" - warn_if: "<10" - name: SERIAL_NUMBER description: "{{ doc('serial_number') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - severity: error - error_if: ">10" - warn_if: "<10" - name: TOTAL_CIRCULATION description: "{{ doc('total_circulation') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER - severity: error - error_if: ">10" - warn_if: "<10" - name: MOMENT_DESCRIPTION description: "{{ doc('moment_description') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -106,10 +63,7 @@ models: - name: PLAYER description: "{{ doc('player') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -118,10 +72,7 @@ models: - name: TEAM description: "{{ doc('team') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -130,10 +81,7 @@ models: - name: SEASON description: "{{ doc('season') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -142,10 +90,7 @@ models: - name: WEEK description: "{{ doc('week') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -154,10 +99,7 @@ models: - name: CLASSIFICATION description: "{{ doc('classification') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -166,10 +108,7 @@ models: - name: PLAY_TYPE description: "{{ doc('play_type') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -178,10 +117,7 @@ models: - name: MOMENT_DATE description: "{{ doc('moment_date') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ @@ -189,10 +125,7 @@ models: - name: SERIES description: "{{ doc('series') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -201,10 +134,7 @@ models: - name: SET_NAME description: "{{ doc('set_name') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -213,10 +143,7 @@ models: - name: VIDEO_URLS description: "{{ doc('video_urls') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - ARRAY @@ -224,10 +151,7 @@ models: - name: MOMENT_STATS_FULL description: "{{ doc('moment_stats_full') }}" tests: - - not_null: - severity: error - error_if: ">10" - warn_if: "<10" + - not_null - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - OBJECT diff --git a/models/gold/nft/nft__dim_moment_metadata.sql b/models/gold/nft/nft__dim_moment_metadata.sql index 47571e8..5d7a532 100644 --- a/models/gold/nft/nft__dim_moment_metadata.sql +++ b/models/gold/nft/nft__dim_moment_metadata.sql @@ -4,80 +4,8 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT, ALLDAY, GOLAZOS, TOPSHOT' }} } ) }} -WITH chainwalkers AS ( - - SELECT - NULL AS nft_moment_metadata_id, - event_contract AS nft_collection, - nft_id, - serial_number, - max_mint_size, - play_id, - series_id, - series_name, - set_id, - set_name, - edition_id, - tier, - metadata, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__nft_moment_metadata_final') }} - WHERE - NOT ( - nft_collection = 'A.87ca73a41bb50ad5.Golazos' - AND edition_id = 486 - ) - AND NOT ( - nft_collection = 'A.e4cf4bdc1751c65d.AllDay' - AND edition_id = 1486 - ) -), -streamline AS ( - SELECT - nft_moment_metadata_id, - event_contract AS nft_collection, - nft_id, - serial_number, - max_mint_size, - play_id, - series_id, - series_name, - set_id, - set_name, - edition_id, - tier, - metadata, - _inserted_timestamp, - inserted_timestamp, - modified_timestamp - FROM - {{ ref('silver__nft_moment_metadata_final_s') }} - WHERE - NOT ( - nft_collection = 'A.87ca73a41bb50ad5.Golazos' - AND edition_id = 486 - ) - AND NOT ( - nft_collection = 'A.e4cf4bdc1751c65d.AllDay' - AND edition_id = 1486 - ) -), -FINAL AS ( - SELECT - * - FROM - chainwalkers - UNION ALL - SELECT - * - FROM - streamline -) SELECT - nft_collection, + event_contract AS nft_collection, nft_id, serial_number, max_mint_size, @@ -95,17 +23,23 @@ SELECT ['nft_collection','edition_id','nft_id'] ) }} ) AS dim_moment_metadata_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY dim_moment_metadata_id - ORDER BY - _inserted_timestamp DESC - ) = 1 + {{ ref('silver__nft_moment_metadata_final_s') }} +WHERE + NOT ( + nft_collection = 'A.87ca73a41bb50ad5.Golazos' + AND edition_id = 486 + ) + AND NOT ( + nft_collection = 'A.e4cf4bdc1751c65d.AllDay' + AND edition_id = 1486 + ) + AND metadata IS NOT NULL + +qualify ROW_NUMBER() over ( + PARTITION BY dim_moment_metadata_id + ORDER BY + inserted_timestamp DESC +) = 1 diff --git a/models/gold/nft/nft__dim_moment_metadata.yml b/models/gold/nft/nft__dim_moment_metadata.yml index 5f30c62..c46c096 100644 --- a/models/gold/nft/nft__dim_moment_metadata.yml +++ b/models/gold/nft/nft__dim_moment_metadata.yml @@ -5,54 +5,47 @@ models: description: |- NFT Moment Metadata scraped from on-chain activity, where available. This should be joinable on sales or mints using the nft_collection (event_contract) and nft id. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - nft_collection - - edition_id - - nft_id - columns: - name: NFT_COLLECTION description: "{{ doc('nft_collection') }}" tests: - not_null + - name: NFT_ID description: "{{ doc('nft_id') }}" tests: - not_null + - name: SERIAL_NUMBER description: "{{ doc('serial_number') }}" tests: - not_null - name: MAX_MINT_SIZE description: "{{ doc('max_mint_size') }}" - tests: - - not_null + - name: PLAY_ID description: "{{ doc('play_id') }}" tests: - not_null + - name: SERIES_ID description: "{{ doc('series_id') }}" - tests: - - not_null + - name: SERIES_NAME description: "{{ doc('series_name') }}" + - name: SET_ID description: "{{ doc('set_id') }}" - tests: - - not_null + - name: SET_NAME description: "{{ doc('set_name') }}" + - name: EDITION_ID description: "{{ doc('edition_id') }}" - tests: - - not_null + - name: TIER description: "{{ doc('tier') }}" - tests: - - not_null + - name: METADATA description: "{{ doc('metadata') }}" tests: diff --git a/models/gold/nft/nft__ez_nft_sales.sql b/models/gold/nft/nft__ez_nft_sales.sql index b70482f..f53b433 100644 --- a/models/gold/nft/nft__ez_nft_sales.sql +++ b/models/gold/nft/nft__ez_nft_sales.sql @@ -4,70 +4,6 @@ meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'NFT' }} } ) }} -WITH chainwalkers AS ( - - SELECT - * - FROM - {{ ref('silver__nft_sales') }} - WHERE - block_height < {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -streamline AS ( - SELECT - * - FROM - {{ ref('silver__nft_sales_s') }} - WHERE - block_height >= {{ var( - 'STREAMLINE_START_BLOCK' - ) }} -), -FINAL AS ( - SELECT - NULL AS nft_sales_id, - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - tokenflow, - counterparties, - NULL AS inserted_timestamp, - _inserted_timestamp, - NULL AS modified_timestamp - FROM - chainwalkers - UNION ALL - SELECT - nft_sales_id, - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - tokenflow, - counterparties, - inserted_timestamp, - _inserted_timestamp, - modified_timestamp - FROM - streamline -) SELECT tx_id, block_height, @@ -86,13 +22,7 @@ SELECT nft_sales_id, {{ dbt_utils.generate_surrogate_key(['tx_id']) }} ) AS ez_nft_sales_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + inserted_timestamp, + modified_timestamp FROM - FINAL + {{ ref('silver__nft_sales_s') }} diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml index ba4f038..9946b8b 100644 --- a/models/gold/nft/nft__ez_nft_sales.yml +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -13,9 +13,8 @@ models: - buyer - nft_collection - nft_id - severity: error - error_if: ">10" - warn_if: "<10" + severity: warn + error_if: ">5000" columns: - name: TX_ID diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql new file mode 100644 index 0000000..16b02e0 --- /dev/null +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + token_address, + asset_id, + symbol, + name, + platform AS blockchain, + platform_id AS blockchain_id, + provider, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id AS dim_asset_metadata_id +FROM + {{ ref('silver__complete_provider_asset_metadata') }} diff --git a/models/gold/price/price__dim_asset_metadata.yml b/models/gold/price/price__dim_asset_metadata.yml new file mode 100644 index 0000000..46234b2 --- /dev/null +++ b/models/gold/price/price__dim_asset_metadata.yml @@ -0,0 +1,23 @@ +version: 2 +models: + - name: price__dim_asset_metadata + description: '{{ doc("prices_dim_asset_metadata_table_doc") }}' + + columns: + - name: PROVIDER + description: '{{ doc("prices_provider")}}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: NAME + description: '{{ doc("prices_name") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: BLOCKCHAIN_ID + description: '{{ doc("prices_blockchain_id") }}' + - name: DIM_ASSET_METADATA_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql new file mode 100644 index 0000000..bc1ec5c --- /dev/null +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -0,0 +1,35 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + FALSE AS is_native, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id AS ez_asset_metadata_id +FROM + {{ ref('silver__complete_token_asset_metadata') }} +UNION ALL +SELECT + NULL AS token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + TRUE AS is_native, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id AS ez_asset_metadata_id +FROM + {{ ref('silver__complete_native_asset_metadata') }} diff --git a/models/gold/price/price__ez_asset_metadata.yml b/models/gold/price/price__ez_asset_metadata.yml new file mode 100644 index 0000000..ceb646c --- /dev/null +++ b/models/gold/price/price__ez_asset_metadata.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: price__ez_asset_metadata + description: '{{ doc("prices_ez_asset_metadata_table_doc") }}' + + columns: + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: NAME + description: '{{ doc("prices_name") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: EZ_ASSET_METADATA_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/price/price__ez_prices_hourly.sql b/models/gold/price/price__ez_prices_hourly.sql new file mode 100644 index 0000000..dfda724 --- /dev/null +++ b/models/gold/price/price__ez_prices_hourly.sql @@ -0,0 +1,39 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + HOUR, + token_address, + symbol, + NAME, + decimals, + price, + blockchain, + FALSE AS is_native, + is_imputed, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id AS ez_prices_hourly_id +FROM + {{ ref('silver__complete_token_prices') }} +UNION ALL +SELECT + HOUR, + NULL AS token_address, + symbol, + NAME, + decimals, + price, + blockchain, + TRUE AS is_native, + is_imputed, + is_deprecated, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id AS ez_prices_hourly_id +FROM + {{ ref('silver__complete_native_prices') }} diff --git a/models/gold/price/price__ez_prices_hourly.yml b/models/gold/price/price__ez_prices_hourly.yml new file mode 100644 index 0000000..cc65a9b --- /dev/null +++ b/models/gold/price/price__ez_prices_hourly.yml @@ -0,0 +1,30 @@ +version: 2 +models: + - name: price__ez_prices_hourly + description: '{{ doc("prices_ez_prices_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + - name: TOKEN_ADDRESS + description: '{{ doc("prices_token_address") }}' + - name: SYMBOL + description: '{{ doc("prices_symbol") }}' + - name: BLOCKCHAIN + description: '{{ doc("prices_blockchain") }}' + - name: DECIMALS + description: '{{ doc("prices_decimals") }}' + - name: PRICE + description: '{{ doc("prices_price") }}' + - name: IS_NATIVE + description: '{{ doc("prices_is_native") }}' + - name: IS_IMPUTED + description: '{{ doc("prices_is_imputed") }}' + - name: IS_DEPRECATED + description: '{{ doc("prices_is_deprecated") }}' + - name: EZ_PRICES_HOURLY_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/price/price__fact_hourly_prices.sql b/models/gold/price/price__fact_hourly_prices.sql deleted file mode 100644 index f11fe37..0000000 --- a/models/gold/price/price__fact_hourly_prices.sql +++ /dev/null @@ -1,124 +0,0 @@ -{{ config( - materialized = 'view', - tag = ['scheduled'] -) }} - -WITH api AS ( - - SELECT - NULL AS prices_swaps_hourly_id, - recorded_hour, - id, - token, - OPEN, - high, - low, - CLOSE, - provider, - NULL AS _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__prices_hourly') }} -), -swaps_cw AS ( - SELECT - NULL AS prices_swaps_hourly_id, - recorded_hour, - id, - CASE - WHEN id = 'A.1654653399040a61.FlowToken' THEN 'Flow' - WHEN id = 'A.cfdd90d4a00f7b5b.TeleportedTetherToken' THEN 'USDT' - WHEN id = 'A.3c5959b568896393.FUSD' THEN 'FUSD' - WHEN id = 'A.0f9df91c9121c460.BloctoToken' THEN 'Blocto' - WHEN id = 'A.d01e482eb680ec9f.REVV' THEN 'Revv' - WHEN id = 'A.b19436aae4d94622.FiatToken' THEN 'USDC' - WHEN id = 'A.142fa6570b62fd97.StarlyToken' THEN 'Starly' - WHEN id = 'A.475755d2c9dccc3a.TeleportedSportiumToken' THEN 'Sportium' - ELSE NULL -- will trigger alert if swaps model picks up another token - END AS token, - OPEN, - high, - low, - CLOSE, - provider, - NULL AS _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__prices_swaps_hourly') }} -), -swaps_s AS ( - SELECT - prices_swaps_hourly_id, - recorded_hour, - id, - CASE - WHEN id = 'A.1654653399040a61.FlowToken' THEN 'Flow' - WHEN id = 'A.cfdd90d4a00f7b5b.TeleportedTetherToken' THEN 'USDT' - WHEN id = 'A.3c5959b568896393.FUSD' THEN 'FUSD' - WHEN id = 'A.0f9df91c9121c460.BloctoToken' THEN 'Blocto' - WHEN id = 'A.d01e482eb680ec9f.REVV' THEN 'Revv' - WHEN id = 'A.b19436aae4d94622.FiatToken' THEN 'USDC' - WHEN id = 'A.142fa6570b62fd97.StarlyToken' THEN 'Starly' - WHEN id = 'A.475755d2c9dccc3a.TeleportedSportiumToken' THEN 'Sportium' - ELSE NULL -- will trigger alert if swaps model picks up another token - END AS token, - OPEN, - high, - low, - CLOSE, - provider, - NULL AS _inserted_timestamp, - COALESCE( - inserted_timestamp, - '2000-01-01' :: timestamp_ntz - ) AS inserted_timestamp, - COALESCE( - modified_timestamp, - '2000-01-01' :: timestamp_ntz - ) AS modified_timestamp - FROM - {{ ref('silver__prices_swaps_hourly_s') }} -), -FINAL AS ( - SELECT - * - FROM - api - UNION ALL - SELECT - * - FROM - swaps_cw - UNION ALL - SELECT - * - FROM - swaps_s -) -SELECT - COALESCE ( - prices_swaps_hourly_id, - {{ dbt_utils.generate_surrogate_key(['recorded_hour', 'token']) }} - ) AS fact_hourly_prices_id, - recorded_hour, - id, - token, - OPEN, - high, - low, - CLOSE, - provider, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp -FROM - FINAL -WHERE - recorded_hour IS NOT NULL diff --git a/models/gold/price/price__fact_hourly_prices.yml b/models/gold/price/price__fact_hourly_prices.yml deleted file mode 100644 index c9fd88f..0000000 --- a/models/gold/price/price__fact_hourly_prices.yml +++ /dev/null @@ -1,86 +0,0 @@ -version: 2 - -models: - - name: price__fact_hourly_prices - description: |- - This table provides hourly token price data for FLOW tokens from CoinGecko and CoinMarketCap. - - columns: - - name: RECORDED_HOUR - description: "{{ doc('recorded_hour') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: ID - description: "{{ doc('asset_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TOKEN - description: "{{ doc('token') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: OPEN - description: "{{ doc('open') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - FLOAT - - DOUBLE - - - name: HIGH - description: "{{ doc('high') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - FLOAT - - DOUBLE - - - name: LOW - description: "{{ doc('low') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - FLOAT - - DOUBLE - - - name: CLOSE - description: "{{ doc('asset_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - FLOAT - - DOUBLE - - - name: PROVIDER - description: "{{ doc('provider') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: FACT_HOURLY_PRICES_ID - description: "{{ doc('pk_id') }}" - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/price/price__fact_prices.sql b/models/gold/price/price__fact_prices.sql deleted file mode 100644 index f6c92aa..0000000 --- a/models/gold/price/price__fact_prices.sql +++ /dev/null @@ -1,132 +0,0 @@ -{{ config( - materialized = 'view', - tag = ['scheduled'] -) }} - -WITH token_labels AS ( - - SELECT - token, - UPPER(symbol) AS symbol, - token_contract - FROM - {{ ref('seeds__token_labels') }} -), -prices AS ( - SELECT - recorded_at AS TIMESTAMP, - token, - UPPER(symbol) AS symbol, - price_usd, - source - FROM - {{ this.database }}.silver.prices -), -prices_swaps_cw AS ( - SELECT - tx_id, - block_timestamp AS TIMESTAMP, - token_contract, - swap_price AS price_usd, - source, - _inserted_timestamp, - NULL AS prices_swaps_id, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__prices_swaps') }} -), -prices_swaps_s AS ( - SELECT - prices_swaps_id, - tx_id, - block_timestamp AS TIMESTAMP, - token_contract, - swap_price AS price_usd, - source, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - {{ ref('silver__prices_swaps_s') }} -), -viewnion AS ( - SELECT - TIMESTAMP, - p.token, - p.symbol, - l.token_contract, - price_usd, - source, - NULL AS tx_id, - NULL AS prices_swaps_id, - NULL AS _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - prices p - LEFT JOIN token_labels l USING (symbol) - UNION ALL - SELECT - TIMESTAMP, - l.token, - l.symbol, - ps.token_contract, - price_usd, - source, - tx_id, - prices_swaps_id, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - prices_swaps_cw ps - LEFT JOIN token_labels l USING (token_contract) - UNION ALL - SELECT - TIMESTAMP, - l.token, - l.symbol, - pss.token_contract, - price_usd, - source, - tx_id, - prices_swaps_id, - _inserted_timestamp, - NULL AS inserted_timestamp, - NULL AS modified_timestamp - FROM - prices_swaps_s pss - LEFT JOIN token_labels l USING (token_contract) -) -SELECT - TIMESTAMP, - token, - symbol, - token_contract, - price_usd, - source, - tx_id, - COALESCE ( - prices_swaps_id, - {{ dbt_utils.generate_surrogate_key(['TIMESTAMP','TOKEN', 'TOKEN_CONTRACT', 'SOURCE']) }} - ) AS fact_prices_id, - COALESCE ( - inserted_timestamp, - TIMESTAMP - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - TIMESTAMP - ) AS modified_timestamp -FROM - viewnion -WHERE - TIMESTAMP IS NOT NULL qualify ROW_NUMBER() over ( - PARTITION BY TIMESTAMP, - token, - token_contract, - source - ORDER BY - _inserted_timestamp DESC - ) = 1 diff --git a/models/gold/price/price__fact_prices.yml b/models/gold/price/price__fact_prices.yml deleted file mode 100644 index 52eee8d..0000000 --- a/models/gold/price/price__fact_prices.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 - -models: - - name: price__fact_prices - description: |- - This table reports prices derived from various on-chain sources. CoinGecko and CoinMarketCap price feeds can be found in the hourly price table. - Note that prices from swaps may be volatile and are an approximation of price in USD terms, at best. A tx_hash is included for prices from swaps to provide a source. - Low cap coins may have questionable prices due to low liquidity. - - columns: - - name: TIMESTAMP - description: "{{ doc('timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: TOKEN - description: "{{ doc('token') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: SYMBOL - description: "{{ doc('symbol') }}" - tests: - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: TOKEN_CONTRACT - description: "{{ doc('token_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: PRICE_USD - description: "{{ doc('price_usd') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - DOUBLE - - FLOAT - - - name: SOURCE - description: "{{ doc('source') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: FACT_PRICES_ID - description: "{{ doc('pk_id') }}" - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp') }}" diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.sql b/models/gold/price/price__fact_prices_ohlc_hourly.sql new file mode 100644 index 0000000..2be7d75 --- /dev/null +++ b/models/gold/price/price__fact_prices_ohlc_hourly.sql @@ -0,0 +1,19 @@ +{{ config( + materialized = 'view', + persist_docs ={ "relation": true, + "columns": true } +) }} + +SELECT + asset_id, + recorded_hour AS HOUR, + OPEN, + high, + low, + CLOSE, + provider, + inserted_timestamp, + modified_timestamp, + complete_provider_prices_id AS fact_prices_ohlc_hourly_id +FROM + {{ ref('silver__complete_provider_prices') }} diff --git a/models/gold/price/price__fact_prices_ohlc_hourly.yml b/models/gold/price/price__fact_prices_ohlc_hourly.yml new file mode 100644 index 0000000..7dc31b5 --- /dev/null +++ b/models/gold/price/price__fact_prices_ohlc_hourly.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: price__fact_prices_ohlc_hourly + description: '{{ doc("prices_fact_prices_ohlc_hourly_table_doc") }}' + + columns: + - name: HOUR + description: '{{ doc("prices_hour")}}' + - name: ASSET_ID + description: '{{ doc("prices_asset_id") }}' + - name: OPEN + description: '{{ doc("prices_open") }}' + - name: HIGH + description: '{{ doc("prices_high") }}' + - name: LOW + description: '{{ doc("prices_low") }}' + - name: CLOSE + description: '{{ doc("prices_close") }}' + - name: FACT_PRICES_OHLC_HOURLY_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql index dd20bed..056e948 100644 --- a/models/gold/stats/stats__ez_core_metrics_hourly.sql +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -14,7 +14,7 @@ WITH txs AS ( unique_from_count, total_fees AS total_fees_native, LAST_VALUE( - p.close ignore nulls + p.price ignore nulls ) over ( ORDER BY block_timestamp_hour rows unbounded preceding @@ -25,10 +25,10 @@ WITH txs AS ( FROM {{ ref('silver_stats__core_metrics_hourly') }} s - LEFT JOIN {{ ref('silver__prices_hourly') }} + LEFT JOIN {{ ref('silver__complete_native_prices') }} p - ON s.block_timestamp_hour = p.recorded_hour - AND p.id = 'Flow' + ON s.block_timestamp_hour = p.hour + AND p.asset_id = 'flow' ) SELECT A.block_timestamp_hour, @@ -41,7 +41,7 @@ SELECT b.unique_from_count, b.total_fees_native, ROUND( - b.total_fees_native * b.imputed_close, + b.total_fees_native * ZEROIFNULL(b.imputed_close), 2 ) AS total_fees_usd, A.core_metrics_block_hourly_id AS ez_core_metrics_hourly_id, diff --git a/models/silver/_observability/silver_observability__blocks_completeness.sql b/models/silver/_observability/silver_observability__blocks_completeness.sql index 5a77c36..b9f7be7 100644 --- a/models/silver/_observability/silver_observability__blocks_completeness.sql +++ b/models/silver/_observability/silver_observability__blocks_completeness.sql @@ -66,7 +66,7 @@ block_range AS ( _id AS block_height FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'number_sequence' ) }} WHERE @@ -111,7 +111,7 @@ block_gen AS ( _id AS block_height FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'number_sequence' ) }} WHERE diff --git a/models/silver/_observability/silver_observability__txs_completeness.sql b/models/silver/_observability/silver_observability__txs_completeness.sql index e1b6eb6..bf2f4ef 100644 --- a/models/silver/_observability/silver_observability__txs_completeness.sql +++ b/models/silver/_observability/silver_observability__txs_completeness.sql @@ -66,7 +66,7 @@ block_range AS ( _id AS block_height FROM {{ source( - 'crosschain_silver', + 'silver_crosschain', 'number_sequence' ) }} WHERE diff --git a/models/silver/core/silver__streamline_blocks.sql b/models/silver/core/silver__streamline_blocks.sql index 8359745..74b7d51 100644 --- a/models/silver/core/silver__streamline_blocks.sql +++ b/models/silver/core/silver__streamline_blocks.sql @@ -1,4 +1,6 @@ -- depends_on: {{ ref('bronze__streamline_blocks') }} +-- depends_on: {{ ref('bronze__streamline_blocks_history') }} + {{ config( materialized = 'incremental', unique_key = "block_number", @@ -10,7 +12,7 @@ WITH -{% if is_incremental() %} +{% if is_incremental() and not var('LOAD_BACKFILL', False) %} tx_count_lookback AS ( -- lookback to ensure tx count is correct @@ -21,7 +23,7 @@ tx_count_lookback AS ( WHERE block_height >= {{ var( 'STREAMLINE_START_BLOCK' - ) }} + ) }} -- TODO, remove AFTER backfill is complete -- limit to 3 day lookback for performance AND _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' AND ( @@ -47,7 +49,11 @@ streamline_blocks AS ( _partition_by_block_id, _inserted_timestamp FROM - +{% if var('LOAD_BACKFILL', False) %} + {{ ref('bronze__streamline_blocks_history') }} + -- TODO need incremental logic of some sort probably (for those 5800 missing txs) + -- where inserted timestamp >= max from this where network version = backfill version OR block range between root and end +{% else %} {% if is_incremental() %} {{ ref('bronze__streamline_blocks') }} WHERE @@ -66,6 +72,7 @@ WHERE {% else %} {{ ref('bronze__streamline_fr_blocks') }} {% endif %} +{% endif %} qualify(ROW_NUMBER() over (PARTITION BY block_number ORDER BY @@ -86,7 +93,14 @@ collections AS ( * FROM {{ ref('silver__streamline_collections') }} - +{% if var('LOAD_BACKFILL', False) %} +WHERE + block_number between ( + SELECT root_height FROM network_version WHERE lower(network_version) = lower('{{ var('LOAD_BACKFILL_VERSION').replace('_', '-') }}') + ) AND ( + SELECT end_height FROM network_version WHERE lower(network_version) = lower('{{ var('LOAD_BACKFILL_VERSION').replace('_', '-') }}') + ) +{% else %} {% if is_incremental() %} WHERE _inserted_timestamp >= ( @@ -102,6 +116,7 @@ WHERE tx_count_lookback ) {% endif %} +{% endif %} ), tx_count AS ( SELECT diff --git a/models/silver/core/silver__streamline_collections.sql b/models/silver/core/silver__streamline_collections.sql index 2f1f7af..d7d1bc4 100644 --- a/models/silver/core/silver__streamline_collections.sql +++ b/models/silver/core/silver__streamline_collections.sql @@ -25,6 +25,12 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM +{% if var('LOAD_BACKFILL', False) %} + {{ ref('bronze__streamline_collections_history') }} + -- TODO need incremental logic of some sort probably (for those 5800 missing txs) + -- where inserted timestamp >= max from this where network version = backfill version OR block range between root and end +{% else %} + {% if is_incremental() %} {{ ref('bronze__streamline_collections') }} WHERE @@ -38,6 +44,8 @@ WHERE {{ ref('bronze__streamline_fr_collections') }} {% endif %} +{% endif %} + qualify(ROW_NUMBER() over (PARTITION BY collection_id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__streamline_events.sql b/models/silver/core/silver__streamline_events.sql index faac045..341f15b 100644 --- a/models/silver/core/silver__streamline_events.sql +++ b/models/silver/core/silver__streamline_events.sql @@ -3,7 +3,8 @@ unique_key = 'event_id', incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - cluster_by = "_inserted_timestamp::date", + cluster_by = "block_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,event_id,event_contract,event_type);", tags = ['core', 'streamline_scheduled', 'scheduled', 'scheduled_core'] ) }} @@ -14,12 +15,12 @@ WITH transactions AS ( FROM {{ ref('silver__streamline_transactions_final') }} WHERE - NOT pending_result_response -- inserted timestamp will update w TR ingestion, so should flow thru to events and curated + NOT pending_result_response {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -75,6 +76,8 @@ flatten_events AS ( LATERAL FLATTEN( input => events ) e + QUALIFY ROW_NUMBER() OVER (PARTITION BY event_id ORDER BY _inserted_timestamp DESC) = 1 + ), attributes AS ( SELECT diff --git a/models/silver/core/silver__streamline_transaction_results.sql b/models/silver/core/silver__streamline_transaction_results.sql index 2436a39..48fe987 100644 --- a/models/silver/core/silver__streamline_transaction_results.sql +++ b/models/silver/core/silver__streamline_transaction_results.sql @@ -26,6 +26,12 @@ SELECT '{{ invocation_id }}' AS _invocation_id FROM +{% if var('LOAD_BACKFILL', False) %} + {{ ref('bronze__streamline_transaction_results_history') }} + -- TODO need incremental logic of some sort probably (for those 5800 missing txs) + -- where inserted timestamp >= max from this where network version = backfill version OR block range between root and end +{% else %} + {% if is_incremental() %} {{ ref('bronze__streamline_transaction_results') }} WHERE @@ -39,6 +45,8 @@ WHERE {{ ref('bronze__streamline_fr_transaction_results') }} {% endif %} +{% endif %} + qualify(ROW_NUMBER() over (PARTITION BY tx_id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__streamline_transactions.sql b/models/silver/core/silver__streamline_transactions.sql index d05fb23..cf7b680 100644 --- a/models/silver/core/silver__streamline_transactions.sql +++ b/models/silver/core/silver__streamline_transactions.sql @@ -30,6 +30,12 @@ SELECT _inserted_timestamp FROM +{% if var('LOAD_BACKFILL', False) %} + {{ ref('bronze__streamline_transactions_history') }} + -- TODO need incremental logic of some sort probably (for those 5800 missing txs) + -- where inserted timestamp >= max from this where network version = backfill version OR block range between root and end +{% else %} + {% if is_incremental() %} {{ ref('bronze__streamline_transactions') }} WHERE @@ -43,6 +49,8 @@ WHERE {{ ref('bronze__streamline_fr_transactions') }} {% endif %} +{% endif %} + qualify(ROW_NUMBER() over (PARTITION BY tx_id ORDER BY _inserted_timestamp DESC)) = 1 diff --git a/models/silver/core/silver__streamline_transactions_final.sql b/models/silver/core/silver__streamline_transactions_final.sql index f9a7339..9295568 100644 --- a/models/silver/core/silver__streamline_transactions_final.sql +++ b/models/silver/core/silver__streamline_transactions_final.sql @@ -4,7 +4,8 @@ unique_key = "tx_id", incremental_strategy = 'merge', merge_exclude_columns = ["inserted_timestamp"], - cluster_by = "_inserted_timestamp::date", + cluster_by = "block_timestamp::date", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,proposer,payer,authorizers);", tags = ['core', 'streamline_scheduled', 'scheduled', 'scheduled_core'] ) }} @@ -17,7 +18,7 @@ block_height FROM """ ~ this ~ """ WHERE - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + modified_timestamp >= SYSDATE() - INTERVAL '3 days' AND ( block_timestamp IS NULL OR pending_result_response @@ -30,17 +31,19 @@ %} {% set incr = "" %} {% if is_incremental() %} + {% set incr = """ + WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) _inserted_timestamp + MAX(modified_timestamp) modified_timestamp FROM """ ~ this ~ """ ) OR -- re-run record if block comes in later than tx records ( - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + modified_timestamp >= SYSDATE() - INTERVAL '3 days' AND tx_id IN ( SELECT @@ -50,7 +53,7 @@ ) ) """ %} - {% endif %} + {% endif %} {% set run = run_query(query ~ incr) %} {% endif %} @@ -60,7 +63,7 @@ Coalesce in case there are 0 txs returned by the temp table */ {% if execute %} - {% set min_time = run_query("select coalesce(min(_inserted_timestamp),current_timestamp()) from silver.streamline_transactions_final_intermediate_tmp").columns [0].values() [0] %} + {% set min_time = run_query("select coalesce(min(modified_timestamp),sysdate()) from silver.streamline_transactions_final_intermediate_tmp").columns [0].values() [0] %} {% endif %} WITH txs AS ( @@ -78,7 +81,7 @@ tx_results AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + modified_timestamp >= SYSDATE() - INTERVAL '3 days' AND tx_id IN ( SELECT DISTINCT tx_id @@ -86,6 +89,7 @@ WHERE silver.streamline_transactions_final_intermediate_tmp ) {% endif %} + ), blocks AS ( SELECT @@ -93,9 +97,10 @@ blocks AS ( FROM {{ ref('silver__streamline_blocks') }} + {% if is_incremental() %} WHERE - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + modified_timestamp >= SYSDATE() - INTERVAL '3 days' AND block_number IN ( SELECT DISTINCT block_number @@ -103,6 +108,7 @@ WHERE silver.streamline_transactions_final_intermediate_tmp ) {% endif %} + ), FINAL AS ( SELECT diff --git a/models/silver/defi/silver__swaps_events_s.sql b/models/silver/defi/silver__swaps_events_s.sql index 16ed125..d263242 100644 --- a/models/silver/defi/silver__swaps_events_s.sql +++ b/models/silver/defi/silver__swaps_events_s.sql @@ -9,16 +9,20 @@ WITH swaps_txs AS ( SELECT - * + block_height, + tx_id, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} WHERE event_contract LIKE '%SwapPair%' {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -26,7 +30,19 @@ AND _inserted_timestamp >= ( ), swap_events AS ( SELECT - * + tx_id, + block_height, + block_timestamp, + event_id, + event_index, + events_count, + payload, + event_contract, + event_type, + event_data, + tx_succeeded, + _inserted_timestamp, + _partition_by_block_id FROM {{ ref('silver__streamline_events') }} WHERE @@ -40,9 +56,9 @@ swap_events AS ( AND event_index < events_count - 3 {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) diff --git a/models/silver/defi/silver__swaps_final.sql b/models/silver/defi/silver__swaps_final.sql index 61fda66..d367813 100644 --- a/models/silver/defi/silver__swaps_final.sql +++ b/models/silver/defi/silver__swaps_final.sql @@ -4,6 +4,7 @@ merge_exclude_columns = ["inserted_timestamp"], cluster_by = ['block_timestamp::DATE'], unique_key = 'swaps_final_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,trader);", tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/defi/silver__swaps_s.sql b/models/silver/defi/silver__swaps_s.sql index eca6d66..8dc97e1 100644 --- a/models/silver/defi/silver__swaps_s.sql +++ b/models/silver/defi/silver__swaps_s.sql @@ -9,15 +9,23 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_index, + event_contract, + event_type, + event_data, + _inserted_timestamp, + _partition_by_block_id FROM {{ ref('silver__swaps_events_s') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -112,7 +120,8 @@ token_withdraws AS ( ) - 1 AS unique_order, event_contract, event_data, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM swap_events WHERE @@ -166,6 +175,7 @@ link_token_movement AS ( w.block_timestamp, w.block_height, w._inserted_timestamp, + w._partition_by_block_id, -- set transfer index based on execution via deposit, not withdraw, event RANK() over ( PARTITION BY w.tx_id @@ -283,6 +293,7 @@ boilerplate AS ( block_timestamp, block_height, _inserted_timestamp, + _partition_by_block_id, withdraw_from AS trader FROM link_token_movement @@ -310,6 +321,7 @@ FINAL AS ( tokens :token1 :: STRING AS token_in_contract, amounts :amount1 :: DOUBLE AS token_in_amount, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id', 'swap_index'] ) }} AS swaps_id, diff --git a/models/silver/defi/silver__swaps_single_trade_s.sql b/models/silver/defi/silver__swaps_single_trade_s.sql index 2ca4c73..bab3320 100644 --- a/models/silver/defi/silver__swaps_single_trade_s.sql +++ b/models/silver/defi/silver__swaps_single_trade_s.sql @@ -9,15 +9,23 @@ WITH swaps_events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_index, + event_contract, + event_type, + event_data, + _inserted_timestamp, + _partition_by_block_id FROM {{ ref('silver__swaps_events_s') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -97,7 +105,8 @@ token_out_data AS ( LOWER( event_data :from :: STRING ) AS trader_token_out, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM index_id ii LEFT JOIN swaps_single_trade sst USING ( @@ -120,7 +129,7 @@ trade_data AS ( _inserted_timestamp FROM swaps_single_trade sst - LEFT JOIN {{ ref('silver__contract_labels') }} + LEFT JOIN {{ ref('silver__contract_labels_s') }} l USING (event_contract) WHERE event_type = 'Trade' @@ -160,6 +169,7 @@ combo AS ( td.token_1_amount, td.token_2_amount, tod._inserted_timestamp, + tod._partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id'] ) }} AS swaps_single_trade_id, diff --git a/models/silver/labels/silver__contract_labels_s.sql b/models/silver/labels/silver__contract_labels_s.sql index 54e6daf..4294dd6 100644 --- a/models/silver/labels/silver__contract_labels_s.sql +++ b/models/silver/labels/silver__contract_labels_s.sql @@ -14,15 +14,17 @@ WITH splt AS ( event_contract, '.' ) AS ec_s, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp AS _modified_timestamp FROM {{ ref('silver__streamline_events') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) _inserted_timestamp + MAX(modified_timestamp) FROM {{ this }} ) @@ -36,7 +38,9 @@ FINAL AS ( '0x', ec_s [array_size(ec_s)-2] :: STRING ) AS account_address, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id, + _modified_timestamp FROM splt WHERE @@ -54,5 +58,5 @@ FROM FINAL qualify ROW_NUMBER() over ( PARTITION BY event_contract ORDER BY - _inserted_timestamp DESC + _modified_timestamp DESC ) = 1 diff --git a/models/silver/labels/silver__labels_pools_metapier_s.sql b/models/silver/labels/silver__labels_pools_metapier_s.sql index 35ad0e1..434e85f 100644 --- a/models/silver/labels/silver__labels_pools_metapier_s.sql +++ b/models/silver/labels/silver__labels_pools_metapier_s.sql @@ -9,15 +9,24 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -56,7 +65,8 @@ token_withdraws AS ( block_timestamp, event_contract, event_index, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM pier_events WHERE @@ -73,7 +83,8 @@ pairs AS ( ORDER BY event_index ) AS token1_contract, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM token_withdraws ), @@ -106,6 +117,7 @@ FINAL AS ( e.vault_address, pa.swap_contract, C._inserted_timestamp, + C._partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id'] ) }} AS labels_pools_metapier_id, diff --git a/models/silver/labels/silver__labels_pools_s.sql b/models/silver/labels/silver__labels_pools_s.sql index 115139c..5f7bafe 100644 --- a/models/silver/labels/silver__labels_pools_s.sql +++ b/models/silver/labels/silver__labels_pools_s.sql @@ -9,15 +9,24 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -27,7 +36,7 @@ pair_labels AS ( SELECT * FROM - {{ ref('silver__contract_labels') }} + {{ ref('silver__contract_labels_s') }} WHERE contract_name ILIKE '%swappair%' ), @@ -40,7 +49,8 @@ pair_creation AS ( event_data :pairAddress :: STRING AS account_address, event_data :token0Key :: STRING AS token0_contract, event_data :token1Key :: STRING AS token1_contract, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM events WHERE @@ -61,7 +71,8 @@ FINAL AS ( SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM pair_creation p LEFT JOIN pair_labels l USING (account_address) diff --git a/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py index ebcc8d1..847e0f4 100644 --- a/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py +++ b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py @@ -67,7 +67,7 @@ def model(dbt, session): inputs = dbt.ref( 'livequery__topshot_moments_metadata_needed').select( "EVENT_CONTRACT", "MOMENT_ID" - ).limit(600) + ).limit(100) # Note prior limit of 3500 leads to 429 error / rate limit by system # Per Dapper team, 50 reqs per 10 seconds. If exceeded, blocked for 30s. diff --git a/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql b/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql index 1737679..cae9d64 100644 --- a/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql +++ b/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql @@ -9,7 +9,7 @@ WITH mints AS ( event_contract, event_data :momentID :: STRING AS moment_id FROM - {{ ref('silver__nft_moments') }} + {{ ref('silver__nft_moments_s') }} WHERE event_contract = 'A.0b2a3299cc857e29.TopShot' AND event_type = 'MomentMinted' @@ -19,7 +19,7 @@ sales AS ( nft_collection AS event_contract, nft_id AS moment_id FROM - {{ ref('silver__nft_sales') }} + {{ ref('silver__nft_sales_s') }} WHERE nft_collection ILIKE '%topshot%' ), diff --git a/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql b/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql index 154d8a3..c59f2c5 100644 --- a/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql +++ b/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql @@ -3,7 +3,8 @@ incremental_strategy = 'delete+insert', cluster_by = ['_inserted_timestamp::DATE'], unique_key = 'nft_id', - tags = ['livequery'], + tags = ['livequery', 'topshot'], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(nft_id,nbatopshot_id);", full_refresh = False ) }} {# NFT Metadata from legacy process lives in external table, deleted CTE and set FR=False diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_editions_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_editions_s.sql index 9e68670..f4764c6 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_editions_s.sql +++ b/models/silver/nft/metadata/onchain/silver__nft_moment_editions_s.sql @@ -9,42 +9,50 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} WHERE event_type = 'EditionCreated' - AND event_contract IN ( - 'A.e4cf4bdc1751c65d.AllDay', - 'A.b715b81853fef53f.AllDay', - 'A.87ca73a41bb50ad5.Golazos' - ) {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS edition_id, - event_data :maxMintSize :: STRING AS max_mint_size, - event_data :playID :: STRING AS play_id, - event_data :seriesID :: STRING AS series_id, - event_data :setID :: STRING AS set_id, - event_data :tier :: STRING AS tier, - _inserted_timestamp - FROM - events ) SELECT - * + tx_id, + event_id, + block_timestamp, + event_contract, + event_data AS edition_data, + event_data :id :: STRING AS edition_id, + event_data :maxMintSize :: STRING AS max_mint_size, + event_data :playID :: STRING AS play_id, + event_data :seriesID :: STRING AS series_id, + event_data :setID :: STRING AS set_id, + event_data :tier :: STRING AS tier, + _inserted_timestamp, + _partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS nft_moment_editions_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - org + events diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_final_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_final_s.sql index f0395c2..dbdaca3 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_final_s.sql +++ b/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_final_s.sql @@ -5,7 +5,7 @@ tags = ['nft', 'dapper', 'scheduled', 'streamline_scheduled', 'scheduled_non_core'] ) }} -WITH moments AS ( +WITH moments_minted AS ( SELECT * @@ -30,7 +30,7 @@ series AS ( FROM {{ ref('silver__nft_moment_series_s') }} ), -set_nm AS ( +nft_sets AS ( SELECT * FROM @@ -38,47 +38,69 @@ set_nm AS ( ), FINAL AS ( SELECT - -- tx id and block timestamp don't matter for the final table m.tx_id, m.block_timestamp, m.event_contract, m.nft_id, m.serial_number, e.max_mint_size, - e.play_id, - e.series_id, + COALESCE( + m.play_id, + e.play_id + ) AS play_id, + COALESCE( + m.series_id, + e.series_id, + sn.series_id + ) AS series_id, s.series_name, - e.set_id, + COALESCE( + m.set_id, + e.set_id + ) AS set_id, sn.set_name, - e.edition_id, + COALESCE( + m.edition_id, + e.edition_id + ) AS edition_id, e.tier, pl.metadata, + set_data, + series_data, + edition_data, m._inserted_timestamp, sn._inserted_timestamp AS _inserted_timestamp_set FROM - moments m - LEFT JOIN editions e USING ( - event_contract, - edition_id - ) - LEFT JOIN metadata pl USING ( - event_contract, - play_id - ) - LEFT JOIN series s USING ( - event_contract, - series_id - ) - LEFT JOIN set_nm sn USING ( - event_contract, - set_id - ) + moments_minted m + LEFT JOIN editions e + ON m.event_contract = e.event_contract + AND m.edition_id = e.edition_id + LEFT JOIN metadata pl + ON m.event_contract = pl.event_contract + AND COALESCE( + m.play_id, + e.play_id + ) = pl.play_id + LEFT JOIN nft_sets sn + ON m.event_contract = sn.event_contract + AND COALESCE( + m.set_id, + e.set_id + ) = sn.set_id + LEFT JOIN series s + ON m.event_contract = s.event_contract + AND COALESCE( + m.series_id, + e.series_id, + sn.series_id + ) = s.series_id + ) SELECT *, {{ dbt_utils.generate_surrogate_key( - ['event_contract','edition_id','nft_id'] - ) }} AS nft_moment_metadata_id, + ['event_contract','nft_id', 'play_id'] + ) }} AS nft_moment_metadata_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_s.sql index 3fb7299..c0efb1b 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_s.sql +++ b/models/silver/nft/metadata/onchain/silver__nft_moment_metadata_s.sql @@ -9,7 +9,17 @@ WITH play_creation AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} WHERE @@ -28,9 +38,9 @@ A.87ca73a41bb50ad5.Golazos {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -39,12 +49,14 @@ AND _inserted_timestamp >= ( play_metadata AS ( SELECT tx_id, + event_id, block_timestamp, event_contract, event_data :id :: NUMBER AS play_id, VALUE :key :value :: STRING AS column_header, VALUE :value :value :: STRING AS column_value, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM play_creation, LATERAL FLATTEN(input => TRY_PARSE_JSON(event_data :metadata)) @@ -52,10 +64,12 @@ play_metadata AS ( FINAL AS ( SELECT tx_id, + event_id, block_timestamp, event_contract, play_id, _inserted_timestamp, + _partition_by_block_id, OBJECT_AGG( column_header :: variant, column_value :: variant @@ -67,9 +81,17 @@ FINAL AS ( 2, 3, 4, - 5 + 5, + 6, + 7 ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS nft_moment_metadata_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM FINAL diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_minted_2_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_minted_2_s.sql deleted file mode 100644 index 1df7d8a..0000000 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_minted_2_s.sql +++ /dev/null @@ -1,43 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, moment_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'streamline_scheduled', 'scheduled_non_core'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__streamline_events') }} - WHERE - event_type = 'MomentMinted' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :momentID :: STRING AS moment_id, - event_data :serialNumber :: STRING AS serial_number, - event_data :seriesID :: STRING AS series_id, - event_data :setID :: STRING AS set_id, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_minted_2_s.yml b/models/silver/nft/metadata/onchain/silver__nft_moment_minted_2_s.yml deleted file mode 100644 index 5c6bd03..0000000 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_minted_2_s.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_minted_2_s - description: |- - Cleaned MomentNFTMinted events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - moment_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.c38aea683c0c4d38.Eternal" - - "A.d4ad4740ee426334.Moments" - - "A.67af7ecf76556cd3.ABD" - - "A.0b2a3299cc857e29.TopShot" - - name: moment_id - description: "{{ doc('moment_id') }}" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_minted_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_minted_s.sql index 94e1008..e017c7d 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_minted_s.sql +++ b/models/silver/nft/metadata/onchain/silver__nft_moment_minted_s.sql @@ -9,34 +9,59 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} WHERE - event_type = 'MomentNFTMinted' + event_type IN ( + 'MomentNFTMinted', + 'MomentMinted' + ) {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :editionID :: STRING AS edition_id, - event_data :id :: STRING AS nft_id, - event_data :serialNumber :: STRING AS serial_number, - _inserted_timestamp - FROM - events ) SELECT - * + tx_id, + event_id, + block_timestamp, + event_contract, + event_data, + event_data :editionID :: STRING AS edition_id, + event_data :subeditionID :: STRING AS subedition_id, + event_data :seriesID :: STRING AS series_id, + event_data :setID :: STRING AS set_id, + COALESCE( + event_data :id, + event_data :momentID + ) :: STRING AS nft_id, + event_data :serialNumber :: STRING AS serial_number, + event_data :playID :: STRING AS play_id, + event_data :contentEditionID :: STRING AS content_edition_id, + event_data :contentID :: STRING AS content_id, + _inserted_timestamp, + _partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS nft_moment_minted_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - org + events diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_series_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_series_s.sql index 91a363b..26e51e9 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_series_s.sql +++ b/models/silver/nft/metadata/onchain/silver__nft_moment_series_s.sql @@ -9,34 +9,46 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} WHERE event_type = 'SeriesCreated' - AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data)) {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS series_id, - event_data :name :: STRING AS series_name, - _inserted_timestamp - FROM - events ) SELECT - * + tx_id, + event_id, + block_timestamp, + event_contract, + event_data AS series_data, + event_data :id :: STRING AS series_id, + event_data :name :: STRING AS series_name, + _inserted_timestamp, + _partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS nft_moment_series_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - org + events diff --git a/models/silver/nft/metadata/onchain/silver__nft_moment_set_s.sql b/models/silver/nft/metadata/onchain/silver__nft_moment_set_s.sql index 39d69f6..e31d0ea 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moment_set_s.sql +++ b/models/silver/nft/metadata/onchain/silver__nft_moment_set_s.sql @@ -9,36 +9,52 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} WHERE event_type = 'SetCreated' - AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data)) {% if is_incremental() %} -AND _inserted_timestamp >= ( +AND modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) {% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS set_id, - event_data :name :: STRING AS set_name, - _inserted_timestamp - FROM - events - WHERE - set_id IS NOT NULL ) SELECT - * + tx_id, + event_id, + block_timestamp, + event_contract, + event_data AS set_data, + COALESCE( + event_data :id :: STRING, + event_data :setID :: STRING + ) AS set_id, + event_data :name :: STRING AS set_name, + event_data :series :: STRING AS series_id, + _inserted_timestamp, + _partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS nft_moment_set_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM - org + events +WHERE + set_id IS NOT NULL diff --git a/models/silver/nft/metadata/silver__nft_allday_metadata.sql b/models/silver/nft/metadata/silver__nft_allday_metadata.sql index e07f0b1..b4ab825 100644 --- a/models/silver/nft/metadata/silver__nft_allday_metadata.sql +++ b/models/silver/nft/metadata/silver__nft_allday_metadata.sql @@ -6,6 +6,7 @@ ) }} {# Note - removed schedule tag as the legacy lambda workflow is inactive. No need to query external table #} +{# Not updating incremental to modts logic due to above comment JMF 6/7/2024 #} WITH metadata AS ( SELECT diff --git a/models/silver/nft/metadata/onchain/silver__nft_moments_s.sql b/models/silver/nft/metadata/silver__nft_moments_s.sql similarity index 59% rename from models/silver/nft/metadata/onchain/silver__nft_moments_s.sql rename to models/silver/nft/metadata/silver__nft_moments_s.sql index 8bfd1c2..3839d89 100644 --- a/models/silver/nft/metadata/onchain/silver__nft_moments_s.sql +++ b/models/silver/nft/metadata/silver__nft_moments_s.sql @@ -6,28 +6,21 @@ tags = ['nft', 'scheduled', 'streamline_scheduled', 'scheduled_non_core'] ) }} -WITH events AS ( +WITH moment_events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id FROM {{ ref('silver__streamline_events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -moment_events AS ( - SELECT - * - FROM - events WHERE event_type IN ( 'MomentPurchased', @@ -40,8 +33,23 @@ moment_events AS ( 'MomentMinted', 'MomentNFTMinted' ) +{% if is_incremental() %} +AND + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} ) SELECT - * + *, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS nft_moments_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id FROM moment_events diff --git a/models/silver/nft/metadata/silver__nft_ufc_strike_metadata.sql b/models/silver/nft/metadata/silver__nft_ufc_strike_metadata.sql index 79427a9..440c488 100644 --- a/models/silver/nft/metadata/silver__nft_ufc_strike_metadata.sql +++ b/models/silver/nft/metadata/silver__nft_ufc_strike_metadata.sql @@ -3,6 +3,7 @@ incremental_strategy = 'merge', unique_key = ['nft_id'], merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(nft_id);", tags = ['scheduled_non_core'] ) }} diff --git a/models/silver/nft/silver__nft_allday_metadata_s.sql b/models/silver/nft/silver__nft_allday_metadata_s.sql index 75b94e9..c958df1 100644 --- a/models/silver/nft/silver__nft_allday_metadata_s.sql +++ b/models/silver/nft/silver__nft_allday_metadata_s.sql @@ -4,6 +4,7 @@ cluster_by = ['_inserted_timestamp::DATE'], merge_exclude_columns = ["inserted_timestamp"], unique_key = 'nft_allday_metadata_s_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(nft_id,nflallday_id);", tags = ['nft'] ) }} diff --git a/models/silver/nft/silver__nft_giglabs_s.sql b/models/silver/nft/silver__nft_giglabs_s.sql index 8ca3c8b..7c4d9d3 100644 --- a/models/silver/nft/silver__nft_giglabs_s.sql +++ b/models/silver/nft/silver__nft_giglabs_s.sql @@ -9,15 +9,25 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + tx_succeeded, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -36,9 +46,9 @@ mapped_sales AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -54,7 +64,16 @@ duc AS ( ), duc_events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM events WHERE @@ -132,6 +151,7 @@ missing_contract AS ( block_height, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, event_contract AS currency, event_data :amount :: DOUBLE AS amount, event_data :from :: STRING AS forwarded_from, @@ -149,6 +169,7 @@ purchase_amt AS ( block_height, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, 'A.ead892083b3e2c6c.DapperUtilityCoin' AS currency, event_data :amount :: DOUBLE AS amount, event_data :from :: STRING AS forwarded_from, @@ -212,6 +233,7 @@ gl_sales AS ( p.block_height, p.tx_succeeded, p._inserted_timestamp, + p._partition_by_block_id, 'Gigantik Primary Market' AS marketplace, p.missing, p.currency, @@ -266,7 +288,8 @@ giglabs_final AS ( withdraw_nft_id AS nft_id, m.nfts, tx_succeeded, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM gl_sales s LEFT JOIN multi m USING (tx_id) @@ -330,6 +353,7 @@ FINAL AS ( counterparties, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id diff --git a/models/silver/nft/silver__nft_sales_s.sql b/models/silver/nft/silver__nft_sales_s.sql index c693b83..8656928 100644 --- a/models/silver/nft/silver__nft_sales_s.sql +++ b/models/silver/nft/silver__nft_sales_s.sql @@ -1,8 +1,9 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], + cluster_by = ['block_timestamp::DATE'], unique_key = 'tx_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,nft_id,buyer,seller);", tags = ['nft', 'scheduled', 'streamline_scheduled', 'scheduled_non_core'] ) }} @@ -15,9 +16,9 @@ WITH topshot AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -31,9 +32,9 @@ secondary_mkts AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -47,9 +48,9 @@ giglabs AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -69,6 +70,7 @@ combo AS ( currency, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id','seller', 'buyer', 'nft_collection', 'nft_id'] ) }} AS nft_sales_id, @@ -79,7 +81,7 @@ combo AS ( counterparties FROM topshot - UNION + UNION ALL SELECT tx_id, block_height, @@ -93,6 +95,7 @@ combo AS ( currency, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id','seller', 'buyer', 'nft_collection', 'nft_id'] ) }} AS nft_sales_id, @@ -103,7 +106,7 @@ combo AS ( counterparties FROM secondary_mkts - UNION + UNION ALL SELECT tx_id, block_height, @@ -117,6 +120,7 @@ combo AS ( currency, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id','seller', 'buyer', 'nft_collection', 'nft_id'] ) }} AS nft_sales_id, diff --git a/models/silver/nft/silver__nft_topshot_sales_s.sql b/models/silver/nft/silver__nft_topshot_sales_s.sql index 3787826..87f823a 100644 --- a/models/silver/nft/silver__nft_topshot_sales_s.sql +++ b/models/silver/nft/silver__nft_topshot_sales_s.sql @@ -9,7 +9,17 @@ WITH silver_events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + tx_succeeded, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} -- WHERE @@ -17,9 +27,9 @@ WITH silver_events AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -35,7 +45,8 @@ moment_data AS ( event_data :price :: DOUBLE AS price, event_data :seller :: STRING AS seller, tx_succeeded, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM silver_events WHERE @@ -86,7 +97,8 @@ combo AS ( price, currency, tx_succeeded, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM moment_data LEFT JOIN currency_data USING (tx_id) @@ -143,6 +155,7 @@ FINAL AS ( currency, tx_succeeded, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id'] ) }} AS nft_topshot_sales_id, diff --git a/models/silver/nft/silver__nft_transactions_secondary_market_s.sql b/models/silver/nft/silver__nft_transactions_secondary_market_s.sql index 7a95dd1..61d19c9 100644 --- a/models/silver/nft/silver__nft_transactions_secondary_market_s.sql +++ b/models/silver/nft/silver__nft_transactions_secondary_market_s.sql @@ -9,15 +9,25 @@ WITH silver_events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + tx_succeeded, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -42,7 +52,8 @@ sale_trigger AS ( ), TRUE ) AS is_purchased, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM silver_events WHERE @@ -389,7 +400,8 @@ nft_sales AS ( b.nft_collection_deposit, b.nft_id_deposit, b.buyer_deposit, - e._inserted_timestamp + e._inserted_timestamp, + e._partition_by_block_id FROM sale_trigger e LEFT JOIN token_withdraw_event w USING (tx_id) @@ -470,7 +482,8 @@ FINAL AS ( cd.step_data, cd.counterparties, tx_succeeded, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM nft_sales ns LEFT JOIN counterparty_data cd USING (tx_id) diff --git a/models/silver/price/silver__prices.sql b/models/silver/price/silver__prices.sql deleted file mode 100644 index b782d62..0000000 --- a/models/silver/price/silver__prices.sql +++ /dev/null @@ -1,78 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['recorded_at::DATE'], - unique_key = "concat_ws( '-', recorded_at, asset_id )", - enabled = False -) }} - -WITH token_prices AS ( - - SELECT - * - FROM - {# LEGACY FLIPSIDE_PROD_DB DEPRECATED. - No data since Feb 2023, disabling table and updating from ref to direct select on table of historical data living in SF #} - {{ ref('bronze__prices') }} - -{% if is_incremental() %} -WHERE - recorded_at :: DATE >= CURRENT_DATE - 2 -{% endif %} -), -prices AS ( - SELECT - recorded_at, - asset_id, - COALESCE( - NAME, - INITCAP(SPLIT(asset_id, '-') [0]) - ) AS token, - SPLIT( - symbol, - '$' - ) AS symbol_split, - symbol_split [array_size(symbol_split) - 1] :: STRING AS symbol, - price, - provider AS source - FROM - token_prices -), -adj_token_names AS ( - SELECT - recorded_at, - asset_id, - CASE - WHEN token LIKE 'Flow (%' THEN 'Flow' - WHEN token = 'Blocto Token' THEN 'Blocto' - ELSE token - END AS token, - COALESCE( - symbol, - CASE - WHEN token = 'Flow' THEN 'FLOW' - WHEN token = 'Blocto' THEN 'BLT' - WHEN token = 'Starly' THEN 'STARLY' - ELSE 'Error' - END - ) AS symbol, - price, - source - FROM - prices -), -FINAL AS ( - SELECT - recorded_at, - asset_id, - token, - symbol, - price AS price_usd, - source - FROM - adj_token_names -) -SELECT - * -FROM - FINAL diff --git a/models/silver/price/silver__prices.yml b/models/silver/price/silver__prices.yml deleted file mode 100644 index 51f438e..0000000 --- a/models/silver/price/silver__prices.yml +++ /dev/null @@ -1,63 +0,0 @@ - -version: 2 - -models: - - name: silver__prices - description: |- - This table provides token price data for FLOW tokens. - - columns: - - name: recorded_at - description: "{{ doc('recorded_at') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: asset_id - description: "{{ doc('asset_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - NUMBER - - VARCHAR - - - name: token - description: "{{ doc('token') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: symbol - description: "{{ doc('symbol') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: price_usd - description: "{{ doc('price_usd') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - DOUBLE - - - name: source - description: "{{ doc('source') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR diff --git a/models/silver/price/silver__prices_hourly.sql b/models/silver/price/silver__prices_hourly.sql deleted file mode 100644 index ea0e593..0000000 --- a/models/silver/price/silver__prices_hourly.sql +++ /dev/null @@ -1,82 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = "concat_ws( '-', recorded_hour, id )", - tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core'] -) }} --- model named prices hourly but core view is hourly prices --- this is to organize the silver model with other prices models -WITH token_prices AS ( - - SELECT - * - FROM - {{ ref('bronze__prices_v2') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -prices AS ( - SELECT - recorded_hour, - id AS asset_id, - INITCAP(SPLIT(asset_id, '-') [0]) AS token, - OPEN, - high, - low, - CLOSE, - provider, - _inserted_timestamp - FROM - token_prices -), -adj_token_names AS ( - SELECT - recorded_hour, - asset_id, - CASE - WHEN token LIKE 'Flow (%' THEN 'Flow' - WHEN token = 'Blocto Token' THEN 'Blocto' - WHEN token = '4558' THEN 'Flow' - WHEN token = '6993' THEN 'Revv' - WHEN token = '8075' THEN 'Rally' - WHEN token = '12182' THEN 'Blocto' - WHEN token = '15139' THEN 'Starly' - WHEN token = '15194' THEN 'Sportium' - ELSE token - END AS token, - OPEN, - high, - low, - CLOSE, - provider, - _inserted_timestamp - FROM - prices -), -FINAL AS ( - SELECT - recorded_hour, - asset_id AS id, - token, - OPEN, - high, - low, - CLOSE, - provider, - _inserted_timestamp - FROM - adj_token_names -) -SELECT - * -FROM - FINAL diff --git a/models/silver/price/silver__prices_hourly.yml b/models/silver/price/silver__prices_hourly.yml deleted file mode 100644 index 83002bf..0000000 --- a/models/silver/price/silver__prices_hourly.yml +++ /dev/null @@ -1,32 +0,0 @@ - -version: 2 - -models: - - name: silver__prices_hourly - description: |- - This table provides hourly token price data for FLOW tokens from CoinGecko and CoinMarketCap. - - columns: - - name: recorded_hour - description: "{{ doc('recorded_hour') }}" - - - name: id - description: "{{ doc('asset_id') }}" - - - name: token - description: "{{ doc('token') }}" - - - name: open - description: "{{ doc('open') }}" - - - name: high - description: "{{ doc('high') }}" - - - name: low - description: "{{ doc('low') }}" - - - name: close - description: "{{ doc('asset_id') }}" - - - name: provider - description: "{{ doc('provider') }}" diff --git a/models/silver/price/silver__prices_swaps_hourly_s.sql b/models/silver/price/silver__prices_swaps_hourly_s.sql deleted file mode 100644 index 946017f..0000000 --- a/models/silver/price/silver__prices_swaps_hourly_s.sql +++ /dev/null @@ -1,84 +0,0 @@ -{{ config( - materialized = 'table', - cluster_by = ['recorded_hour::date'], - unique_key = "CONCAT_WS('-', recorded_hour, token)", - tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core'] -) }} - -WITH swap_prices AS ( - - SELECT - * - FROM - {{ ref('silver__prices_swaps_s') }} - -), -lowhigh AS ( - SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS recorded_hour, - token_contract AS token, - MIN(swap_price) AS low, - MAX(swap_price) AS high, - COUNT( - DISTINCT tx_id - ) AS num_swaps - FROM - swap_prices - GROUP BY - 1, - 2 -), -openclose AS ( - SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS recorded_hour, - token_contract AS token, - FIRST_VALUE(swap_price) over ( - PARTITION BY recorded_hour, - token - ORDER BY - block_timestamp - ) AS OPEN, - LAST_VALUE(swap_price) over ( - PARTITION BY recorded_hour, - token - ORDER BY - block_timestamp - ) AS CLOSE - FROM - swap_prices qualify ROW_NUMBER() over (PARTITION BY concat_ws('-', recorded_hour, token) - ORDER BY - recorded_hour DESC) = 1 -), -FINAL AS ( - SELECT - l.recorded_hour, - l.token as id, - OPEN, - high, - low, - CLOSE, - num_swaps, - 'Swaps' as provider, - {{ dbt_utils.generate_surrogate_key( - ['recorded_hour', 'token'] - ) }} AS prices_swaps_hourly_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id - FROM - lowhigh l - LEFT JOIN openclose o USING ( - recorded_hour, - token - ) -) -SELECT - * -FROM - FINAL diff --git a/models/silver/price/silver__prices_swaps_hourly_s.yml b/models/silver/price/silver__prices_swaps_hourly_s.yml deleted file mode 100644 index 6e33a29..0000000 --- a/models/silver/price/silver__prices_swaps_hourly_s.yml +++ /dev/null @@ -1,43 +0,0 @@ -version: 2 - -models: - - name: silver__prices_swaps_hourly_s - description: |- - This table provides hourly token price data for FLOW tokens from on-chain swaps - - columns: - - name: recorded_hour - description: "{{ doc('recorded_hour') }}" - - - name: id - description: "{{ doc('asset_id') }}" - - - name: token - description: "{{ doc('token') }}" - - - name: open - description: "{{ doc('open') }}" - - - name: high - description: "{{ doc('high') }}" - - - name: low - description: "{{ doc('low') }}" - - - name: close - description: "{{ doc('asset_id') }}" - - - name: provider - description: "{{ doc('provider') }}" - - - name: prices_swaps_hourly_id - description: "{{ doc('pk_id') }}" - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp') }}" - - - name: _INVOCATION_ID - description: "{{ doc('invocation_id') }}" diff --git a/models/silver/price/silver__prices_swaps_s.sql b/models/silver/price/silver__prices_swaps_s.sql deleted file mode 100644 index 182e055..0000000 --- a/models/silver/price/silver__prices_swaps_s.sql +++ /dev/null @@ -1,187 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::date'], - unique_key = "CONCAT_WS('-', block_timestamp, token_contract)", - tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core'] -) }} - -WITH swaps AS ( - - SELECT - tx_id, - block_timestamp, - block_height, - swap_contract, - trader, - token_out_amount, - token_out_contract, - token_in_amount, - token_in_contract, - _inserted_timestamp - FROM - {{ ref('silver__swaps_s') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -flow_price AS ( - SELECT - recorded_hour as _timestamp, - open as price_usd - FROM - {{ ref('silver__prices_hourly') }} - WHERE - token = 'Flow' -), -stable_out AS ( - SELECT - tx_id, - block_timestamp, - token_in_contract AS token_contract, - token_out_amount / nullifzero(token_in_amount) AS swap_price, - _inserted_timestamp, - 'stableswap' AS source - FROM - swaps - WHERE - token_out_contract IN ( - 'A.cfdd90d4a00f7b5b.TeleportedTetherToken', - 'A.3c5959b568896393.FUSD', - 'A.b19436aae4d94622.FiatToken' - ) -), -stable_in AS ( - SELECT - tx_id, - block_timestamp, - token_out_contract AS token_contract, - token_in_amount / nullifzero(token_out_amount) AS swap_price, - _inserted_timestamp, - 'stableswap' AS source - FROM - swaps - WHERE - token_in_contract IN ( - 'A.cfdd90d4a00f7b5b.TeleportedTetherToken', - 'A.3c5959b568896393.FUSD', - 'A.b19436aae4d94622.FiatToken' - ) -), -stbl_tbl_union AS ( - SELECT - * - FROM - stable_out - UNION - SELECT - * - FROM - stable_in -), -flow_out AS ( - SELECT - tx_id, - block_timestamp, - token_in_contract AS token_contract, - token_out_amount / nullifzero(token_in_amount) AS swap_price_in_flow, - _inserted_timestamp - FROM - swaps - WHERE - token_out_contract = 'A.1654653399040a61.FlowToken' -), -flow_in AS ( - SELECT - tx_id, - block_timestamp, - token_in_contract AS token_contract, - token_out_amount / nullifzero(token_in_amount) AS swap_price_in_flow, - _inserted_timestamp - FROM - swaps - WHERE - token_out_contract = 'A.1654653399040a61.FlowToken' -), -flow_tbl_union AS ( - SELECT - tx_id, - DATE_TRUNC( - 'hour', - block_timestamp - ) AS _timestamp, - token_contract, - swap_price_in_flow, - _inserted_timestamp, - 'flowswap' AS source - FROM - flow_out - UNION - SELECT - tx_id, - DATE_TRUNC( - 'hour', - block_timestamp - ) AS _timestamp, - token_contract, - swap_price_in_flow, - _inserted_timestamp, - 'flowswap' AS source - FROM - flow_in -), -to_usd AS ( - SELECT - tx_id, - ftu._timestamp, - token_contract, - swap_price_in_flow, - swap_price_in_flow * p.price_usd AS swap_price_usd, - _inserted_timestamp, - source - FROM - flow_tbl_union ftu - LEFT JOIN flow_price p USING (_timestamp) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - token_contract, - swap_price, - _inserted_timestamp, - source - FROM - stbl_tbl_union - UNION - SELECT - tx_id, - _timestamp AS block_timestamp, - token_contract, - swap_price_usd AS swap_price, - _inserted_timestamp, - source - FROM - to_usd - WHERE - swap_price IS NOT NULL -) -SELECT - *, - {{ dbt_utils.generate_surrogate_key( - ['block_timestamp', 'token_contract'] - ) }} AS prices_swaps_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id -FROM - FINAL -WHERE - swap_price IS NOT NULL diff --git a/models/silver/price/silver__prices_swaps_s.yml b/models/silver/price/silver__prices_swaps_s.yml deleted file mode 100644 index 0e1dc98..0000000 --- a/models/silver/price/silver__prices_swaps_s.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2 - -models: - - name: silver__prices_swaps_s - description: |- - This table provides token price data for FLOW tokens. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARCHAR - - STRING - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: token_contract - description: "{{ doc('token_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARCHAR - - STRING - - - name: swap_price - description: "{{ doc('swap_price') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - FLOAT - - DOUBLE - - NUMBER - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - - - name: source - description: "{{ doc('source') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: prices_swaps_id - description: "{{ doc('pk_id') }}" - - - name: INSERTED_TIMESTAMP - description: "{{ doc('inserted_timestamp') }}" - - - name: MODIFIED_TIMESTAMP - description: "{{ doc('modified_timestamp') }}" - - - name: _INVOCATION_ID - description: "{{ doc('invocation_id') }}" diff --git a/models/silver/prices/silver__complete_native_asset_metadata.sql b/models/silver/prices/silver__complete_native_asset_metadata.sql new file mode 100644 index 0000000..98055f7 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_asset_metadata_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + asset_id, + symbol, + NAME, + decimals, + blockchain, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_asset_metadata.yml b/models/silver/prices/silver__complete_native_asset_metadata.yml new file mode 100644 index 0000000..1c1fdf1 --- /dev/null +++ b/models/silver/prices/silver__complete_native_asset_metadata.yml @@ -0,0 +1,24 @@ +version: 2 +models: + - name: silver__complete_native_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - SYMBOL + + columns: + - name: PROVIDER + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_native_prices.sql b/models/silver/prices/silver__complete_native_prices.sql new file mode 100644 index 0000000..5df4e9b --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.sql @@ -0,0 +1,41 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_native_prices_id', + cluster_by = ['HOUR::DATE'], + tags = ['scheduled_non_core'] +) }} + +SELECT + HOUR, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_native_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_native_prices' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_native_prices.yml b/models/silver/prices/silver__complete_native_prices.yml new file mode 100644 index 0000000..bb72433 --- /dev/null +++ b/models/silver/prices/silver__complete_native_prices.yml @@ -0,0 +1,37 @@ +version: 2 +models: + - name: silver__complete_native_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - SYMBOL + + columns: + - name: HOUR + tests: + - not_null + - name: SYMBOL + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_NATIVE_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.sql b/models/silver/prices/silver__complete_provider_asset_metadata.sql new file mode 100644 index 0000000..b9132ed --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.sql @@ -0,0 +1,37 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_asset_metadata_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + asset_id, + token_address, + NAME, + symbol, + platform, + platform_id, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_provider_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_provider_asset_metadata' + ) }} + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_asset_metadata.yml b/models/silver/prices/silver__complete_provider_asset_metadata.yml new file mode 100644 index 0000000..8fa56fd --- /dev/null +++ b/models/silver/prices/silver__complete_provider_asset_metadata.yml @@ -0,0 +1,26 @@ +version: 2 +models: + - name: silver__complete_provider_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - TOKEN_ADDRESS + - NAME + - SYMBOL + - PLATFORM + - PLATFORM_ID + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_provider_prices.sql b/models/silver/prices/silver__complete_provider_prices.sql new file mode 100644 index 0000000..bd1d85f --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.sql @@ -0,0 +1,46 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_provider_prices_id', + cluster_by = ['recorded_hour::DATE'], + tags = ['scheduled_non_core'] +) }} + +SELECT + p.asset_id, + recorded_hour, + OPEN, + high, + low, + CLOSE, + p.provider, + p.source, + p._inserted_timestamp, + p.inserted_timestamp, + p.modified_timestamp, + p.complete_provider_prices_id, + p._invocation_id +FROM + {{ ref( + 'bronze__complete_provider_prices' + ) }} + p + INNER JOIN {{ ref('bronze__complete_provider_asset_metadata') }} + m + ON p.asset_id = m.asset_id + +{% if is_incremental() %} +WHERE + p.modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY p.asset_id, recorded_hour, p.provider +ORDER BY + p.modified_timestamp DESC)) = 1 diff --git a/models/silver/prices/silver__complete_provider_prices.yml b/models/silver/prices/silver__complete_provider_prices.yml new file mode 100644 index 0000000..8db23c7 --- /dev/null +++ b/models/silver/prices/silver__complete_provider_prices.yml @@ -0,0 +1,25 @@ +version: 2 +models: + - name: silver__complete_provider_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ASSET_ID + - RECORDED_HOUR + - PROVIDER + columns: + - name: PROVIDER + tests: + - not_null + - name: ASSET_ID + tests: + - not_null + - name: RECORDED_HOUR + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_PROVIDER_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_asset_metadata.sql b/models/silver/prices/silver__complete_token_asset_metadata.sql new file mode 100644 index 0000000..88dbead --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.sql @@ -0,0 +1,42 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_asset_metadata_id', + tags = ['scheduled_non_core'] +) }} + +SELECT + LOWER( + A.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + blockchain, + blockchain_name, + blockchain_id, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_asset_metadata_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_asset_metadata' + ) }} A + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_asset_metadata.yml b/models/silver/prices/silver__complete_token_asset_metadata.yml new file mode 100644 index 0000000..3eea590 --- /dev/null +++ b/models/silver/prices/silver__complete_token_asset_metadata.yml @@ -0,0 +1,28 @@ +version: 2 +models: + - name: silver__complete_token_asset_metadata + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: PROVIDER + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_ASSET_METADATA_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/prices/silver__complete_token_prices.sql b/models/silver/prices/silver__complete_token_prices.sql new file mode 100644 index 0000000..54ed860 --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.sql @@ -0,0 +1,47 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = 'complete_token_prices_id', + cluster_by = ['HOUR::DATE'], + tags = ['scheduled_non_core'] +) }} + +SELECT + HOUR, + LOWER( + p.token_address + ) AS token_address, + asset_id, + symbol, + NAME, + decimals, + price, + blockchain, + blockchain_name, + blockchain_id, + is_imputed, + is_deprecated, + provider, + source, + _inserted_timestamp, + inserted_timestamp, + modified_timestamp, + complete_token_prices_id, + _invocation_id +FROM + {{ ref( + 'bronze__complete_token_prices' + ) }} + p + +{% if is_incremental() %} +WHERE + modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) +{% endif %} diff --git a/models/silver/prices/silver__complete_token_prices.yml b/models/silver/prices/silver__complete_token_prices.yml new file mode 100644 index 0000000..d2ac57f --- /dev/null +++ b/models/silver/prices/silver__complete_token_prices.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: silver__complete_token_prices + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - HOUR + - TOKEN_ADDRESS + - BLOCKCHAIN + + columns: + - name: HOUR + tests: + - not_null + - name: TOKEN_ADDRESS + tests: + - not_null + - name: BLOCKCHAIN + tests: + - not_null + - name: BLOCKCHAIN_ID + tests: + - not_null + - name: PROVIDER + tests: + - not_null + - name: PRICE + tests: + - not_null + - name: IS_IMPUTED + tests: + - not_null + - name: _INSERTED_TIMESTAMP + tests: + - not_null + - name: MODIFIED_TIMESTAMP + tests: + - not_null + - name: COMPLETE_TOKEN_PRICES_ID + tests: + - unique \ No newline at end of file diff --git a/models/silver/staking/silver__staking_actions_s.sql b/models/silver/staking/silver__staking_actions_s.sql index b20741c..a94a3f2 100644 --- a/models/silver/staking/silver__staking_actions_s.sql +++ b/models/silver/staking/silver__staking_actions_s.sql @@ -1,23 +1,34 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], + cluster_by = ['block_timestamp::DATE'], unique_key = 'tx_id', + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,delegator);", tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core'] ) }} WITH silver_events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + tx_succeeded, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -27,7 +38,7 @@ WHERE FROM {{ this }} WHERE - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + modified_timestamp >= SYSDATE() - INTERVAL '3 days' AND delegator IS NULL ) {% endif %} @@ -44,7 +55,8 @@ flow_staking AS ( event_data :amount :: FLOAT AS amount, event_data :delegatorID :: STRING AS delegator_id, event_data :nodeID :: STRING AS node_id, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM silver_events WHERE @@ -77,9 +89,9 @@ add_auth AS ( {% if is_incremental() %} AND ( - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -89,7 +101,7 @@ AND ( FROM {{ this }} WHERE - _inserted_timestamp >= SYSDATE() - INTERVAL '3 days' + modified_timestamp >= SYSDATE() - INTERVAL '3 days' AND delegator IS NULL ) ) @@ -107,6 +119,7 @@ FINAL AS ( amount, node_id, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id', 'event_index', 'action'] ) }} AS staking_actions_id, diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql index fd1085c..949f51f 100644 --- a/models/silver/stats/silver_stats__core_metrics_hourly.sql +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -74,7 +74,10 @@ transactions AS ( END ) AS transaction_count_failed, COUNT( - DISTINCT proposer + DISTINCT COALESCE( + authorizers [1], + authorizers [0] + ) :: STRING ) AS unique_from_count, COUNT( payer diff --git a/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql index d0c04e5..2901691 100644 --- a/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql +++ b/models/silver/streamline/core/complete/streamline__complete_get_blocks_history.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_blocks_history') }} +-- depends_on: {{ ref('bronze__streamline_complete_blocks_history') }} {{ config ( materialized = "incremental", unique_key = "block_number", @@ -15,7 +15,7 @@ SELECT _inserted_timestamp FROM -{{ ref('bronze__streamline_blocks_history') }} +{{ ref('bronze__streamline_complete_blocks_history') }} WHERE TRUE diff --git a/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql index 79ec85d..a1c306c 100644 --- a/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql +++ b/models/silver/streamline/core/complete/streamline__complete_get_collections_history.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_collections_history') }} +-- depends_on: {{ ref('bronze__streamline_complete_collections_history') }} {{ config ( materialized = "incremental", unique_key = "id", @@ -16,7 +16,7 @@ SELECT _inserted_timestamp FROM -{{ ref('bronze__streamline_collections_history') }} +{{ ref('bronze__streamline_complete_collections_history') }} WHERE TRUE {% if is_incremental() %} diff --git a/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql index dc42240..c7e4a7b 100644 --- a/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql +++ b/models/silver/streamline/core/complete/streamline__complete_get_transaction_results_history.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_transaction_results_history') }} +-- depends_on: {{ ref('bronze__streamline_complete_transaction_results_history') }} {{ config ( materialized = "incremental", unique_key = "id", @@ -16,7 +16,7 @@ SELECT _inserted_timestamp FROM -{{ ref('bronze__streamline_transaction_results_history') }} +{{ ref('bronze__streamline_complete_transaction_results_history') }} WHERE TRUE {% if is_incremental() %} diff --git a/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql b/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql index b1b2e09..71c676d 100644 --- a/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql +++ b/models/silver/streamline/core/complete/streamline__complete_get_transactions_history.sql @@ -1,4 +1,4 @@ --- depends_on: {{ ref('bronze__streamline_transactions_history') }} +-- depends_on: {{ ref('bronze__streamline_complete_transactions_history') }} {{ config ( materialized = "incremental", unique_key = "id", @@ -16,7 +16,7 @@ SELECT _inserted_timestamp FROM -{{ ref('bronze__streamline_transactions_history') }} +{{ ref('bronze__streamline_complete_transactions_history') }} WHERE TRUE {% if is_incremental() %} diff --git a/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_blocks.sql b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_blocks.sql new file mode 100644 index 0000000..0c8fbcd --- /dev/null +++ b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_blocks.sql @@ -0,0 +1,41 @@ +-- depends_on: {{ ref('bronze__streamline_testnet_blocks') }} +-- depends_on: {{ ref('bronze__streamline_fr_testnet_blocks') }} +{{ config ( + materialized = "incremental", + unique_key = "block_number", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["block_number"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(block_number)", + tags = ['streamline_complete_testnet'] +) }} + +SELECT + DATA, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_testnet_blocks') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze__streamline_fr_testnet_blocks') }} +WHERE + TRUE +{% endif %} +AND NOT ( + DATA :status :: INT != 2 +) +qualify(ROW_NUMBER() over (PARTITION BY block_number +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_collections.sql b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_collections.sql new file mode 100644 index 0000000..5657cd4 --- /dev/null +++ b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_collections.sql @@ -0,0 +1,39 @@ +-- depends_on: {{ ref('bronze__streamline_testnet_collections') }} +-- depends_on: {{ ref('bronze__streamline_fr_testnet_collections') }} + +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_complete_testnet'] +) }} + +SELECT + id, + data, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_testnet_collections') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01'::timestamp_ntz + ) +{% else %} + {{ ref('bronze__streamline_fr_testnet_collections') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 \ No newline at end of file diff --git a/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_transaction_results.sql b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_transaction_results.sql new file mode 100644 index 0000000..7c9fd49 --- /dev/null +++ b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_transaction_results.sql @@ -0,0 +1,41 @@ +-- depends_on: {{ ref('bronze__streamline_testnet_transaction_results') }} +-- depends_on: {{ ref('bronze__streamline_fr_testnet_transaction_results') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_complete_testnet'] +) }} + +SELECT + id, + DATA, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_testnet_transaction_results') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01' :: timestamp_ntz + ) +{% else %} + {{ ref('bronze__streamline_fr_testnet_transaction_results') }} +WHERE + TRUE +{% endif %} +AND NOT ( + DATA :status :: INT < 4 +) qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_transactions.sql b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_transactions.sql new file mode 100644 index 0000000..30cbd77 --- /dev/null +++ b/models/silver/streamline/core/complete_testnet/streamline__complete_get_testnet_transactions.sql @@ -0,0 +1,38 @@ +-- depends_on: {{ ref('bronze__streamline_testnet_transactions') }} +-- depends_on: {{ ref('bronze__streamline_fr_testnet_transactions') }} +{{ config ( + materialized = "incremental", + unique_key = "id", + cluster_by = "ROUND(block_number, -3)", + merge_update_columns = ["id"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION on equality(id)", + tags = ['streamline_complete_testnet'] +) }} + +SELECT + id, + data, + block_number, + _partition_by_block_id, + _inserted_timestamp +FROM + +{% if is_incremental() %} +{{ ref('bronze__streamline_testnet_transactions') }} +WHERE + _inserted_timestamp >= COALESCE( + ( + SELECT + MAX(_inserted_timestamp) _inserted_timestamp + FROM + {{ this }} + ), + '1900-01-01'::timestamp_ntz + ) +{% else %} + {{ ref('bronze__streamline_fr_testnet_transactions') }} +{% endif %} + +qualify(ROW_NUMBER() over (PARTITION BY id +ORDER BY + _inserted_timestamp DESC)) = 1 diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql index b698fcb..066844a 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_14', 'sql_limit', '750000', 'producer_batch_size','50000', 'worker_batch_size','500', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_14', 'sql_limit', '5000', 'producer_batch_size','500', 'worker_batch_size','25', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "streamline.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql index 557ac97..7a2c9f2 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_15','sql_limit', '750000', 'producer_batch_size','50000', 'worker_batch_size','500', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_15','sql_limit', '5000', 'producer_batch_size','500', 'worker_batch_size','25', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "streamline.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql index b30f08c..6f00ab2 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_16', 'sql_limit', '3000000', 'producer_batch_size','30000', 'worker_batch_size','300', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_16', 'sql_limit', '5000', 'producer_batch_size','500', 'worker_batch_size','50', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "streamline.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql index 2ea397a..f9d4b00 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_17', 'sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_17', 'sql_limit', '500000', 'producer_batch_size','10000', 'worker_batch_size','250', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql index 3a2e2e0..19dec75 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql @@ -1,20 +1,10 @@ {{ config ( materialized = "view", - - post_hook = fsc_utils.if_data_call_function_v2( - func = 'udf_bulk_grpc_us_east_2', - target = "{{this.schema}}.{{this.identifier}}", - params = { - "node_url":"access-001.mainnet18.nodes.onflow.org:9000", - "external_table": "transaction_results_mainnet_18", - "sql_limit": "25000", - "producer_batch_size": "1000", - "worker_batch_size": "200", - "sql_source": "{{this.identifier}}" - } + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_18', 'sql_limit', '500000', 'producer_batch_size','10000', 'worker_batch_size','250', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" ) ) }} - WITH collection_transactions AS ( SELECT diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql index e0b9b8f..4a84ca0 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_19','sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_19','sql_limit', '500000', 'producer_batch_size','10000', 'worker_batch_size','250', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql index 7220632..ec0d48e 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_22', 'sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_22', 'sql_limit', '500000', 'producer_batch_size','10000', 'worker_batch_size','250', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_blocks_realtime.sql b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_blocks_realtime.sql new file mode 100644 index 0000000..e144a8d --- /dev/null +++ b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_blocks_realtime.sql @@ -0,0 +1,61 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access.devnet.nodes.onflow.org:9000','external_table', 'testnet_blocks', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_realtime_testnet'] +) }} +WITH post_crescendo AS ( + SELECT + 185000000 AS block_height +), + tbl AS ( + SELECT + block_height + FROM + {{ ref('streamline__testnet_blocks') }} + WHERE + ( + block_height >= ( + SELECT + block_height + FROM + post_crescendo + ) + ) + AND block_height IS NOT NULL + EXCEPT + SELECT + block_number AS block_height + FROM + {{ ref('streamline__complete_get_testnet_blocks') }} + WHERE + ( + block_height >= ( + SELECT + block_height + FROM + post_crescendo + ) + ) + AND block_height IS NOT NULL + ) +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_block_by_height', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'height', + block_height + ) + ) AS request +FROM + tbl +ORDER BY + block_height DESC diff --git a/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_collections_realtime.sql b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_collections_realtime.sql new file mode 100644 index 0000000..b9eb8ae --- /dev/null +++ b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_collections_realtime.sql @@ -0,0 +1,64 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access.devnet.nodes.onflow.org:9000','external_table', 'testnet_collections', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_realtime_testnet'] +) }} + +WITH +post_crescendo AS ( + SELECT + 185000000 AS block_height +), + -- CTE to get targeted block_heights and their associated collection_ids from the complete_get_blocks table + block_collections AS ( + SELECT + cb.block_number AS block_height, + collection_guarantee.value :collection_id AS collection_id + FROM + {{ ref("streamline__complete_get_testnet_blocks") }} + cb, + LATERAL FLATTEN( + input => cb.data :collection_guarantees + ) AS collection_guarantee + WHERE + block_height >= ( + SELECT + block_height + FROM + post_crescendo + ) + ), + -- CTE to identify collections that haven't been ingested yet + collections_to_ingest AS ( + SELECT + bc.block_height, + bc.collection_id + FROM + block_collections bc + LEFT JOIN {{ ref("streamline__complete_get_testnet_collections") }} C + ON bc.block_height = C.block_number + AND bc.collection_id = C.id + WHERE + C.id IS NULL + ) +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_collection_by_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + collection_id + ) + ) AS request +FROM + collections_to_ingest +ORDER BY + block_height DESC diff --git a/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_transaction_results_realtime.sql b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_transaction_results_realtime.sql new file mode 100644 index 0000000..94e02ab --- /dev/null +++ b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_transaction_results_realtime.sql @@ -0,0 +1,64 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access.devnet.nodes.onflow.org:9000','external_table', 'testnet_transaction_results', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_realtime_testnet'] +) }} + +WITH post_crescendo AS ( + SELECT + 185000000 AS block_height +), + collection_transactions AS ( + SELECT + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id + FROM + {{ ref("streamline__complete_get_testnet_collections") }} + cc, + LATERAL FLATTEN( + input => cc.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height >= ( + SELECT + block_height + FROM + post_crescendo + ) + ), + -- CTE to identify transactions that haven't been ingested yet + transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_testnet_transaction_results") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL + ) -- Generate the requests based on the missing transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_result', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id :: STRING, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id :: STRING + ) + ) AS request +FROM + transactions_to_ingest +ORDER BY + block_height DESC diff --git a/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_transactions_realtime.sql b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_transactions_realtime.sql new file mode 100644 index 0000000..e5a7956 --- /dev/null +++ b/models/silver/streamline/core/realtime_testnet/streamline__get_testnet_transactions_realtime.sql @@ -0,0 +1,64 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access.devnet.nodes.onflow.org:9000','external_table', 'testnet_transactions', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "{{this.schema}}.{{this.identifier}}" + ), + tags = ['streamline_realtime_testnet'] +) }} + +WITH post_crescendo AS ( + SELECT + 185000000 AS block_height +), + collection_transactions AS ( + SELECT + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id + FROM + {{ ref("streamline__complete_get_testnet_collections") }} + cc, + LATERAL FLATTEN( + input => cc.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height >= ( + SELECT + block_height + FROM + post_crescendo + ) + ), + -- CTE to identify transactions that haven't been ingested yet + transactions_to_ingest AS ( + SELECT + ct.block_height, + ct.transaction_id + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_testnet_transactions") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL + ) -- Generate the requests based on the missing transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction', + 'block_height', + block_height :: INTEGER, + 'transaction_id', + transaction_id :: STRING, + 'method_params', + OBJECT_CONSTRUCT( + 'id', + transaction_id :: STRING + ) + ) AS request +FROM + transactions_to_ingest +ORDER BY + block_height DESC diff --git a/models/silver/streamline/core/streamline__testnet_blocks.sql b/models/silver/streamline/core/streamline__testnet_blocks.sql new file mode 100644 index 0000000..aa0540b --- /dev/null +++ b/models/silver/streamline/core/streamline__testnet_blocks.sql @@ -0,0 +1,19 @@ +{{ config ( + materialized = "view", + tags = ['streamline_realtime_testnet'] +) }} + +{% if execute %} + +{% set height = run_query('SELECT streamline.udf_get_chainhead_testnet()') %} +{% set block_height = height.columns[0].values()[0] %} +{% else %} +{% set block_height = 0 %} +{% endif %} + +SELECT + height as block_height +FROM + TABLE(streamline.udtf_get_base_table({{block_height}})) +WHERE + block_height > 185000000 diff --git a/models/silver/transfers/silver__bridge_blocto_s.sql b/models/silver/transfers/silver__bridge_blocto_s.sql index 589ab4c..4aa3a73 100644 --- a/models/silver/transfers/silver__bridge_blocto_s.sql +++ b/models/silver/transfers/silver__bridge_blocto_s.sql @@ -9,7 +9,16 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} -- WHERE @@ -17,9 +26,9 @@ WITH events AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -33,7 +42,8 @@ teleport_events AS ( event_contract AS teleport_contract_fee, event_data :amount :: DOUBLE AS amount_fee, event_data :type :: NUMBER AS teleport_direction, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM events WHERE @@ -113,7 +123,8 @@ blocto_inbound AS ( d.to_deposits AS flow_wallet_address, f.teleport_direction, 'blocto' AS bridge, - f._inserted_timestamp + f._inserted_timestamp, + f._partition_by_block_id FROM teleports_in t LEFT JOIN deposits d USING (tx_id) @@ -217,7 +228,8 @@ blocto_outbound AS ( w.from_withdraw AS flow_wallet_address, f.teleport_direction, 'blocto' AS bridge, - f._inserted_timestamp + f._inserted_timestamp, + f._partition_by_block_id FROM teleports_out t LEFT JOIN teleports_out_withdraw w USING (tx_id) @@ -236,7 +248,8 @@ tbl_union AS ( flow_wallet_address, teleport_direction, bridge, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM blocto_inbound UNION @@ -252,7 +265,8 @@ tbl_union AS ( flow_wallet_address, teleport_direction, bridge, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM blocto_outbound ), @@ -281,6 +295,7 @@ FINAL AS ( l.blockchain, bridge, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id'] ) }} AS bridge_blocto_id, diff --git a/models/silver/transfers/silver__bridge_celer_s.sql b/models/silver/transfers/silver__bridge_celer_s.sql index 7c5a339..f8a4597 100644 --- a/models/silver/transfers/silver__bridge_celer_s.sql +++ b/models/silver/transfers/silver__bridge_celer_s.sql @@ -9,7 +9,17 @@ WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + tx_succeeded, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} -- WHERE @@ -17,9 +27,9 @@ WITH events AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -35,7 +45,8 @@ cbridge_txs AS ( event_contract, event_type, event_data, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM events WHERE @@ -54,7 +65,8 @@ inbound AS ( event_data :refChId :: NUMBER AS chain_id, 'inbound' AS direction, 'cbridge' AS bridge, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM events WHERE @@ -82,7 +94,8 @@ outbound AS ( event_data :toChain :: NUMBER AS chain_id, 'outbound' AS direction, 'cbridge' AS bridge, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM events WHERE @@ -107,7 +120,8 @@ tbl_union AS ( chain_id, direction, bridge, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM inbound UNION @@ -123,7 +137,8 @@ tbl_union AS ( chain_id, direction, bridge, - _inserted_timestamp + _inserted_timestamp, + _partition_by_block_id FROM outbound ), @@ -149,6 +164,7 @@ FINAL AS ( direction, bridge, _inserted_timestamp, + _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id'] ) }} AS bridge_celer_id, diff --git a/models/silver/transfers/silver__bridge_complete.sql b/models/silver/transfers/silver__bridge_complete.sql new file mode 100644 index 0000000..a248ad1 --- /dev/null +++ b/models/silver/transfers/silver__bridge_complete.sql @@ -0,0 +1,99 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::date'], + unique_key = 'bridge_complete_id', + tags = ['bridge', 'scheduled', 'streamline_scheduled', 'scheduled_non_core'] +) }} + +WITH +blocto AS ( + SELECT + tx_id, + block_timestamp, + block_height, + teleport_contract AS bridge_address, + token_contract AS token_address, + gross_amount, + amount_fee, + net_amount, + IFF(teleport_direction = 'inbound', flow_wallet_address, null) AS destination_address, + IFF(teleport_direction = 'outbound', flow_wallet_address, null) AS source_address, + IFF(teleport_direction = 'inbound', 'flow', blockchain) AS destination_chain, + IFF(teleport_direction = 'outbound', 'flow', blockchain) AS source_chain, + bridge AS platform, + inserted_timestamp, + modified_timestamp, + bridge_blocto_id AS bridge_complete_id + FROM + {{ ref('silver__bridge_blocto_s') }} +{% if is_incremental() %} +WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +celer AS ( + SELECT + tx_id, + block_timestamp, + block_height, + bridge_contract AS bridge_address, + token_contract AS token_address, + amount AS gross_amount, + NULL AS amount_fee, + amount AS net_amount, + IFF(direction = 'inbound', flow_wallet_address, counterparty) AS destination_address, + IFF(direction = 'outbound', flow_wallet_address, counterparty) AS source_address, + IFF(direction = 'inbound', 'flow', blockchain) AS destination_chain, + IFF(direction = 'outbound', 'flow', blockchain) AS source_chain, + bridge AS platform, + inserted_timestamp, + modified_timestamp, + bridge_celer_id AS bridge_complete_id + FROM + {{ ref('silver__bridge_celer_s') }} +{% if is_incremental() %} +WHERE modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +combo AS ( + SELECT + * + FROM + blocto + UNION ALL + SELECT + * + FROM + celer +) +SELECT + tx_id, + block_timestamp, + block_height, + bridge_address, + token_address, + gross_amount, + amount_fee, + net_amount, + source_address, + destination_address, + source_chain, + destination_chain, + platform, + bridge_complete_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id, +FROM + combo diff --git a/models/silver/transfers/silver__token_transfers_s.sql b/models/silver/transfers/silver__token_transfers_s.sql index cadea9d..b88a169 100644 --- a/models/silver/transfers/silver__token_transfers_s.sql +++ b/models/silver/transfers/silver__token_transfers_s.sql @@ -1,15 +1,26 @@ {{ config( materialized = 'incremental', incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], + cluster_by = ['block_timestamp::date'], unique_key = "CONCAT_WS('-', tx_id, sender, recipient, token_contract, amount)", + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(tx_id,sender,recipient,token_contract);", tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core'] ) }} WITH events AS ( SELECT - * + block_height, + block_timestamp, + tx_id, + tx_succeeded, + event_index, + event_type, + event_contract, + event_data, + _inserted_timestamp, + _partition_by_block_id, + modified_timestamp FROM {{ ref('silver__streamline_events') }} -- WHERE @@ -17,9 +28,9 @@ WITH events AS ( {% if is_incremental() %} WHERE - _inserted_timestamp >= ( + modified_timestamp >= ( SELECT - MAX(_inserted_timestamp) + MAX(modified_timestamp) FROM {{ this }} ) @@ -144,6 +155,7 @@ FINAL AS ( ) SELECT *, + round(block_height, -5) AS _partition_by_block_id, {{ dbt_utils.generate_surrogate_key( ['tx_id','sender', 'recipient','token_contract', 'amount'] ) }} AS token_transfers_id, diff --git a/models/silver_cw/README.md b/models/silver_cw/README.md deleted file mode 100644 index 612189f..0000000 --- a/models/silver_cw/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# SILVER_CW / The Streamline Migration - -This directory is a copy of the silver directory with models downstream of the Chainwalkers source. Chainwalkers is being de-commissioned in October 2023, but the Streamline backfill will take some time to finish. These models are thus copied over as an archive of the Chainwalkers data, and to keep the new (streamline) models separate and organized. - -This entire directory may be deleted once Chainwalkers is fully deprecated. - -New models that have been migrated to use Streamline as a source are suffixed with `_s`. No suffix was added for models that do not require a migration (prices, labels, external APIs, etc.). While I would prefer the new models have no suffix, the decision was made to append `_s` to the new streamline models (instead of copying data into new tables and appending `_cw` to the deprecating data) to minimize the touchpoints on prod data. A decision may be made once the CW data is fully deprecated to drop the suffix from all models and rebuild in cleanly named tables. - -The `scheduled` tag will remain on all models that should run on an hourly schedule. 2 new tags have been added for org purposes: `streamline_scheduled` and `chainwalkers_scheduled` to enable clear model selection. These tags were added only where a model was already tagged with `scheduled` to limit net-new. diff --git a/models/silver_cw/core/silver__blocks.sql b/models/silver_cw/core/silver__blocks.sql deleted file mode 100644 index 570d70f..0000000 --- a/models/silver_cw/core/silver__blocks.sql +++ /dev/null @@ -1,85 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'block_height', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH bronze_blocks AS ( - - SELECT - * - FROM - {{ ref('bronze__blocks') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} - -qualify ROW_NUMBER() over ( - PARTITION BY block_id - ORDER BY - _ingested_at DESC -) = 1 -), -silver_blocks AS ( - SELECT - block_id AS block_height, - block_timestamp, - network, - chain_id, - tx_count, - COALESCE( - header :block_id, - header :block_header :block_id, - header :id - ) :: STRING AS id, - COALESCE( - header :parent_id, - header :parentId, - header :block_header :parent_id - ) :: STRING AS parent_id, - _ingested_at, - _inserted_timestamp - FROM - bronze_blocks -), -network_version AS ( - SELECT - root_height, - network_version, - COALESCE(LAG(root_height) over ( - ORDER BY - network_version DESC) - 1, 'inf' :: FLOAT) AS end_height - FROM - {{ ref('seeds__network_version') }} -), -add_version AS ( - SELECT - block_height, - block_timestamp, - network, - v.network_version, - chain_id, - tx_count, - id, - parent_id, - _ingested_at, - _inserted_timestamp - FROM - silver_blocks b - LEFT JOIN network_version v - ON b.block_height BETWEEN v.root_height - AND v.end_height -) -SELECT - * -FROM - add_version diff --git a/models/silver_cw/core/silver__blocks.yml b/models/silver_cw/core/silver__blocks.yml deleted file mode 100644 index bf704c3..0000000 --- a/models/silver_cw/core/silver__blocks.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 - -models: - - name: silver__blocks - description: Information about blocks on the FLOW network and corresponding metadata. - - columns: - - name: block_height - description: "{{ doc('block_height') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: network - description: "{{ doc('network') }}" - - - name: network_version - description: "{{ doc('network_version') }}" - - - name: chain_id - description: "{{ doc('chain_id') }}" - - - name: tx_count - description: "{{ doc('tx_count') }}" - - - name: id - description: "{{ doc('id') }}" - - - name: parent_id - description: "{{ doc('parent_id') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/core/silver__event_attributes.sql b/models/silver_cw/core/silver__event_attributes.sql deleted file mode 100644 index 7bb506c..0000000 --- a/models/silver_cw/core/silver__event_attributes.sql +++ /dev/null @@ -1,170 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'attribute_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events') }} - WHERE - _inserted_timestamp :: DATE < '2022-07-18' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -events_data AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - _event_data_type, - _event_data_fields, - _ingested_at, - COALESCE( - _event_data_type :fields, - _event_data_type :Fields - ) AS event_data_type_fields, - _inserted_timestamp - FROM - events -), -attributes AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - COALESCE( - VALUE :identifier, - VALUE :Identifier - ) :: STRING AS attribute_key, - COALESCE( - _event_data_fields [index] :Value, - _event_data_fields [index] - ) AS attribute_value, - concat_ws( - '-', - event_id, - INDEX - ) AS attribute_id, - INDEX AS attribute_index, - _ingested_at, - _inserted_timestamp - FROM - events_data, - LATERAL FLATTEN( - input => event_data_type_fields - ) -), -handle_address_arrays AS ( - SELECT - attribute_id, - b.index, - LPAD(TRIM(to_char(b.value :: INT, 'XXXXXXX')) :: STRING, 2, '0') AS hex - FROM - attributes A, - TABLE(FLATTEN(attribute_value, recursive => TRUE)) b - WHERE - IS_ARRAY(attribute_value) = TRUE - ORDER BY - 1, - 2 -), -recombine_address AS ( - SELECT - attribute_id, - CONCAT( - '0x', - ARRAY_TO_STRING(ARRAY_AGG(hex) within GROUP ( - ORDER BY - INDEX ASC), '') - ) AS decoded_address - FROM - handle_address_arrays - GROUP BY - 1 -), -replace_arrays AS ( - SELECT - A.attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value, - decoded_address, - COALESCE( - decoded_address, - attribute_value - ) :: STRING AS attribute_value_adj, - _ingested_at, - _inserted_timestamp - FROM - attributes A - LEFT JOIN recombine_address USING (attribute_id) -), -address_adjustment AS ( - SELECT - attribute_id, - LENGTH(attribute_value_adj) AS ava_len, - CONCAT( - '0x', - LPAD(SPLIT(attribute_value_adj, '0x') [1], 16, '0') :: STRING - ) AS address_adj - FROM - replace_arrays - WHERE - attribute_value_adj LIKE '0x%' - AND ava_len < 19 -), -FINAL AS ( - SELECT - A.attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - decoded_address, - attribute_value, - REPLACE( - COALESCE( - address_adj, - attribute_value_adj - ), - '"' - ) AS attribute_value_adj, - _ingested_at, - _inserted_timestamp - FROM - replace_arrays A - LEFT JOIN address_adjustment USING (attribute_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__event_attributes_https.sql b/models/silver_cw/core/silver__event_attributes_https.sql deleted file mode 100644 index 08c1d03..0000000 --- a/models/silver_cw/core/silver__event_attributes_https.sql +++ /dev/null @@ -1,154 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'attribute_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events') }} - WHERE - _inserted_timestamp :: DATE >= '2022-07-18' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -payload_method AS ( - SELECT - * - FROM - events - WHERE - _try_parse_payload IS NOT NULL -), -events_data AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - _event_data_type, - _event_data_fields, - _ingested_at, - COALESCE( - _event_data_type :fields, - _event_data_type :Fields - ) AS event_data_type_fields, - _inserted_timestamp - FROM - events - WHERE - _try_parse_payload IS NULL -), -attributes AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - COALESCE( - VALUE :identifier, - VALUE :Identifier - ) :: STRING AS attribute_key, - COALESCE( - _event_data_fields [index] :fields, - _event_data_fields [index] :staticType :typeID, - _event_data_fields [index] :value :value, - _event_data_fields [index] :value, - _event_data_fields [index] :Value, - _event_data_fields [index] - ) :: STRING AS attribute_value, - concat_ws( - '-', - event_id, - INDEX - ) AS attribute_id, - INDEX AS attribute_index, - _ingested_at, - _inserted_timestamp, - 'attributes' AS _cte - FROM - events_data, - LATERAL FLATTEN( - input => event_data_type_fields - ) -), -attributes_2 AS ( - SELECT - event_id, - tx_id, - block_timestamp, - event_index, - event_contract, - event_type, - VALUE :name :: STRING AS attribute_key, - COALESCE( - VALUE :value :value :fields, - VALUE :value :value :staticType, - VALUE :value :value :value :value, - VALUE :value :value :value, - VALUE :value :value - ) :: STRING AS attribute_value, - concat_ws( - '-', - event_id, - INDEX - ) AS attribute_id, - INDEX AS attribute_index, - _ingested_at, - _inserted_timestamp, - 'attributes_2' AS _cte - FROM - payload_method, - LATERAL FLATTEN( - _try_parse_payload :value :fields - ) -), -combo AS ( - SELECT - * - FROM - attributes - UNION - SELECT - * - FROM - attributes_2 -), -FINAL AS ( - SELECT - attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value, - _ingested_at, - _inserted_timestamp, - _cte - FROM - combo -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__event_attributes_https.yml b/models/silver_cw/core/silver__event_attributes_https.yml deleted file mode 100644 index e1f0847..0000000 --- a/models/silver_cw/core/silver__event_attributes_https.yml +++ /dev/null @@ -1,114 +0,0 @@ -version: 2 - -models: - - name: silver__event_attributes_https - description: |- - This table cleans and transform attributes from each event in the events table. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - event_index - - attribute_index - - columns: - - name: attribute_id - description: "{{ doc('attribute_id') }}" - tests: - - not_null - - unique - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_id - description: "{{ doc('event_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: event_index - description: "{{ doc('event_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: attribute_index - description: "{{ doc('attribute_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_type - description: "{{ doc('event_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: attribute_key - description: "{{ doc('attribute_key') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: decoded_address - description: "{{ doc('decoded_address') }}" - - - name: attribute_value - description: "{{ doc('attribute_value') }}" - - - name: attribute_value_adj - description: "{{ doc('attribute_value_adj') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/core/silver__events.sql b/models/silver_cw/core/silver__events.sql deleted file mode 100644 index 98285c7..0000000 --- a/models/silver_cw/core/silver__events.sql +++ /dev/null @@ -1,100 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'event_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH transactions AS ( - - SELECT - * - FROM - {{ ref('silver__transactions') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -events AS ( - SELECT - VALUE, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - INDEX AS _index_from_flatten, - COALESCE( - VALUE :event_index, - VALUE :eventIndex - ) :: NUMBER AS event_index, - VALUE :value :: variant AS event_data, - COALESCE( - VALUE :value :EventType, - VALUE :value :eventType - ) :: variant AS event_data_type, - COALESCE( - VALUE :value :Fields, - VALUE :value :fields - ) :: variant AS event_data_fields, - SPLIT( - IFF( - ( - event_data_type :qualifiedIdentifier LIKE 'A.%' - OR event_data_type :qualifiedIdentifier LIKE 'flow%' - ), - event_data_type :qualifiedIdentifier, - VALUE :type - ), - '.' - ) AS type_split, - ARRAY_TO_STRING( - ARRAY_SLICE(type_split, 0, ARRAY_SIZE(type_split) -1), - '.') AS event_contract, - type_split [array_size(type_split)-1] :: STRING AS event_type, - concat_ws( - '-', - tx_id, - event_index - ) AS event_id, - TRY_PARSE_JSON(BASE64_DECODE_STRING(VALUE :payload)) AS try_parse_payload, - _ingested_at, - _inserted_timestamp - FROM - transactions, - LATERAL FLATTEN( - input => transaction_result :events - ) - ), - FINAL AS ( - SELECT - event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - event_data_type AS _event_data_type, - event_data_fields AS _event_data_fields, - try_parse_payload AS _try_parse_payload, - _event_data_type :fields AS _attribute_fields, - _index_from_flatten, - _ingested_at, - _inserted_timestamp - FROM - events - ) - SELECT - * - FROM - FINAL diff --git a/models/silver_cw/core/silver__events.yml b/models/silver_cw/core/silver__events.yml deleted file mode 100644 index c703cc7..0000000 --- a/models/silver_cw/core/silver__events.yml +++ /dev/null @@ -1,116 +0,0 @@ -version: 2 - -models: - - name: silver__events - description: |- - This table records events from each transaction on the FLOW blockchain. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - event_index - - columns: - - name: event_id - description: "{{ doc('event_id') }}" - tests: - - not_null - - unique - - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: event_index - description: "{{ doc('event_index') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_type - description: "{{ doc('event_type') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: event_data - description: "{{ doc('event_data') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - - - name: _event_data_type - description: "{{ doc('_event_data_type') }}" - tests: - - not_null: - name: event_data_type__not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - name: event_data_type__expected_type - - - name: _event_data_fields - description: "{{ doc('_event_data_fields') }}" - tests: - - not_null: - name: event_data_fields__not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARIANT - name: event_data_fields__expected_type - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/core/silver__events_final.sql b/models/silver_cw/core/silver__events_final.sql deleted file mode 100644 index 91e1218..0000000 --- a/models/silver_cw/core/silver__events_final.sql +++ /dev/null @@ -1,183 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = "CONCAT_WS('-', tx_id, event_index)", - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -silver_event_attributes AS ( - SELECT - * - FROM - {{ ref('silver__event_attributes') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -silver_event_attributes_https AS ( - SELECT - * - FROM - {{ ref('silver__event_attributes_https') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -all_event_attributes AS ( - SELECT - attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value_adj AS attribute_value, - _ingested_at, - _inserted_timestamp - FROM - silver_event_attributes - UNION - SELECT - attribute_id, - event_id, - tx_id, - block_timestamp, - event_index, - attribute_index, - event_contract, - event_type, - attribute_key, - attribute_value, - _ingested_at, - _inserted_timestamp - FROM - silver_event_attributes_https -), -objs AS ( - SELECT - event_id, - OBJECT_AGG( - attribute_key, - IFF( - attribute_value IS NULL, - 'null', - attribute_value - ) :: variant - ) AS event_data - FROM - all_event_attributes - GROUP BY - 1 -), -location_object AS ( - SELECT - event_id, - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - COALESCE( - _event_data_type :location, - _event_data_type :Location - ) AS event_data, - _ingested_at, - _inserted_timestamp - FROM - silver_events - WHERE - _event_data_fields = '[]' - AND tx_id NOT IN ( - SELECT - tx_id - FROM - silver_event_attributes_https - ) -), -gold_events AS ( - SELECT - e.event_id, - e.tx_id, - e.block_timestamp, - e.block_height, - e.tx_succeeded, - e.event_index, - e.event_contract, - e.event_type, - A.event_data, - e._ingested_at, - e._inserted_timestamp - FROM - objs A - LEFT JOIN silver_events e USING (event_id) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _ingested_at, - _inserted_timestamp - FROM - gold_events - UNION - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _ingested_at, - _inserted_timestamp - FROM - location_object -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__events_final.yml b/models/silver_cw/core/silver__events_final.yml deleted file mode 100644 index e296473..0000000 --- a/models/silver_cw/core/silver__events_final.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -models: - - name: silver__events_final - description: |- - This table records events from each transaction on the FLOW blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: event_index - description: "{{ doc('event_index') }}" - - - name: event_contract - description: "{{ doc('event_contract') }}" - - - name: event_type - description: "{{ doc('event_type') }}" - - - name: event_data - description: "{{ doc('event_attributes') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/core/silver__transactions.sql b/models/silver_cw/core/silver__transactions.sql deleted file mode 100644 index 4eb73e7..0000000 --- a/models/silver_cw/core/silver__transactions.sql +++ /dev/null @@ -1,123 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH bronze_txs AS ( - - SELECT - * - FROM - {{ ref('bronze__transactions') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} - -qualify ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - _ingested_at DESC -) = 1 -), -silver_txs AS ( - SELECT - tx_id, - block_timestamp, - block_id AS block_height, - chain_id, - tx_block_index AS tx_index, - COALESCE( - tx :proposal_key :Address, - tx :proposalKeyAddress - ) :: STRING AS proposer, - tx :payer :: STRING AS payer, - tx :authorizers :: ARRAY AS authorizers, - ARRAY_SIZE(authorizers) AS count_authorizers, - COALESCE( - tx :gas_limit, - tx :gasLimit - ) :: NUMBER AS gas_limit, - COALESCE( - tx :transaction_result, - tx :result - ) :: variant AS transaction_result, - CASE - WHEN transaction_result :error = '' THEN TRUE - WHEN transaction_result :error :: STRING IS NULL THEN TRUE - ELSE FALSE - END AS tx_succeeded, - COALESCE( - transaction_result :error, - '' - ) :: STRING AS error_msg, - _ingested_at, - _inserted_timestamp - FROM - bronze_txs -), -concat_authorizers AS ( - SELECT - tx_id, - CONCAT( - '0x', - VALUE - ) AS x_auth - FROM - silver_txs, - LATERAL FLATTEN ( - input => authorizers - ) -), -authorizers_array AS ( - SELECT - tx_id, - ARRAY_AGG(x_auth) AS authorizers - FROM - concat_authorizers - GROUP BY - 1 -), -FINAL AS ( - SELECT - t.tx_id, - block_timestamp, - block_height, - chain_id, - tx_index, - CONCAT( - '0x', - proposer - ) AS proposer, - CONCAT( - '0x', - payer - ) AS payer, - COALESCE( - aa.authorizers, - t.authorizers - ) AS authorizers, - count_authorizers, - gas_limit, - transaction_result, - tx_succeeded, - error_msg, - _ingested_at, - _inserted_timestamp - FROM - silver_txs t - LEFT JOIN authorizers_array aa USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/core/silver__transactions.yml b/models/silver_cw/core/silver__transactions.yml deleted file mode 100644 index 5a7d9db..0000000 --- a/models/silver_cw/core/silver__transactions.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 - -models: - - name: silver__transactions - description: |- - This table records all the transactions of the FLOW blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: chain_id - description: "{{ doc('chain_id') }}" - - - name: tx_index - description: tbd - - - name: proposer - description: "{{ doc('proposer') }}" - - - name: payer - description: "{{ doc('payer') }}" - - - name: authorizers - description: "{{ doc('authorizers') }}" - - - name: count_authorizers - description: "{{ doc('count_authorizers') }}" - - - name: gas_limit - description: "{{ doc('gas_limit') }}" - - - name: transaction_result - description: "{{ doc('transaction_result') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: error_msg - description: tbd - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/labels/silver__contract_labels.sql b/models/silver_cw/labels/silver__contract_labels.sql deleted file mode 100644 index 076597b..0000000 --- a/models/silver_cw/labels/silver__contract_labels.sql +++ /dev/null @@ -1,52 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['event_contract'], - unique_key = 'event_contract', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH splt AS ( - - SELECT - event_contract, - SPLIT( - event_contract, - '.' - ) AS ec_s, - _inserted_timestamp - FROM - {{ ref('silver__events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -FINAL AS ( - SELECT - DISTINCT event_contract, - ec_s [array_size(ec_s)-1] :: STRING AS contract_name, - CONCAT( - '0x', - ec_s [array_size(ec_s)-2] :: STRING - ) AS account_address, - _inserted_timestamp - FROM - splt - WHERE - ec_s [0] != 'flow' -) -SELECT - * -FROM - FINAL qualify ROW_NUMBER() over ( - PARTITION BY event_contract - ORDER BY - _inserted_timestamp DESC - ) = 1 diff --git a/models/silver_cw/labels/silver__contract_labels.yml b/models/silver_cw/labels/silver__contract_labels.yml deleted file mode 100644 index 1a4ac8c..0000000 --- a/models/silver_cw/labels/silver__contract_labels.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 - -models: - - name: silver__contract_labels - description: |- - This table extracts all contract labels referenced in the events item of a Flow transaction. - - columns: - - name: event_contract - description: "{{ doc('event_contract') }}" - - - name: contract_name - description: "{{ doc('contract_name') }}" - - - name: account_address - description: "{{ doc('account_address') }}" diff --git a/models/silver_cw/labels/silver__labels_pools.sql b/models/silver_cw/labels/silver__labels_pools.sql deleted file mode 100644 index 2d229c0..0000000 --- a/models/silver_cw/labels/silver__labels_pools.sql +++ /dev/null @@ -1,66 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -pair_labels AS ( - SELECT - * - FROM - {{ ref('silver__contract_labels') }} - WHERE - contract_name ILIKE '%swappair%' -), -pair_creation AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :numPairs :: NUMBER AS pair_number, - event_data :pairAddress :: STRING AS account_address, - event_data :token0Key :: STRING AS token0_contract, - event_data :token1Key :: STRING AS token1_contract, - _inserted_timestamp - FROM - events - WHERE - event_type = 'PairCreated' -), -FINAL AS ( - SELECT - tx_id, - block_timestamp AS deployment_timestamp, - pair_number as pool_id, - p.account_address as vault_address, - token0_contract, - token1_contract, - l.event_contract AS swap_contract, - _inserted_timestamp - FROM - pair_creation p - LEFT JOIN pair_labels l USING (account_address) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/labels/silver__labels_pools.yml b/models/silver_cw/labels/silver__labels_pools.yml deleted file mode 100644 index 9a53f2b..0000000 --- a/models/silver_cw/labels/silver__labels_pools.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 2 - -models: - - name: silver__labels_pools - description: |- - Looks for new PairCreated events from the Swap pair factory and records info. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - unique - - - name: deployment_timestamp - description: "{{ doc('deployment_timestamp') }}" - tests: - - not_null - - - name: token0_contract - description: "{{ doc('token0_contract') }}" - tests: - - not_null - - - name: token1_contract - description: "{{ doc('token1_contract') }}" - tests: - - not_null - - - name: pool_id - description: "{{ doc('pool_id') }}" - tests: - - not_null - - - name: vault_address - description: "{{ doc('vault_address') }}" - tests: - - not_null - - unique - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null diff --git a/models/silver_cw/labels/silver__labels_pools_metapier.sql b/models/silver_cw/labels/silver__labels_pools_metapier.sql deleted file mode 100644 index fb4f6b1..0000000 --- a/models/silver_cw/labels/silver__labels_pools_metapier.sql +++ /dev/null @@ -1,120 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -pierpools AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_type, - event_data :poolId :: STRING AS pool_id, - _inserted_timestamp - FROM - events - WHERE - event_contract = 'A.609e10301860b683.PierSwapFactory' - AND event_type = 'NewPoolCreated' -), -pier_events AS ( - SELECT - * - FROM - events - WHERE - tx_id IN ( - SELECT - DISTINCT tx_id - FROM - pierpools - ) -), -token_withdraws AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_index, - _inserted_timestamp - FROM - pier_events - WHERE - event_type = 'TokensWithdrawn' - AND event_data :amount :: DOUBLE = 0 -), -pairs AS ( - SELECT - tx_id, - block_timestamp AS deployment_timestamp, - event_contract AS token0_contract, - LAG(event_contract) over ( - PARTITION BY tx_id - ORDER BY - event_index - ) AS token1_contract, - _inserted_timestamp - FROM - token_withdraws -), -escrow AS ( - SELECT - tx_id, - event_data :address :: STRING AS vault_address - FROM - pier_events - WHERE - event_contract = 'flow' - AND event_type = 'AccountCreated' -), -pool_addr AS ( - SELECT - tx_id, - event_contract AS swap_contract - FROM - pier_events - WHERE - event_type = 'Mint' -), -FINAL AS ( - SELECT - C.tx_id, - C.deployment_timestamp, - C.token0_contract, - C.token1_contract, - p.pool_id, - e.vault_address, - pa.swap_contract, - C._inserted_timestamp - FROM - pairs C - LEFT JOIN pierpools p USING (tx_id) - LEFT JOIN escrow e USING (tx_id) - LEFT JOIN pool_addr pa USING (tx_id) - WHERE - token1_contract IS NOT NULL -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/labels/silver__labels_pools_metapier.yml b/models/silver_cw/labels/silver__labels_pools_metapier.yml deleted file mode 100644 index be1a85d..0000000 --- a/models/silver_cw/labels/silver__labels_pools_metapier.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: 2 - -models: - - name: silver__labels_pools_metapier - description: |- - Looks for new PoolCreated events from the Metapier factory and recordes info. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - unique - - - name: deployment_timestamp - description: "{{ doc('deployment_timestamp') }}" - tests: - - not_null - - - name: token0_contract - description: "{{ doc('token0_contract') }}" - tests: - - not_null - - - name: token1_contract - description: "{{ doc('token1_contract') }}" - tests: - - not_null - - - name: pool_id - description: "{{ doc('pool_id') }}" - tests: - - not_null - - unique - - - name: vault_address - description: "{{ doc('vault_address') }}" - tests: - - not_null - - unique - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null diff --git a/models/silver_cw/nft/silver__nft_giglabs.sql b/models/silver_cw/nft/silver__nft_giglabs.sql deleted file mode 100644 index d3daa13..0000000 --- a/models/silver_cw/nft/silver__nft_giglabs.sql +++ /dev/null @@ -1,337 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -mapped_sales AS ( - SELECT - DISTINCT tx_id - FROM - {{ ref('silver__nft_transactions_secondary_market') }} - UNION - SELECT - DISTINCT tx_id - FROM - {{ ref('silver__nft_topshot_sales') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -duc AS ( - SELECT - DISTINCT tx_id - FROM - events - WHERE - event_contract = 'A.ead892083b3e2c6c.DapperUtilityCoin' -), -duc_events AS ( - SELECT - * - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - duc - ) - AND tx_id NOT IN ( - SELECT - tx_id - FROM - mapped_sales - ) -), -duc_transfers AS ( - SELECT - _inserted_timestamp, - tx_id, - COUNT(event_type) AS event_count, - MAX( - event_index + 1 - ) AS max_index - FROM - duc_events - WHERE - event_type IN ( - 'TokensDeposited', - 'TokensWithdrawn', - 'FeesDeducted', - 'ForwardedDeposit' - ) - GROUP BY - _inserted_timestamp, - tx_id - HAVING - event_count = max_index -), -gig_nfts AS ( - SELECT - * - FROM - duc_events - WHERE - tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - duc_transfers - ) - AND event_contract ILIKE 'A.329feb3ab062d289%' - AND event_type IN ( - 'Withdraw', - 'Deposit' - ) -), -gig_sales_events AS ( - SELECT - * - FROM - events - WHERE - tx_id IN ( - SELECT - DISTINCT tx_id - FROM - gig_nfts - ) -), -missing_contract AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - _inserted_timestamp, - event_contract AS currency, - event_data :amount :: DOUBLE AS amount, - event_data :from :: STRING AS forwarded_from, - TRUE AS missing - FROM - gig_sales_events - WHERE - event_index = 0 - AND event_type = 'TokensWithdrawn' -), -purchase_amt AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - _inserted_timestamp, - 'A.ead892083b3e2c6c.DapperUtilityCoin' AS currency, - event_data :amount :: DOUBLE AS amount, - event_data :from :: STRING AS forwarded_from, - FALSE AS missing - FROM - gig_sales_events - WHERE - event_type = 'ForwardedDeposit' - AND tx_id NOT IN ( - SELECT - tx_id - FROM - missing_contract - ) -), -triage AS ( - SELECT - * - FROM - missing_contract - UNION - SELECT - * - FROM - purchase_amt -), -withdraw_event AS ( - SELECT - tx_id, - block_timestamp, - block_height, - _inserted_timestamp, - event_contract AS nft_collection, - event_data :from :: STRING AS seller, - event_data :id :: STRING AS nft_id - FROM - gig_sales_events - WHERE - event_type = 'Withdraw' - AND event_data :from :: STRING != 'null' -), -deposit_event AS ( - SELECT - tx_id, - block_timestamp, - block_height, - _inserted_timestamp, - event_contract AS nft_collection, - event_data :to :: STRING AS buyer, - event_data :id :: STRING AS nft_id - FROM - gig_sales_events - WHERE - event_type = 'Deposit' - AND event_data :to :: STRING != 'null' -), -gl_sales AS ( - SELECT - p.tx_id, - p.block_timestamp, - p.block_height, - p.tx_succeeded, - p._inserted_timestamp, - 'Gigantik Primary Market' AS marketplace, - p.missing, - p.currency, - p.amount, - p.forwarded_from, - w.seller, - d.buyer, - w.nft_collection, - w.nft_id AS withdraw_nft_id, - d.nft_id AS deposit_nft_id, - w.nft_collection = d.nft_collection AS collection_check, - w.nft_id = d.nft_id AS nft_id_check - FROM - triage p - LEFT JOIN withdraw_event w USING ( - tx_id, - block_timestamp, - block_height, - _inserted_timestamp - ) - LEFT JOIN deposit_event d USING ( - tx_id, - block_timestamp, - block_height, - _inserted_timestamp - ) -), -multi AS ( - SELECT - tx_id, - COUNT( - DISTINCT deposit_nft_id - ) AS nfts - FROM - gl_sales - WHERE - nft_id_check - GROUP BY - 1 -), -giglabs_final AS ( - SELECT - s.tx_id, - block_timestamp, - block_height, - marketplace, - currency, - amount / m.nfts AS price, - seller, - buyer, - nft_collection, - withdraw_nft_id AS nft_id, - m.nfts, - tx_succeeded, - _inserted_timestamp - FROM - gl_sales s - LEFT JOIN multi m USING (tx_id) - WHERE - nft_id_check -), -step_data AS ( - SELECT - tx_id, - event_index, - event_type, - event_data - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - giglabs_final - ) - AND event_type IN ( - 'TokensWithdrawn', - 'TokensDeposited', - 'ForwardedDeposit' - ) -), -counterparty_data AS ( - SELECT - tx_id, - ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( - ORDER BY - event_index - ) AS tokenflow, - ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( - ORDER BY - event_index - ) AS counterparties - FROM - step_data - GROUP BY - 1 -), -FINAL AS ( - SELECT - s.tx_id, - block_timestamp, - block_height, - marketplace, - currency, - price, - seller, - buyer, - nft_collection, - nft_id, - nfts, - tokenflow, - counterparties, - tx_succeeded, - _inserted_timestamp - FROM - giglabs_final s - LEFT JOIN counterparty_data C USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_giglabs.yml b/models/silver_cw/nft/silver__nft_giglabs.yml deleted file mode 100644 index f25b696..0000000 --- a/models/silver_cw/nft/silver__nft_giglabs.yml +++ /dev/null @@ -1,74 +0,0 @@ -version: 2 - -models: - - name: silver__nft_giglabs - description: |- - NFT primary market sales for Giglabs marketplaces. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - seller - - buyer - - nft_collection - - nft_id - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - tests: - - not_null - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - - name: seller - description: "{{ doc('seller') }}" - tests: - - not_null - - - name: price - description: "{{ doc('price') }}" - tests: - - not_null - - - name: currency - description: "{{ doc('currency') }}" - tests: - - not_null - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - - - name: counterparties - description: "{{ doc('counterparties') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_editions.sql b/models/silver_cw/nft/silver__nft_moment_editions.sql deleted file mode 100644 index cdc071f..0000000 --- a/models/silver_cw/nft/silver__nft_moment_editions.sql +++ /dev/null @@ -1,50 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, edition_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'EditionCreated' - AND event_contract IN ( - 'A.e4cf4bdc1751c65d.AllDay', - 'A.b715b81853fef53f.AllDay', - 'A.87ca73a41bb50ad5.Golazos' - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS edition_id, - event_data :maxMintSize :: STRING AS max_mint_size, - event_data :playID :: STRING AS play_id, - event_data :seriesID :: STRING AS series_id, - event_data :setID :: STRING AS set_id, - event_data :tier :: STRING AS tier, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_editions.yml b/models/silver_cw/nft/silver__nft_moment_editions.yml deleted file mode 100644 index 76b4c00..0000000 --- a/models/silver_cw/nft/silver__nft_moment_editions.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_editions - description: |- - Cleaned EditionCreated events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - edition_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: edition_id - description: "{{ doc('edition_id') }}" - - name: max_mint_size - description: "{{ doc('max_mint_size') }}" - - name: play_id - description: "{{ doc('play_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: tier - description: "{{ doc('tier') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_metadata.sql b/models/silver_cw/nft/silver__nft_moment_metadata.sql deleted file mode 100644 index 8634c40..0000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata.sql +++ /dev/null @@ -1,75 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['play_id'], - unique_key = "concat_ws('-', event_contract, play_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH play_creation AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'PlayCreated' - -{# -currently includes the following contracts -A.c38aea683c0c4d38.Eternal -A.b715b81853fef53f.AllDay -A.67af7ecf76556cd3.ABD -A.0b2a3299cc857e29.TopShot -A.5c0992b465832a94.TKNZ -A.e4cf4bdc1751c65d.AllDay -A.87ca73a41bb50ad5.Golazos - #} - - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -play_metadata AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: NUMBER AS play_id, - VALUE :key :value :: STRING AS column_header, - VALUE :value :value :: STRING AS column_value, - _inserted_timestamp - FROM - play_creation, - LATERAL FLATTEN(input => TRY_PARSE_JSON(event_data :metadata)) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - play_id, - _inserted_timestamp, - OBJECT_AGG( - column_header :: variant, - column_value :: variant - ) AS metadata - FROM - play_metadata - GROUP BY - 1, - 2, - 3, - 4, - 5 -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_moment_metadata.yml b/models/silver_cw/nft/silver__nft_moment_metadata.yml deleted file mode 100644 index 2d13a40..0000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata.yml +++ /dev/null @@ -1,22 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_metadata - description: |- - Cleaned PlayCreated events, which is what records the metadata on-chain. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - play_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: play_id - description: "{{ doc('play_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_metadata_final.sql b/models/silver_cw/nft/silver__nft_moment_metadata_final.sql deleted file mode 100644 index 2982e51..0000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata_final.sql +++ /dev/null @@ -1,80 +0,0 @@ -{{ config( - materialized = 'table', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-',event_contract,edition_id,nft_id)", - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH moments AS ( - - SELECT - * - FROM - {{ ref('silver__nft_moment_minted') }} -), -metadata AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_metadata') }} -), -editions AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_editions') }} -), -series AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_series') }} -), -set_nm AS ( - SELECT - * - FROM - {{ ref('silver__nft_moment_set') }} -), -FINAL AS ( - SELECT - -- tx id and block timestamp don't matter for the final table - m.tx_id, - m.block_timestamp, - m.event_contract, - m.nft_id, - m.serial_number, - e.max_mint_size, - e.play_id, - e.series_id, - s.series_name, - e.set_id, - sn.set_name, - e.edition_id, - e.tier, - pl.metadata, - m._inserted_timestamp, - sn._inserted_timestamp AS _inserted_timestamp_set - FROM - moments m - LEFT JOIN editions e USING ( - event_contract, - edition_id - ) - LEFT JOIN metadata pl USING ( - event_contract, - play_id - ) - LEFT JOIN series s USING ( - event_contract, - series_id - ) - LEFT JOIN set_nm sn USING ( - event_contract, - set_id - ) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_moment_metadata_final.yml b/models/silver_cw/nft/silver__nft_moment_metadata_final.yml deleted file mode 100644 index f7e0478..0000000 --- a/models/silver_cw/nft/silver__nft_moment_metadata_final.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_metadata_final - description: |- - Cleaned PlayCreated events. - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: nft_id - description: "{{ doc('nft_id') }}" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: max_mint_size - description: "{{ doc('max_mint_size') }}" - - name: play_id - description: "{{ doc('play_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: series_name - description: "{{ doc('series_name') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: set_name - description: "{{ doc('set_name') }}" - - name: edition_id - description: "{{ doc('edition_id') }}" - - name: tier - description: "{{ doc('tier') }}" - - name: metadata - description: "{{ doc('metadata') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null diff --git a/models/silver_cw/nft/silver__nft_moment_minted.sql b/models/silver_cw/nft/silver__nft_moment_minted.sql deleted file mode 100644 index 879f2b4..0000000 --- a/models/silver_cw/nft/silver__nft_moment_minted.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, edition_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'MomentNFTMinted' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :editionID :: STRING AS edition_id, - event_data :id :: STRING AS nft_id, - event_data :serialNumber :: STRING AS serial_number, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_minted.yml b/models/silver_cw/nft/silver__nft_moment_minted.yml deleted file mode 100644 index a994c45..0000000 --- a/models/silver_cw/nft/silver__nft_moment_minted.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_minted - description: |- - Cleaned MomentNFTMinted events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - nft_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.e4cf4bdc1751c65d.AllDay" - - "A.b715b81853fef53f.AllDay" - - "A.87ca73a41bb50ad5.Golazos" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: edition_id - description: "{{ doc('edition_id') }}" - - name: nft_id - description: "{{ doc('nft_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_minted_2.sql b/models/silver_cw/nft/silver__nft_moment_minted_2.sql deleted file mode 100644 index 1974c65..0000000 --- a/models/silver_cw/nft/silver__nft_moment_minted_2.sql +++ /dev/null @@ -1,43 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, moment_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'MomentMinted' - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :momentID :: STRING AS moment_id, - event_data :serialNumber :: STRING AS serial_number, - event_data :seriesID :: STRING AS series_id, - event_data :setID :: STRING AS set_id, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_minted_2.yml b/models/silver_cw/nft/silver__nft_moment_minted_2.yml deleted file mode 100644 index 88b75c8..0000000 --- a/models/silver_cw/nft/silver__nft_moment_minted_2.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_minted_2 - description: |- - Cleaned MomentNFTMinted events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - moment_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.c38aea683c0c4d38.Eternal" - - "A.d4ad4740ee426334.Moments" - - "A.67af7ecf76556cd3.ABD" - - "A.0b2a3299cc857e29.TopShot" - - name: moment_id - description: "{{ doc('moment_id') }}" - - name: serial_number - description: "{{ doc('serial_number') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: series_id - description: "{{ doc('series_id') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_series.sql b/models/silver_cw/nft/silver__nft_moment_series.sql deleted file mode 100644 index 36dddb0..0000000 --- a/models/silver_cw/nft/silver__nft_moment_series.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, series_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'SeriesCreated' - AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data)) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS series_id, - event_data :name :: STRING AS series_name, - _inserted_timestamp - FROM - events -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_series.yml b/models/silver_cw/nft/silver__nft_moment_series.yml deleted file mode 100644 index 915a0b1..0000000 --- a/models/silver_cw/nft/silver__nft_moment_series.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_series - description: |- - Cleaned SeriesCreated events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - series_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - tests: - - accepted_values: - values: - - "A.e4cf4bdc1751c65d.AllDay" - - "A.b715b81853fef53f.AllDay" - - "A.87ca73a41bb50ad5.Golazos" - - name: series_id - description: "{{ doc('series_id') }}" - - name: series_name - description: "{{ doc('series_name') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moment_set.sql b/models/silver_cw/nft/silver__nft_moment_set.sql deleted file mode 100644 index d96e9ec..0000000 --- a/models/silver_cw/nft/silver__nft_moment_set.sql +++ /dev/null @@ -1,44 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp'], - unique_key = "concat_ws('-', event_contract, set_id)", - incremental_strategy = 'delete+insert', - tags = ['nft', 'dapper', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_type = 'SetCreated' - AND ARRAY_CONTAINS('name' :: variant, object_keys(event_data)) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -org AS ( - SELECT - tx_id, - block_timestamp, - event_contract, - event_data :id :: STRING AS set_id, - event_data :name :: STRING AS set_name, - _inserted_timestamp - FROM - events - WHERE - set_id IS NOT NULL -) -SELECT - * -FROM - org diff --git a/models/silver_cw/nft/silver__nft_moment_set.yml b/models/silver_cw/nft/silver__nft_moment_set.yml deleted file mode 100644 index a86e2f2..0000000 --- a/models/silver_cw/nft/silver__nft_moment_set.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: 2 - -models: - - name: silver__nft_moment_set - description: |- - Cleaned SetCreated events. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - event_contract - - set_id - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - name: event_contract - description: "{{ doc('event_contract') }}" - - name: set_id - description: "{{ doc('set_id') }}" - - name: set_name - description: "{{ doc('set_name') }}" - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_moments.sql b/models/silver_cw/nft/silver__nft_moments.sql deleted file mode 100644 index 93705cf..0000000 --- a/models/silver_cw/nft/silver__nft_moments.sql +++ /dev/null @@ -1,47 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = "CONCAT_WS('-', tx_id, event_index)", - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -moment_events AS ( - SELECT - * - FROM - events - WHERE - event_type IN ( - 'MomentPurchased', - 'MomentLocked', - 'MomentCreated', - 'MomentNFTBurned', - 'MomentListed', - 'MomentDestroyed', - 'MomentWithdrawn', - 'MomentMinted', - 'MomentNFTMinted' - ) -) -SELECT - * -FROM - moment_events diff --git a/models/silver_cw/nft/silver__nft_sales.sql b/models/silver_cw/nft/silver__nft_sales.sql deleted file mode 100644 index f86e2af..0000000 --- a/models/silver_cw/nft/silver__nft_sales.sql +++ /dev/null @@ -1,116 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH topshot AS ( - - SELECT - * - FROM - {{ ref('silver__nft_topshot_sales') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -secondary_mkts AS ( - SELECT - * - FROM - {{ ref('silver__nft_transactions_secondary_market') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -giglabs AS ( - SELECT - * - FROM - {{ ref('silver__nft_giglabs') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -combo AS ( - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _inserted_timestamp, - tokenflow, - counterparties - FROM - topshot - UNION - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _inserted_timestamp, - tokenflow, - counterparties - FROM - secondary_mkts - UNION - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _inserted_timestamp, - tokenflow, - counterparties - FROM - giglabs -) -SELECT - * -FROM - combo diff --git a/models/silver_cw/nft/silver__nft_sales.yml b/models/silver_cw/nft/silver__nft_sales.yml deleted file mode 100644 index 5a14869..0000000 --- a/models/silver_cw/nft/silver__nft_sales.yml +++ /dev/null @@ -1,52 +0,0 @@ -version: 2 - -models: - - name: silver__nft_sales - description: |- - NFT market sales on the Flow blockchain. This table will only contain successful transactions, as failed transactions will not have the requisite events to determine it was an attempted NFT purchase. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: marketplace - description: "{{ doc('marketplace') }}" - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - - - name: nft_id - description: "{{ doc('nft_id') }}" - - - name: buyer - description: "{{ doc('buyer') }}" - - - name: seller - description: "{{ doc('seller') }}" - - - name: price - description: "{{ doc('price') }}" - - - name: currency - description: "{{ doc('currency') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - - - name: counterparties - description: "{{ doc('counterparties') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/nft/silver__nft_topshot_pack_sales.sql b/models/silver_cw/nft/silver__nft_topshot_pack_sales.sql deleted file mode 100644 index 4fe99fc..0000000 --- a/models/silver_cw/nft/silver__nft_topshot_pack_sales.sql +++ /dev/null @@ -1,66 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'merge', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = ['tx_id','nft_id'], - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -nft_txs AS ( - SELECT - tx_id, - event_index, - block_height, - block_timestamp, - tx_succeeded, - event_data :id :: INT AS nft_id, - _inserted_timestamp - FROM - silver_events - WHERE - event_data :from = '0xe1f2a091f7bb5245' - AND event_contract = 'A.0b2a3299cc857e29.TopShot' -) -SELECT - A.tx_id, - A.block_height, - A.block_timestamp, - 'topshot pack' AS marketplace, - NULL AS nft_collection, - A.nft_id, - b.event_data :to :: STRING buyer, - NULL AS seller, - NULL price, - NULL currency, - A.tx_succeeded, - NULL AS tokenflow, - NULL AS counterparties, - MD5( - CAST(COALESCE(CAST(A.tx_id AS VARCHAR), '') AS VARCHAR) - ) AS pack_id, - A._inserted_timestamp -FROM - nft_txs A - JOIN silver_events b - ON A.tx_id = b.tx_id - AND A.nft_id :: STRING = b.event_data :id :: STRING -WHERE - event_data :to IS NOT NULL - AND A.event_index <> b.event_index diff --git a/models/silver_cw/nft/silver__nft_topshot_pack_sales.yml b/models/silver_cw/nft/silver__nft_topshot_pack_sales.yml deleted file mode 100644 index 7a2bbe1..0000000 --- a/models/silver_cw/nft/silver__nft_topshot_pack_sales.yml +++ /dev/null @@ -1,100 +0,0 @@ -version: 2 - -models: - - name: silver__nft_topshot_pack_sales - description: TopShot pack sales - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - NUMBER - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: seller - description: "{{ doc('seller') }}" - - - name: price - description: "{{ doc('price') }}" - - - name: currency - description: "{{ doc('currency') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - - - name: counterparties - description: "{{ doc('counterparties') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/nft/silver__nft_topshot_sales.sql b/models/silver_cw/nft/silver__nft_topshot_sales.sql deleted file mode 100644 index 05e3545..0000000 --- a/models/silver_cw/nft/silver__nft_topshot_sales.sql +++ /dev/null @@ -1,158 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -moment_data AS ( - SELECT - block_height, - block_timestamp, - tx_id, - event_contract :: STRING AS marketplace, - event_data :id :: STRING AS nft_id, - event_data :price :: DOUBLE AS price, - event_data :seller :: STRING AS seller, - tx_succeeded, - _ingested_at, - _inserted_timestamp - FROM - silver_events - WHERE - event_type = 'MomentPurchased' - AND event_contract LIKE 'A.c1e4f4f4c4257510%' -- topshot -), -currency_data AS ( - SELECT - tx_id, - event_contract :: STRING AS currency - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - moment_data - ) - AND event_index = 0 -), -nft_data AS ( - SELECT - tx_id, - event_contract :: STRING AS nft_collection, - event_data :to :: STRING AS buyer - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - moment_data - ) - AND event_type = 'Deposit' -), -combo AS ( - SELECT - tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _ingested_at, - _inserted_timestamp - FROM - moment_data - LEFT JOIN currency_data USING (tx_id) - LEFT JOIN nft_data USING (tx_id) -), -step_data AS ( - SELECT - tx_id, - event_index, - event_type, - event_data - FROM - {{ ref('silver__events_final') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - combo - ) - AND event_type IN ( - 'TokensWithdrawn', - 'TokensDeposited', - 'ForwardedDeposit' - ) -), -counterparty_data AS ( - SELECT - tx_id, - ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( - ORDER BY - event_index - ) AS tokenflow, - ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( - ORDER BY - event_index - ) AS counterparties - FROM - step_data - GROUP BY - 1 -), -FINAL AS ( - SELECT - C.tx_id, - block_height, - block_timestamp, - marketplace, - nft_collection, - nft_id, - buyer, - seller, - price, - currency, - tx_succeeded, - _ingested_at, - _inserted_timestamp, - cd.tokenflow, - cd.counterparties - FROM - combo C - LEFT JOIN counterparty_data cd USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/nft/silver__nft_topshot_sales.yml b/models/silver_cw/nft/silver__nft_topshot_sales.yml deleted file mode 100644 index 395f2e6..0000000 --- a/models/silver_cw/nft/silver__nft_topshot_sales.yml +++ /dev/null @@ -1,139 +0,0 @@ -version: 2 - -models: - - name: silver__nft_topshot_sales - description: |- - TopShot direct market sales on the Flow blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - NUMBER - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: seller - description: "{{ doc('seller') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: price - description: "{{ doc('price') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: currency - description: "{{ doc('currency') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: counterparties - description: "{{ doc('counterparties') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/nft/silver__nft_transactions_secondary_market.sql b/models/silver_cw/nft/silver__nft_transactions_secondary_market.sql deleted file mode 100644 index a08a6a8..0000000 --- a/models/silver_cw/nft/silver__nft_transactions_secondary_market.sql +++ /dev/null @@ -1,509 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['nft', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -sale_trigger AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_contract AS marketplace, - event_data, - COALESCE( - event_data :purchased :: BOOLEAN, - event_data :accepted :: BOOLEAN, - IFF( - event_data :status = 'sold' - OR event_data :status IS NULL, - TRUE, - FALSE - ), - TRUE - ) AS is_purchased, - _ingested_at, - _inserted_timestamp - FROM - silver_events - WHERE - is_purchased - AND -- each market uses a slightly different sale trigger - ( - ( - event_contract = 'A.30cf5dcf6ea8d379.AeraPack' - AND event_type = 'Purchased' - ) - OR ( - event_contract = 'A.8f9231920da9af6d.AFLPack' - AND event_type = 'PackBought' - ) - OR ( - event_contract = 'A.e2e1689b53e92a82.AniqueMarket' - AND event_type = 'CollectiblePurchased' - ) - OR ( - event_contract = 'A.9969d64233d69723.BlockleteMarket_NFT_V2' - AND event_type = 'BlockletePurchased' - ) - OR ( - event_contract = 'A.64f83c60989ce555.ChainmonstersMarketplace' - AND event_type = 'CollectionRemovedSaleOffer' - ) - OR ( - event_contract = 'A.c8c340cebd11f690.DarkCountryMarket' - AND event_type = 'SaleOfferAccepted' - ) - OR ( - event_contract = 'A.921ea449dffec68a.FlovatarMarketplace' - AND event_type IN ( - 'FlovatarPurchased', - 'FlovatarComponentPurchased' - ) - ) - OR ( - event_contract = 'A.09e03b1f871b3513.TheFabricantMarketplace' - AND event_type = 'NFTPurchased' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindMarketAuctionEscrow' - AND event_type = 'EnglishAuction' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindMarketDirectOfferEscrow' - AND event_type = 'DirectOffer' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindMarketSale' - AND event_type = 'Sale' - ) - OR ( - event_contract = 'A.097bafa4e0b48eef.FindPack' - AND event_type = 'Purchased' - ) - OR ( - event_contract = 'A.8b148183c28ff88f.GaiaOrder' - AND event_type = 'OrderClosed' - ) - OR ( - event_contract = 'A.abda6627c70c7f52.GeniaceMarketplace' - AND event_type = 'SaleOfferCompleted' - ) - OR ( - event_contract = 'A.82ed1b9cba5bb1b3.KaratNFTMarket' - AND event_type = 'SaleOfferAccepted' - ) - OR ( - event_contract = 'A.2162bbe13ade251e.MatrixMarketOpenOffer' - AND event_type = 'OfferCompleted' - ) - OR ( - event_contract = 'A.49b8e5d4d66ae880.MintStoreMarketFactory' - AND event_type = 'MintStoreItemPurchased' - ) - OR ( - event_contract = 'A.a49cc0ee46c54bfb.MotoGPNFTStorefront' - AND event_type = 'SaleOfferCompleted' - ) - OR ( - event_contract = 'A.b8ea91944fd51c43.Offers' - AND event_type = 'OfferCompleted' - ) - OR ( - event_contract = 'A.b8ea91944fd51c43.OffersV2' - AND event_type = 'OfferCompleted' - ) - OR ( - event_contract = 'A.856bd81e73e6752b.PonsNftMarketContract' - AND event_type = 'PonsNFTSold' - ) - OR ( - event_contract = 'A.52cbea4e6f616b8e.PublishedNFTStorefront' - AND event_type = 'ListingCompleted' - ) - OR ( - event_contract = 'A.489fcc527edc21cf.TuneGOMarket' - AND event_type = 'SaleOfferAccepted' - ) - OR ( - event_contract = 'A.4eb8a10cb9f87357.NFTStorefront' -- general storefront - AND event_type = 'ListingCompleted' - ) - OR ( - event_contract = 'A.4eb8a10cb9f87357.NFTStorefrontV2' -- funds move in 2ND TOKEN MVMT not FIRST - AND event_type = 'ListingCompleted' - ) - OR ( - event_contract = 'A.85b8bbf926dcddfa.NFTStoreFront' - AND event_type = 'ListingSold' - ) - OR ( - event_contract = 'A.85b075e08d13f697.OlympicPinMarket' - AND event_type = 'PiecePurchased' - ) - OR ( - event_contract = 'A.5b82f21c0edf76e3.StarlyCardMarket' - AND event_type = 'CollectionRemovedSaleOffer' - ) - OR ( - event_contract = 'A.62b3063fbe672fc8.ZeedzMarketplace' - AND event_type = 'RemovedListing' - ) - ) -), -num_triggers AS ( - SELECT - tx_id, - -- storing the marketplace contract interactions - ARRAY_AGG(marketplace) within GROUP ( - ORDER BY - marketplace - ) AS marketplaces, - -- compare total sales (by listing id) with distinct to eliminate the case where 2 marketplaces - -- (general & gaia) are called for 1 sale - ARRAY_AGG( - COALESCE( - event_data :orderId, - -- general - event_data :listingResourceID, - -- gaia, zeedz - event_data :saleItemID, - --chainmonster - event_data :itemID, - --starly, darkcountry - event_data :id, - -- olympic pin, flovatar, fina - event_data :saleOfferResourceID, - -- moto gp - event_data :bidId, - -- matrix - event_data :listingID, - -- fabricant - event_data :templateId, - -- AFLPack - event_data :saleOfferId, - -- tunego - event_data :offerId, - -- OffersV2 - event_data :nftId, - -- pons doesn't do order ids - event_data :packId -- find pack, aera - ) - ) AS sale_ids, - ARRAY_AGG( - DISTINCT COALESCE( - event_data :orderId, - -- general - event_data :listingResourceID, - -- gaia, zeedz - event_data :saleItemID, - --chainmonster - event_data :itemID, - --starly, darkcountry - event_data :id, - -- olympic pin, flovatar, fina - event_data :saleOfferResourceID, - -- moto gp - event_data :bidId, - -- matrix - event_data :listingID, - -- fabricant - event_data :templateId, - -- AFLPack - event_data :saleOfferId, - -- tunego - event_data :offerId, - -- OffersV2 - event_data :nftId, - -- pons doesn't do order ids - event_data :packId -- find pack, aera - ) - ) AS dist_sale_ids, - COUNT(1) AS sale_trigger_count, - ARRAY_SIZE(dist_sale_ids) AS num_sales - FROM - sale_trigger - GROUP BY - 1 -), -omit_nft_nontransfers AS ( - SELECT - tx_id, - ARRAY_AGG( - DISTINCT event_type - ) AS events, - -- don't forget to update below if adding any new movement method ! - ARRAY_SIZE( - array_intersection( - ['Deposit', 'Withdraw', 'FlovatarSaleWithdrawn', 'FlovatarComponentSaleWithdrawn'], - events - ) - ) = 2 AS nft_transferred, - count_if( - event_type = 'Deposit' - ) AS nft_deposits - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - num_triggers - WHERE - num_sales < 2 - ) - GROUP BY - 1 - HAVING - nft_deposits = 1 -), -first_token_withdraw AS ( - SELECT - tx_id, - MIN(event_index) AS min_index - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_type = 'TokensWithdrawn' - GROUP BY - 1 -), --- 3 most important events are the first TokenWithdraw, then Withdraw and Deposit (NFT movement) -token_withdraw_event AS ( - SELECT - tx_id, - event_contract AS currency, - event_data :amount :: DOUBLE AS amount, - event_data :from :: STRING AS buyer_purchase, - min_index - FROM - silver_events - LEFT JOIN first_token_withdraw USING (tx_id) - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_index = min_index - AND event_type = 'TokensWithdrawn' -), -nft_withdraw_event_seller AS ( - SELECT - tx_id, - event_index AS event_index_seller, - event_contract AS nft_collection_seller, - COALESCE( - event_data :from, - event_data :address - ) :: STRING AS seller, - COALESCE( - event_data :id, - event_data :tokenId - ) :: STRING AS nft_id_seller - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_type IN ( - 'Withdraw', - 'FlovatarSaleWithdrawn', - 'FlovatarComponentSaleWithdrawn' -- if adding anything new, don't forget about omit_nft_nontransfers check! - ) -), -nft_deposit_event_buyer AS ( - SELECT - tx_id, - event_contract AS nft_collection_deposit, - event_data :id :: STRING AS nft_id_deposit, - event_data :to :: STRING AS buyer_deposit - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) - AND event_type = 'Deposit' -), -nft_sales AS ( - SELECT - e.tx_id, - e.block_timestamp, - e.block_height, - e.tx_succeeded, - e.is_purchased, - e.marketplace, - w.currency, - IFF( - e.marketplace = 'A.4eb8a10cb9f87357.NFTStorefrontV2', - e.event_data :salePrice :: DOUBLE, - w.amount - ) AS amount, - w.buyer_purchase, - s.nft_collection_seller, - s.seller, - s.nft_id_seller, - b.nft_collection_deposit, - b.nft_id_deposit, - b.buyer_deposit, - e._ingested_at, - e._inserted_timestamp - FROM - sale_trigger e - LEFT JOIN token_withdraw_event w USING (tx_id) - LEFT JOIN nft_withdraw_event_seller s USING (tx_id) - LEFT JOIN nft_deposit_event_buyer b USING (tx_id) - WHERE - tx_id IN ( - SELECT - tx_id - FROM - omit_nft_nontransfers - WHERE - nft_transferred - ) -), -step_data AS ( - SELECT - tx_id, - event_index, - event_type, - event_data - FROM - silver_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - nft_sales - ) - AND event_type IN ( - 'TokensWithdrawn', - 'TokensDeposited', - 'ForwardedDeposit', - 'RoyaltyDeposited' - ) -), -counterparty_data AS ( - SELECT - tx_id, - ARRAY_AGG(OBJECT_CONSTRUCT(event_type, event_data)) within GROUP ( - ORDER BY - event_index - ) AS tokenflow, - ARRAY_SIZE(tokenflow) AS steps, - ARRAY_AGG(event_type) within GROUP ( - ORDER BY - event_index - ) AS action, - ARRAY_AGG(event_data) within GROUP ( - ORDER BY - event_index - ) AS step_data, - ARRAY_AGG(COALESCE(event_data :to, event_data :from) :: STRING) within GROUP ( - ORDER BY - event_index - ) AS counterparties - FROM - step_data - GROUP BY - 1 -), -FINAL AS ( - SELECT - ns.tx_id, - block_timestamp, - block_height, - marketplace, - nft_collection_deposit AS nft_collection, - nft_id_seller AS nft_id, - currency, - amount AS price, - seller, - buyer_deposit AS buyer, - cd.tokenflow, - cd.steps AS num_steps, - cd.action AS step_action, - cd.step_data, - cd.counterparties, - tx_succeeded, - _ingested_at, - _inserted_timestamp - FROM - nft_sales ns - LEFT JOIN counterparty_data cd USING (tx_id) -), -dedupe_gaia AS ( - SELECT - * - FROM - FINAL - WHERE - tx_id IN ( - SELECT - tx_id - FROM - num_triggers - WHERE - sale_trigger_count = 2 - AND num_sales = 1 - ) qualify ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - marketplace - ) = 1 -) -SELECT - * -FROM - FINAL -EXCEPT -SELECT - * -FROM - dedupe_gaia diff --git a/models/silver_cw/nft/silver__nft_transactions_secondary_market.yml b/models/silver_cw/nft/silver__nft_transactions_secondary_market.yml deleted file mode 100644 index a45fbf4..0000000 --- a/models/silver_cw/nft/silver__nft_transactions_secondary_market.yml +++ /dev/null @@ -1,175 +0,0 @@ -version: 2 - -models: - - name: silver__nft_transactions_secondary_market - description: |- - This table filters all NFT sales that interact with the general purpose contract `A.4eb8a10cb9f87357.NFTStorefront`. Transactions that purchase multiple NFTs in a single transaction are presently excluded from this table! - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - seller - - buyer - - nft_collection - - nft_id - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: marketplace - description: "{{ doc('marketplace') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_collection - description: "{{ doc('nft_collection') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: nft_id - description: "{{ doc('nft_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - NUMBER - - - name: currency - description: "{{ doc('currency') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: price - description: "{{ doc('price') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: seller - description: "{{ doc('seller') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: buyer - description: "{{ doc('buyer') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: tokenflow - description: "{{ doc('tokenflow') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: num_steps - description: "{{ doc('num_steps') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: step_action - description: "{{ doc('step_action') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: step_data - description: "{{ doc('step_data') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: counterparties - description: "{{ doc('counterparties') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - ARRAY - - OBJECT - - VARIANT - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - BOOLEAN - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/price/silver__prices_swaps.sql b/models/silver_cw/price/silver__prices_swaps.sql deleted file mode 100644 index 25309ff..0000000 --- a/models/silver_cw/price/silver__prices_swaps.sql +++ /dev/null @@ -1,181 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['block_timestamp::date'], - unique_key = "CONCAT_WS('-', block_timestamp, token_contract)", - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH swaps AS ( - - SELECT - tx_id, - block_timestamp, - block_height, - swap_contract, - trader, - token_out_amount, - token_out_contract, - token_in_amount, - token_in_contract, - _inserted_timestamp - FROM - {{ ref('silver__swaps') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -flow_price AS ( - SELECT - recorded_hour as _timestamp, - open as price_usd - FROM - {{ ref('silver__prices_hourly') }} - WHERE - token = 'Flow' -), -stable_out AS ( - SELECT - tx_id, - block_timestamp, - token_in_contract AS token_contract, - token_out_amount / nullifzero(token_in_amount) AS swap_price, - _inserted_timestamp, - 'stableswap' AS source - FROM - swaps - WHERE - token_out_contract IN ( - 'A.cfdd90d4a00f7b5b.TeleportedTetherToken', - 'A.3c5959b568896393.FUSD', - 'A.b19436aae4d94622.FiatToken' - ) -), -stable_in AS ( - SELECT - tx_id, - block_timestamp, - token_out_contract AS token_contract, - token_in_amount / nullifzero(token_out_amount) AS swap_price, - _inserted_timestamp, - 'stableswap' AS source - FROM - swaps - WHERE - token_in_contract IN ( - 'A.cfdd90d4a00f7b5b.TeleportedTetherToken', - 'A.3c5959b568896393.FUSD', - 'A.b19436aae4d94622.FiatToken' - ) -), -stbl_tbl_union AS ( - SELECT - * - FROM - stable_out - UNION - SELECT - * - FROM - stable_in -), -flow_out AS ( - SELECT - tx_id, - block_timestamp, - token_in_contract AS token_contract, - token_out_amount / nullifzero(token_in_amount) AS swap_price_in_flow, - _inserted_timestamp - FROM - swaps - WHERE - token_out_contract = 'A.1654653399040a61.FlowToken' -), -flow_in AS ( - SELECT - tx_id, - block_timestamp, - token_in_contract AS token_contract, - token_out_amount / nullifzero(token_in_amount) AS swap_price_in_flow, - _inserted_timestamp - FROM - swaps - WHERE - token_out_contract = 'A.1654653399040a61.FlowToken' -), -flow_tbl_union AS ( - SELECT - tx_id, - DATE_TRUNC( - 'hour', - block_timestamp - ) AS _timestamp, - token_contract, - swap_price_in_flow, - _inserted_timestamp, - 'flowswap' AS source - FROM - flow_out - UNION - SELECT - tx_id, - DATE_TRUNC( - 'hour', - block_timestamp - ) AS _timestamp, - token_contract, - swap_price_in_flow, - _inserted_timestamp, - 'flowswap' AS source - FROM - flow_in -), -to_usd AS ( - SELECT - tx_id, - ftu._timestamp, - token_contract, - swap_price_in_flow, - swap_price_in_flow * p.price_usd AS swap_price_usd, - _inserted_timestamp, - source - FROM - flow_tbl_union ftu - LEFT JOIN flow_price p USING (_timestamp) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - token_contract, - swap_price, - _inserted_timestamp, - source - FROM - stbl_tbl_union - UNION - SELECT - tx_id, - _timestamp AS block_timestamp, - token_contract, - swap_price_usd AS swap_price, - _inserted_timestamp, - source - FROM - to_usd - WHERE - swap_price IS NOT NULL -) -SELECT - * -FROM - FINAL -WHERE - swap_price IS NOT NULL diff --git a/models/silver_cw/price/silver__prices_swaps.yml b/models/silver_cw/price/silver__prices_swaps.yml deleted file mode 100644 index ca0c573..0000000 --- a/models/silver_cw/price/silver__prices_swaps.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: 2 - -models: - - name: silver__prices_swaps - description: |- - This table provides token price data for FLOW tokens. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARCHAR - - STRING - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: token_contract - description: "{{ doc('token_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - VARCHAR - - STRING - - - name: swap_price - description: "{{ doc('swap_price') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - FLOAT - - DOUBLE - - NUMBER - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: source - description: "{{ doc('source') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR diff --git a/models/silver_cw/price/silver__prices_swaps_hourly.sql b/models/silver_cw/price/silver__prices_swaps_hourly.sql deleted file mode 100644 index 2f0038c..0000000 --- a/models/silver_cw/price/silver__prices_swaps_hourly.sql +++ /dev/null @@ -1,78 +0,0 @@ -{{ config( - materialized = 'table', - cluster_by = ['recorded_hour::date'], - unique_key = "CONCAT_WS('-', recorded_hour, token)", - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH swap_prices AS ( - - SELECT - * - FROM - {{ ref('silver__prices_swaps') }} - -), -lowhigh AS ( - SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS recorded_hour, - token_contract AS token, - MIN(swap_price) AS low, - MAX(swap_price) AS high, - COUNT( - DISTINCT tx_id - ) AS num_swaps - FROM - swap_prices - GROUP BY - 1, - 2 -), -openclose AS ( - SELECT - DATE_TRUNC( - 'hour', - block_timestamp - ) AS recorded_hour, - token_contract AS token, - FIRST_VALUE(swap_price) over ( - PARTITION BY recorded_hour, - token - ORDER BY - block_timestamp - ) AS OPEN, - LAST_VALUE(swap_price) over ( - PARTITION BY recorded_hour, - token - ORDER BY - block_timestamp - ) AS CLOSE - FROM - swap_prices qualify ROW_NUMBER() over (PARTITION BY concat_ws('-', recorded_hour, token) - ORDER BY - recorded_hour DESC) = 1 -), -FINAL AS ( - SELECT - l.recorded_hour, - l.token as id, - OPEN, - high, - low, - CLOSE, - num_swaps, - 'Swaps' as provider - FROM - lowhigh l - LEFT JOIN openclose o USING ( - recorded_hour, - token - ) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/price/silver__prices_swaps_hourly.yml b/models/silver_cw/price/silver__prices_swaps_hourly.yml deleted file mode 100644 index f7e6266..0000000 --- a/models/silver_cw/price/silver__prices_swaps_hourly.yml +++ /dev/null @@ -1,32 +0,0 @@ - -version: 2 - -models: - - name: silver__prices_swaps_hourly - description: |- - This table provides hourly token price data for FLOW tokens from on-chain swaps - - columns: - - name: recorded_hour - description: "{{ doc('recorded_hour') }}" - - - name: id - description: "{{ doc('asset_id') }}" - - - name: token - description: "{{ doc('token') }}" - - - name: open - description: "{{ doc('open') }}" - - - name: high - description: "{{ doc('high') }}" - - - name: low - description: "{{ doc('low') }}" - - - name: close - description: "{{ doc('asset_id') }}" - - - name: provider - description: "{{ doc('provider') }}" diff --git a/models/silver_cw/staking/silver__staking_actions.sql b/models/silver_cw/staking/silver__staking_actions.sql deleted file mode 100644 index b230afc..0000000 --- a/models/silver_cw/staking/silver__staking_actions.sql +++ /dev/null @@ -1,99 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH silver_events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -flow_staking AS ( - SELECT - tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - event_contract, - event_type AS action, - event_data :amount :: FLOAT AS amount, - event_data :delegatorID :: STRING AS delegator_id, - event_data :nodeID :: STRING AS node_id, - _inserted_timestamp - FROM - silver_events - WHERE - event_contract = 'A.8624b52f9ddcd04a.FlowIDTableStaking' - AND event_type IN ( - 'DelegatorTokensCommitted', - 'DelegatorRewardTokensWithdrawn', - 'DelegatorUnstakedTokensWithdrawn', - 'TokensCommitted', - 'RewardTokensWithdrawn', - 'UnstakedTokensWithdrawn' - ) -), -add_auth AS ( - SELECT - tx_id, - COALESCE( - authorizers [1], - authorizers [0] - ) :: STRING AS primary_authorizer - FROM - {{ ref('silver__transactions') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - flow_staking - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -FINAL AS ( - SELECT - s.tx_id, - event_index, - block_timestamp, - block_height, - tx_succeeded, - primary_authorizer AS delegator, - action, - amount, - node_id, - _inserted_timestamp - FROM - flow_staking s - LEFT JOIN add_auth A USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/staking/silver__staking_actions.yml b/models/silver_cw/staking/silver__staking_actions.yml deleted file mode 100644 index d770ffa..0000000 --- a/models/silver_cw/staking/silver__staking_actions.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -models: - - name: silver__staking_actions - description: |- - This table provides transaction-level info on FLOW staking activities. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: event_index - description: "{{ doc('event_index') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: delegator - description: "{{ doc('delegator') }}" - - - name: action - description: "{{ doc('action') }}" - - - name: amount - description: "{{ doc('amount') }}" - - - name: node_id - description: "{{ doc('node_id') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/swaps/silver__swaps.sql b/models/silver_cw/swaps/silver__swaps.sql deleted file mode 100644 index 086a737..0000000 --- a/models/silver_cw/swaps/silver__swaps.sql +++ /dev/null @@ -1,318 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = "CONCAT_WS('-', tx_id, swap_index)", - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__swaps_events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -swap_events AS ( - SELECT - * - FROM - events - WHERE - tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - events - WHERE - event_type = 'RewardTokensWithdrawn' - OR event_type = 'NFTReceived' - ) -- PierPair needs a bespoke model as it does not deposit traded token to Pool contract - AND tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - events - WHERE - event_contract LIKE '%PierPair%' - ) -), -pool_info AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_index, - event_type, - RANK() over ( - PARTITION BY tx_id - ORDER BY - event_index - ) - 1 AS swap_index, - event_contract AS pool_contract, - CASE - WHEN object_keys(event_data) [0] :: STRING = 'side' THEN 'Blocto' - WHEN object_keys(event_data) [0] :: STRING = 'direction' THEN 'Increment' - WHEN object_keys(event_data) [3] :: STRING = 'swapAForB' THEN 'Metapier' - ELSE 'Other' - END AS likely_dex, - COALESCE( - event_data :direction :: NUMBER, - event_data :side :: NUMBER - 1, - event_data :swapAForB :: BOOLEAN :: NUMBER - ) AS direction, - COALESCE( - event_data :inTokenAmount, - event_data :token1Amount, - event_data :amountIn - ) :: DOUBLE AS in_token_amount, - COALESCE( - event_data :outTokenAmount, - event_data :token2Amount, - event_data :amountOut - ) :: DOUBLE AS out_token_amount, - _inserted_timestamp - FROM - swap_events - WHERE - event_type IN ( - 'Trade', - 'Swap' - ) -), -token_withdraws AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_index, - RANK() over ( - PARTITION BY tx_id - ORDER BY - event_index - ) - 1 AS token_index, - RANK() over ( - PARTITION BY CONCAT( - tx_id, - event_data :amount :: STRING, - event_data :from :: STRING - ) - ORDER BY - event_index - ) - 1 AS unique_order, - event_contract, - event_data, - _inserted_timestamp - FROM - swap_events - WHERE - event_type = 'TokensWithdrawn' - AND tx_id IN ( - SELECT - DISTINCT tx_id - FROM - pool_info - ) -), -token_deposits AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_index, - RANK() over ( - PARTITION BY tx_id - ORDER BY - event_index - ) - 1 AS token_index, - RANK() over ( - PARTITION BY CONCAT( - tx_id, - event_data :amount :: STRING, - event_data :to :: STRING - ) - ORDER BY - event_index - ) - 1 AS unique_order, - event_contract, - event_data, - _inserted_timestamp - FROM - swap_events - WHERE - event_type = 'TokensDeposited' - AND tx_id IN ( - SELECT - DISTINCT tx_id - FROM - pool_info - ) -), -link_token_movement AS ( - SELECT - w.tx_id, - w.block_timestamp, - w.block_height, - w._inserted_timestamp, - -- set transfer index based on execution via deposit, not withdraw, event - RANK() over ( - PARTITION BY w.tx_id - ORDER BY - d.event_index - ) - 1 AS transfer_index, - w.event_data :from :: STRING AS withdraw_from, - d.event_data :to :: STRING AS deposit_to, - w.event_data :amount :: DOUBLE AS amount, - w.event_contract AS token_contract, - w.event_contract = d.event_contract AS contract_check - FROM - token_withdraws w - LEFT JOIN token_deposits d - ON w.tx_id = d.tx_id - AND w.event_contract = d.event_contract - AND w.event_data :amount :: STRING = d.event_data :amount :: STRING - AND w.unique_order = d.unique_order -), -restructure AS ( - SELECT - t.tx_id, - t.transfer_index, - p.swap_index, - RANK() over ( - PARTITION BY t.tx_id, - swap_index - ORDER BY - transfer_index - ) - 1 AS token_position, - t.withdraw_from, - t.deposit_to, - CONCAT('0x', SPLIT(pool_contract, '.') [1]) AS pool_address, - sub.trader, - ARRAYS_OVERLAP(ARRAY_CONSTRUCT(t.withdraw_from, t.deposit_to), ARRAY_CONSTRUCT(pool_address, sub.trader)) AS transfer_involve_pool_or_trader, - t.amount, - t.token_contract, - p.pool_contract, - p.direction, - p.in_token_amount, - p.out_token_amount - FROM - link_token_movement t - LEFT JOIN pool_info p - ON p.tx_id = t.tx_id - AND ( - p.in_token_amount = t.amount -- blocto takes a 0.3% fee - OR ROUND((p.in_token_amount / 0.997) - t.amount) = 0 - OR p.out_token_amount = t.amount - OR ROUND((p.out_token_amount / 0.997) - t.amount) = 0 - ) - AND transfer_index >= swap_index - LEFT JOIN ( - SELECT - tx_id, - withdraw_from AS trader - FROM - link_token_movement - WHERE - transfer_index = 0 - ) sub - ON t.tx_id = sub.tx_id - WHERE - swap_index IS NOT NULL -- exclude the network fee token movement - AND transfer_involve_pool_or_trader -), --- there are some cases where the same token is transferred multiple times in a single swap --- causing an error with object agg -excl_dup_transfer AS ( - SELECT - tx_id, - pool_contract, - swap_index, - CONCAT( - 'token', - token_position - ), - COUNT(1) AS identical_transfer_count - FROM - restructure - GROUP BY - 1, - 2, - 3, - 4 - HAVING - identical_transfer_count > 1 -), -pool_token_alignment AS ( - SELECT - tx_id, - pool_contract, - swap_index, - OBJECT_AGG(CONCAT('token', token_position), token_contract :: variant) AS tokens, - OBJECT_AGG(CONCAT('amount', token_position), amount) AS amounts, - OBJECT_AGG(CONCAT('from', token_position), withdraw_from :: variant) AS withdraws, - OBJECT_AGG(CONCAT('to', token_position), deposit_to :: variant) AS deposits - FROM - restructure - WHERE - tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - excl_dup_transfer - ) - GROUP BY - 1, - 2, - 3 -), -boilerplate AS ( - SELECT - tx_id, - block_timestamp, - block_height, - _inserted_timestamp, - withdraw_from AS trader - FROM - link_token_movement - WHERE - transfer_index = 0 - AND tx_id NOT IN ( - SELECT - DISTINCT tx_id - FROM - excl_dup_transfer - ) -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - pool_contract AS swap_contract, - swap_index, - trader, - withdraws :from0 :: STRING AS token_out_source, - tokens :token0 :: STRING AS token_out_contract, - amounts :amount0 :: DOUBLE AS token_out_amount, - deposits :to1 :: STRING AS token_in_destination, - tokens :token1 :: STRING AS token_in_contract, - amounts :amount1 :: DOUBLE AS token_in_amount, - _inserted_timestamp - FROM - boilerplate - LEFT JOIN pool_token_alignment USING (tx_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/swaps/silver__swaps.yml b/models/silver_cw/swaps/silver__swaps.yml deleted file mode 100644 index 8929059..0000000 --- a/models/silver_cw/swaps/silver__swaps.yml +++ /dev/null @@ -1,112 +0,0 @@ -version: 2 - -models: - - name: silver__swaps - description: |- - This table records asset swaps on the Flow blockchain. - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - trader - - swap_index - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: swap_index - description: "{{ doc('swap_index') }}" - - - name: trader - description: "{{ doc('trader') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_out_source - description: "{{ doc('token_out_source') }}" - - - name: token_out_amount - description: "{{ doc('token_out_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_out_contract - description: "{{ doc('token_out_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_in_destination - description: "{{ doc('token_in_destination') }}" - - - name: token_in_amount - description: "{{ doc('token_in_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_in_contract - description: "{{ doc('token_in_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/swaps/silver__swaps_events.sql b/models/silver_cw/swaps/silver__swaps_events.sql deleted file mode 100644 index a7d454b..0000000 --- a/models/silver_cw/swaps/silver__swaps_events.sql +++ /dev/null @@ -1,65 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = "CONCAT_WS('-', tx_id, event_index)", - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH swap_contracts AS ( - - SELECT - * - FROM - {{ ref('silver__contract_labels') }} - WHERE - contract_name LIKE '%SwapPair%' -), -swaps_txs AS ( - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - event_contract IN ( - SELECT - event_contract - FROM - swap_contracts - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -), -swap_events AS ( - SELECT - * - FROM - {{ ref('silver__events_final') }} - WHERE - tx_id IN ( - SELECT - tx_id - FROM - swaps_txs - ) - -{% if is_incremental() %} -AND _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} -) -{% endif %} -) -SELECT - * -FROM - swap_events diff --git a/models/silver_cw/swaps/silver__swaps_events.yml b/models/silver_cw/swaps/silver__swaps_events.yml deleted file mode 100644 index 880ba17..0000000 --- a/models/silver_cw/swaps/silver__swaps_events.yml +++ /dev/null @@ -1,37 +0,0 @@ -version: 2 - -models: - - name: silver__swaps_events - description: |- - This table records events from each Swap related transaction on the FLOW blockchain, as determined by an interaction with a SwapPair contract. This is intended to be internal only for downstream usage. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" - - - name: event_index - description: "{{ doc('event_index') }}" - - - name: event_contract - description: "{{ doc('event_contract') }}" - - - name: event_type - description: "{{ doc('event_type') }}" - - - name: event_data - description: "{{ doc('event_attributes') }}" - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" diff --git a/models/silver_cw/swaps/silver__swaps_single_trade.sql b/models/silver_cw/swaps/silver__swaps_single_trade.sql deleted file mode 100644 index 1400ba5..0000000 --- a/models/silver_cw/swaps/silver__swaps_single_trade.sql +++ /dev/null @@ -1,174 +0,0 @@ -{{ config( - materialized = 'incremental', - cluster_by = ['_inserted_timestamp::DATE'], - unique_key = 'tx_id', - incremental_strategy = 'delete+insert', - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH swaps_events AS ( - - SELECT - * - FROM - {{ ref('silver__swaps_events') }} - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -action_ct AS ( - SELECT - tx_id, - event_type, - COUNT(1) AS n - FROM - swaps_events - WHERE - event_type IN ( - 'Trade', - 'Swap' - ) - GROUP BY - 1, - 2 -), -step_ct AS ( - SELECT - tx_id, - OBJECT_AGG( - event_type, - n - ) AS ob - FROM - action_ct - GROUP BY - 1 -), -single_trade AS ( - SELECT - tx_id - FROM - step_ct - WHERE - ob :Trade = 1 - AND ob :Swap IS NULL -), -swaps_single_trade AS ( - SELECT - * - FROM - swaps_events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - single_trade - ) -), -index_id AS ( - SELECT - tx_id, - event_type, - MIN(event_index) AS event_index - FROM - swaps_single_trade - WHERE - event_type = 'TokensWithdrawn' - GROUP BY - 1, - 2 -), -token_out_data AS ( - SELECT - sst.tx_id, - block_timestamp, - block_height, - event_contract AS event_contract_token_out, - event_data AS event_data_token_out, - event_data :amount :: DOUBLE AS token_amount_token_out, - LOWER( - event_data :from :: STRING - ) AS trader_token_out, - _ingested_at, - _inserted_timestamp - FROM - index_id ii - LEFT JOIN swaps_single_trade sst USING ( - tx_id, - event_index - ) -), -trade_data AS ( - SELECT - tx_id, - block_timestamp, - event_type, - event_contract AS event_contract_trade, - event_data AS event_data_trade, - event_data :side :: NUMBER AS swap_side, - event_data :token1Amount :: DOUBLE AS token_1_amount, - -- note some are decimal adjusted, some are not. identify by contract - event_data :token2Amount :: DOUBLE AS token_2_amount, - l.account_address AS swap_account, - _ingested_at, - _inserted_timestamp - FROM - swaps_single_trade sst - LEFT JOIN {{ ref('silver__contract_labels') }} - l USING (event_contract) - WHERE - event_type = 'Trade' -), -token_in_data AS ( - SELECT - sst.tx_id, - sst.block_timestamp, - sst.event_contract AS event_contract_token_in, - sst.event_data AS event_data_token_in, - sst.event_data :amount :: DOUBLE AS amount_token_in - FROM - trade_data t - LEFT JOIN swaps_single_trade sst - ON sst.tx_id = t.tx_id - AND t.swap_account = LOWER( - sst.event_data :from :: STRING - ) - WHERE - sst.event_type = 'TokensWithdrawn' -), -combo AS ( - SELECT - tod.tx_id, - tod.block_timestamp, - tod.block_height, - td.event_contract_trade AS swap_contract, - LOWER( - tod.trader_token_out - ) AS trader, - tod.token_amount_token_out AS token_out_amount, - tod.event_contract_token_out AS token_out_contract, - tid.amount_token_in AS token_in_amount, - tid.event_contract_token_in AS token_in_contract, - -- keep these next 3 columns bc i can derive fees from the difference in token_out_amount and token_[n]_amount where n = swap_side - td.swap_side, - td.token_1_amount, - td.token_2_amount, - tod._ingested_at, - tod._inserted_timestamp - FROM - token_out_data tod - LEFT JOIN trade_data td USING (tx_id) - LEFT JOIN token_in_data tid USING (tx_id) -) -SELECT - * -FROM - combo diff --git a/models/silver_cw/swaps/silver__swaps_single_trade.yml b/models/silver_cw/swaps/silver__swaps_single_trade.yml deleted file mode 100644 index 21d7666..0000000 --- a/models/silver_cw/swaps/silver__swaps_single_trade.yml +++ /dev/null @@ -1,129 +0,0 @@ -version: 2 - -models: - - name: silver__swaps_single_trade - description: |- - This table records single-pool asset swaps on the Flow blockchain. A single-pool asset swap is one where a liquidity pool exists for the trade such that no routing is required. (i.e. Swapping FUSD to USDT using `A.87f3f233f34b0733.FusdUsdtSwapPair`) - tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - tx_id - - trader - - swap_contract - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - tests: - - not_null - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_row_values_to_have_recent_data: - datepart: day - interval: 1 - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ - - - name: block_height - description: "{{ doc('block_height') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: swap_contract - description: "{{ doc('swap_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: trader - description: "{{ doc('trader') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_out_amount - description: "{{ doc('token_out_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_out_contract - description: "{{ doc('token_out_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: token_in_amount - description: "{{ doc('token_in_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_in_contract - description: "{{ doc('token_in_contract') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - STRING - - VARCHAR - - - name: swap_side - description: "{{ doc('swap_side') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - - name: token_1_amount - description: "{{ doc('token_1_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: token_2_amount - description: "{{ doc('token_2_amount') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - NUMBER - - FLOAT - - - name: _ingested_at - description: "{{ doc('_ingested_at') }}" - - - name: _inserted_timestamp - description: "{{ doc('_inserted_timestamp') }}" - tests: - - not_null - - dbt_expectations.expect_column_values_to_be_in_type_list: - column_type_list: - - TIMESTAMP_NTZ diff --git a/models/silver_cw/transfers/silver__bridge_blocto.sql b/models/silver_cw/transfers/silver__bridge_blocto.sql deleted file mode 100644 index 60957d7..0000000 --- a/models/silver_cw/transfers/silver__bridge_blocto.sql +++ /dev/null @@ -1,298 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = 'tx_id', - tags = ['bridge', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -teleport_events AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_contract AS teleport_contract_fee, - event_data :amount :: DOUBLE AS amount_fee, - event_data :type :: NUMBER AS teleport_direction, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - event_type = 'FeeCollected' - AND event_contract LIKE '%Teleport%' -), -teleports_in AS ( - SELECT - tx_id, - event_contract AS event_contract_teleport, - event_data :amount :: DOUBLE AS amount_teleport, - event_data :from AS from_teleport, - STRTOK_TO_ARRAY( - REPLACE(REPLACE(event_data :from :: STRING, '['), ']'), - ', ' - ) :: ARRAY AS from_teleport_array, - COALESCE( - event_data :hash, - event_data :txHash - ) :: STRING AS hash_teleport, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 1 - ) - AND event_index = 0 - AND event_contract LIKE '%Teleport%' - AND event_type IN ( - 'TokensTeleportedIn', - 'Unlocked' - ) - AND from_teleport :: STRING NOT LIKE '%{\"ArrayType%' -), -deposits AS ( - SELECT - tx_id, - event_contract, - event_index, - event_data :amount :: DOUBLE AS amount_deposits, - ROW_NUMBER() over ( - PARTITION BY tx_id - ORDER BY - amount_deposits DESC - ) AS rn, - event_data :to :: STRING AS to_deposits - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleports_in - ) - AND event_type = 'TokensDeposited' - ORDER BY - tx_id, - amount_deposits DESC -), -blocto_inbound AS ( - SELECT - t.tx_id, - f.block_timestamp, - f.block_height, - t.event_contract_teleport AS teleport_contract, - d.event_contract AS token_contract, - t.amount_teleport AS gross_amount, - f.amount_fee, - d.amount_deposits AS net_amount, - d.to_deposits AS flow_wallet_address, - f.teleport_direction, - 'blocto' AS bridge, - f._ingested_at, - f._inserted_timestamp - FROM - teleports_in t - LEFT JOIN deposits d USING (tx_id) - LEFT JOIN teleport_events f USING (tx_id) - WHERE - d.rn = 1 -), -teleports_out_withdraw_non_fiat AS ( - SELECT - tx_id, - event_contract, - event_data :amount :: DOUBLE AS amount_withdraw, - event_data :from :: STRING AS from_withdraw - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 0 - ) - AND event_index = 0 - AND event_contract != 'A.b19436aae4d94622.FiatToken' -), -teleports_out_withdraw_fiat AS ( - SELECT - tx_id, - event_contract, - event_data :amount :: DOUBLE AS amount_withdraw, - event_data :from :: STRING AS from_withdraw - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 0 - ) - AND event_index = 1 - AND event_contract = 'A.b19436aae4d94622.FiatToken' -), -teleports_out_withdraw AS ( - SELECT - tx_id, - event_contract, - amount_withdraw, - from_withdraw - FROM - teleports_out_withdraw_non_fiat - UNION - SELECT - tx_id, - event_contract, - amount_withdraw, - from_withdraw - FROM - teleports_out_withdraw_fiat -), -teleports_out AS ( - SELECT - tx_id, - event_contract AS event_contract_teleport, - event_data :amount :: DOUBLE AS amount_teleport, - event_data :to AS to_teleport, - STRTOK_TO_ARRAY( - REPLACE(REPLACE(event_data :to :: STRING, '['), ']'), - ', ' - ) :: ARRAY AS to_teleport_array - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - teleport_events - WHERE - teleport_direction = 0 - ) - AND event_type IN ( - 'TokensTeleportedOut', - 'Locked' - ) -), -blocto_outbound AS ( - SELECT - t.tx_id, - f.block_timestamp, - f.block_height, - t.event_contract_teleport AS teleport_contract, - w.event_contract AS token_contract, - w.amount_withdraw AS gross_amount, - f.amount_fee, - t.amount_teleport AS net_amount, - w.from_withdraw AS flow_wallet_address, - f.teleport_direction, - 'blocto' AS bridge, - f._ingested_at, - f._inserted_timestamp - FROM - teleports_out t - LEFT JOIN teleports_out_withdraw w USING (tx_id) - LEFT JOIN teleport_events f USING (tx_id) -), -tbl_union AS ( - SELECT - tx_id, - block_timestamp, - block_height, - teleport_contract, - token_contract, - gross_amount, - amount_fee, - net_amount, - flow_wallet_address, - teleport_direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - blocto_inbound - UNION - SELECT - tx_id, - block_timestamp, - block_height, - teleport_contract, - token_contract, - gross_amount, - amount_fee, - net_amount, - flow_wallet_address, - teleport_direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - blocto_outbound -), -tele_labels AS ( - SELECT - teleport_contract, - blockchain - FROM - {{ ref('seeds__blocto_teleport_labels') }} -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - t.teleport_contract, - token_contract, - gross_amount, - amount_fee, - net_amount, - flow_wallet_address, - CASE - WHEN teleport_direction = 0 THEN 'outbound' - ELSE 'inbound' - END AS teleport_direction, - l.blockchain, - bridge, - _ingested_at, - _inserted_timestamp - FROM - tbl_union t - LEFT JOIN tele_labels l USING (teleport_contract) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/transfers/silver__bridge_celer.sql b/models/silver_cw/transfers/silver__bridge_celer.sql deleted file mode 100644 index 50fa8c6..0000000 --- a/models/silver_cw/transfers/silver__bridge_celer.sql +++ /dev/null @@ -1,165 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = 'tx_id', - tags = ['bridge', 'scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -cbridge_txs AS ( - SELECT - tx_id, - block_timestamp, - block_height, - tx_succeeded, - event_index, - event_contract, - event_type, - event_data, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - event_contract = 'A.08dd120226ec2213.PegBridge' -), -inbound AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_contract AS bridge_contract, - REPLACE(REPLACE(event_data :token :: STRING, '.Vault'), '"') AS token_contract, - event_data :amount :: DOUBLE AS amount, - event_data :receiver :: STRING AS flow_wallet_address, - REPLACE(CONCAT('0x', event_data :depositor) :: STRING, '"') AS counterparty, - event_data :refChId :: NUMBER AS chain_id, - 'inbound' AS direction, - 'cbridge' AS bridge, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - cbridge_txs - ) - AND event_type = 'Mint' -), -outbound AS ( - SELECT - tx_id, - block_timestamp, - block_height, - event_contract AS bridge_contract, - REPLACE(REPLACE(event_data :token :: STRING, '.Vault'), '"') AS token_contract, - event_data :amount :: DOUBLE AS amount, - event_data :burner :: STRING AS flow_wallet_address, - REPLACE( - event_data :toAddr :: STRING, - '"' - ) AS counterparty, - event_data :toChain :: NUMBER AS chain_id, - 'outbound' AS direction, - 'cbridge' AS bridge, - _ingested_at, - _inserted_timestamp - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - cbridge_txs - ) - AND event_type = 'Burn' -), -tbl_union AS ( - SELECT - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - counterparty, - chain_id, - direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - inbound - UNION - SELECT - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - counterparty, - chain_id, - direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - outbound -), -chain_ids AS ( - SELECT - chain_id, - blockchain - FROM - {{ ref('seeds__celer_chain_ids') }} -), -FINAL AS ( - SELECT - tx_id, - block_timestamp, - block_height, - bridge_contract, - token_contract, - amount, - flow_wallet_address, - counterparty, - t.chain_id, - l.blockchain, - direction, - bridge, - _ingested_at, - _inserted_timestamp - FROM - tbl_union t - LEFT JOIN chain_ids l USING (chain_id) -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/transfers/silver__token_transfers.sql b/models/silver_cw/transfers/silver__token_transfers.sql deleted file mode 100644 index d384fae..0000000 --- a/models/silver_cw/transfers/silver__token_transfers.sql +++ /dev/null @@ -1,139 +0,0 @@ -{{ config( - materialized = 'incremental', - incremental_strategy = 'delete+insert', - cluster_by = ['_inserted_timestamp::date'], - unique_key = "CONCAT_WS('-', tx_id, sender, recipient, token_contract, amount)", - tags = ['scheduled', 'chainwalkers_scheduled'] -) }} - -WITH events AS ( - - SELECT - * - FROM - {{ ref('silver__events_final') }} - -- WHERE - -- event_data :: STRING != '{}' - -{% if is_incremental() %} -WHERE - _inserted_timestamp >= ( - SELECT - MAX(_inserted_timestamp) - FROM - {{ this }} - ) -{% endif %} -), -transfers AS ( - SELECT - _inserted_timestamp, - tx_id, - event_contract, - COUNT(event_type) AS event_count, - MAX( - event_index + 1 - ) AS max_index - FROM - events - WHERE - event_type IN ( - 'TokensDeposited', - 'TokensWithdrawn', - 'FeesDeducted' - ) - GROUP BY - _inserted_timestamp, - tx_id, - event_contract - HAVING - event_count = max_index - OR event_contract = 'A.b19436aae4d94622.FiatToken' -), -withdraws AS ( - SELECT - block_height, - _inserted_timestamp, - block_timestamp, - tx_id, - event_data :from :: STRING AS sender, - event_contract AS token_contract, - event_data :amount :: FLOAT AS amount, - tx_succeeded - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - transfers - ) - AND event_type = 'TokensWithdrawn' - GROUP BY - block_height, - _inserted_timestamp, - block_timestamp, - tx_id, - sender, - token_contract, - amount, - tx_succeeded -), -deposits AS ( - SELECT - tx_id, - _inserted_timestamp, - event_data :to :: STRING AS recipient, - event_contract AS token_contract, - event_data :amount :: FLOAT AS amount - FROM - events - WHERE - tx_id IN ( - SELECT - tx_id - FROM - transfers - ) - AND event_type = 'TokensDeposited' - GROUP BY - tx_id, - _inserted_timestamp, - recipient, - token_contract, - amount -), -FINAL AS ( - SELECT - block_height, - w._inserted_timestamp AS _inserted_timestamp, - block_timestamp, - w.tx_id, - sender, - recipient, - w.token_contract, - SUM(COALESCE(d.amount, w.amount)) AS amount, - tx_succeeded - FROM - withdraws w - LEFT JOIN deposits d - ON w.tx_id = d.tx_id - AND w.token_contract = d.token_contract - AND w.amount = d.amount - WHERE - sender IS NOT NULL - GROUP BY - block_height, - w._inserted_timestamp, - block_timestamp, - w.tx_id, - sender, - recipient, - w.token_contract, - tx_succeeded -) -SELECT - * -FROM - FINAL diff --git a/models/silver_cw/transfers/silver__token_transfers.yml b/models/silver_cw/transfers/silver__token_transfers.yml deleted file mode 100644 index a9a11a9..0000000 --- a/models/silver_cw/transfers/silver__token_transfers.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: 2 - -models: - - name: silver__token_transfers - description: |- - This table records all token transfers on the FLOW blockchain. - - columns: - - name: tx_id - description: "{{ doc('tx_id') }}" - - - name: block_timestamp - description: "{{ doc('block_timestamp') }}" - - - name: block_height - description: "{{ doc('block_height') }}" - - - name: sender - description: "{{ doc('sender') }}" - - - name: recipient - description: "{{ doc('recipient') }}" - - - name: token_contract - description: "{{ doc('token_contract') }}" - - - name: amount - description: "{{ doc('amount') }}" - - - name: tx_succeeded - description: "{{ doc('tx_succeeded') }}" diff --git a/models/sources.yml b/models/sources.yml index 08f2cfa..7b8ae35 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -8,12 +8,6 @@ sources: - name: flow_blocks - name: flow_txs - - name: crosschain_silver - database: crosschain - schema: silver - tables: - - name: labels_combined - - name: bronze_streamline database: streamline schema: | @@ -124,14 +118,28 @@ sources: - name: TRANSACTION_RESULTS_MAINNET_20 - name: TRANSACTION_RESULTS_MAINNET_21 - name: TRANSACTION_RESULTS_MAINNET_22 - + - name: testnet_blocks + - name: testnet_collections + - name: testnet_transactions + - name: testnet_transaction_results - name: crosschain_silver database: crosschain schema: silver tables: - - name: hourly_prices_coin_gecko - - name: hourly_prices_coin_market_cap + - name: labels_combined + + - name: silver_crosschain + database: "{{ 'crosschain' if target.database == 'FLOW' else 'crosschain_dev' }}" + schema: silver + tables: + - name: complete_token_asset_metadata + - name: complete_token_prices + - name: complete_provider_asset_metadata + - name: complete_provider_prices + - name: complete_native_asset_metadata + - name: complete_native_prices - name: number_sequence + - name: labels_combined - name: bronze_api database: | diff --git a/python/dbt_slack_notification.py b/python/dbt_slack_notification.py new file mode 100644 index 0000000..ca57065 --- /dev/null +++ b/python/dbt_slack_notification.py @@ -0,0 +1,67 @@ +import requests +import json +import os + +def create_message(**kwargs): + messageBody = { + "text": f"Ad hoc :dbt: job run completed for :{os.environ.get('DATABASE').split('_DEV')[0]}: {os.environ.get('DATABASE')}", + "attachments": [ + { + "color": kwargs["color"], + "fields": [ + { + "title": "Environment", + "value": kwargs["environment"], + "short": True + }, + { + "title": "Warehouse", + "value": kwargs["warehouse"], + "short": True + }, + { + "title": "Elapsed Time", + "value": kwargs["elapsed_time"], + "short": True + }, + { + "title": "Run status", + "value": f"{':check:' if kwargs['dbt_run_status'] == 'success' else ':x:'} {kwargs['dbt_run_status'].capitalize()}", + "short": True + }, + { + "title": "dbt Command", + "value": kwargs["dbt_command"], + "short": False + } + ] + } + ] + } + + return messageBody + + +def send_alert(webhook_url, data): + """Sends a message to a slack channel""" + + send_message = create_message( + **data, + color="#008080" if data["dbt_run_status"] == "success" else "#FF0000" + ) + + x = requests.post(webhook_url, json=send_message) + + +if __name__ == '__main__': + data = { + "environment": os.environ.get("ENVIRONMENT"), + "database": os.environ.get("DATABASE"), + "warehouse": os.environ.get("WAREHOUSE"), + "dbt_command": os.environ.get("DBT_COMMAND"), + "elapsed_time": os.environ.get("ELAPSED_TIME"), + "dbt_run_status": os.environ.get("DBT_RUN_STATUS") + } + + webhook_url = os.environ.get("SLACK_WEBHOOK_URL") + send_alert(webhook_url, data) diff --git a/tests/tests__moment_metadata_allday.sql b/tests/tests__moment_metadata_allday.sql index c0b08c8..2d7bba0 100644 --- a/tests/tests__moment_metadata_allday.sql +++ b/tests/tests__moment_metadata_allday.sql @@ -2,7 +2,7 @@ WITH mint_events AS ( SELECT MAX(block_timestamp) :: DATE AS last_mint_date FROM - {{ ref('silver__nft_moments') }} + {{ ref('silver__nft_moments_s') }} WHERE event_contract = 'A.e4cf4bdc1751c65d.AllDay' AND event_type = 'MomentNFTMinted' diff --git a/tests/tests__moment_metadata_topshot.sql b/tests/tests__moment_metadata_topshot.sql index c81cf0b..11a5532 100644 --- a/tests/tests__moment_metadata_topshot.sql +++ b/tests/tests__moment_metadata_topshot.sql @@ -2,7 +2,7 @@ WITH mint_events AS ( SELECT MAX(block_timestamp) :: DATE AS last_mint_date FROM - {{ ref('silver__nft_moments') }} + {{ ref('silver__nft_moments_s') }} WHERE event_contract = 'A.0b2a3299cc857e29.TopShot' AND event_type = 'MomentMinted'