diff --git a/.env.sample b/.env.sample index c0306a1..5e72b5d 100644 --- a/.env.sample +++ b/.env.sample @@ -1,8 +1,8 @@ -SF_ACCOUNT=vna27887.us-east-1 -SF_USERNAME= -SF_PASSWORD= -SF_REGION=us-east-1 -SF_DATABASE=FLOW_COMMUNITY_DEV -SF_WAREHOUSE=FLIPSIDE_COMMUNITY_CURATOR -SF_ROLE=FLIPSIDE_COMMUNITY_CURATOR -SF_SCHEMA=SILVER \ No newline at end of file +ACCOUNT=vna27887.us-east-1 +USER= +PASSWORD= +REGION=us-east-1 +DATABASE=FLOW_DEV +WAREHOUSE=DATA_SCIENCE +ROLE=INTERNAL_DEV +SCHEMA=SILVER diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index 51f2d88..4562a1b 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -22,6 +22,54 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_docs_updates.yml@main - secrets: inherit + run_dbt_jobs: + runs-on: ubuntu-latest + environment: + name: workflow_prod + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: refresh ddl for datashare + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + + - name: checkout docs branch + run: | + git checkout -B docs origin/main + + - name: generate dbt docs + run: dbt docs generate -t prod + + - name: move files to docs directory + run: | + mkdir -p ./docs + cp target/{catalog.json,manifest.json,index.html} docs/ + + - name: clean up target directory + run: dbt clean + + - name: check for changes + run: git status + + - name: stage changed files + run: git add . + + - name: commit changed files + run: | + git config user.email "abc@xyz" + git config user.name "github-actions" + git commit -am "Auto-update docs" + - name: push changes to docs + run: | + git push -f --set-upstream origin docs diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index ed413ac..b0f1f9e 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -58,7 +58,7 @@ jobs: - name: install dependencies run: | - pip3 install dbt-snowflake==${{ vars.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs diff --git a/.github/workflows/dbt_run_daily_dev_refresh.yml b/.github/workflows/dbt_run_daily_dev_refresh.yml index cb65f37..d68a772 100644 --- a/.github/workflows/dbt_run_daily_dev_refresh.yml +++ b/.github/workflows/dbt_run_daily_dev_refresh.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: schedule: # Runs 0800 daily (see https://crontab.guru) - - cron: '0 8 * * *' + - cron: "0 8 * * *" env: USE_VARS: "${{ vars.USE_VARS }}" @@ -24,11 +24,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run-operation run_sp_create_prod_clone - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run-operation run_sp_create_prod_clone + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_deployment.yml b/.github/workflows/dbt_run_deployment.yml index 40da092..79325cc 100644 --- a/.github/workflows/dbt_run_deployment.yml +++ b/.github/workflows/dbt_run_deployment.yml @@ -7,9 +7,9 @@ on: - "main" inputs: warehouse: - type: choice + type: choice description: Snowflake warehouse - required: true + required: true options: - DBT - DBT_CLOUD @@ -17,9 +17,9 @@ on: default: DBT dbt_command: type: string - description: 'DBT Run Command' + description: "DBT Run Command" required: true - + env: USE_VARS: "${{ vars.USE_VARS }}" DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" @@ -33,16 +33,40 @@ env: WAREHOUSE: "${{ inputs.WAREHOUSE }}" SCHEMA: "${{ vars.SCHEMA }}" - concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_deployment_template.yml@main - with: - dbt_command: | - ${{ inputs.dbt_command }} - warehouse: ${{ inputs.WAREHOUSE }} - environment: workflow_prod - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + ${{ inputs.dbt_command }} + + - name: Run datashare model + run: | + cnt=$(dbt ls -m fsc_utils.datashare._datashare___create_gold | wc -l ); if [ $cnt -eq 1 ]; then dbt run -m fsc_utils.datashare._datashare___create_gold; fi; + dbt run-operation run_query --args "{sql: call admin.datashare.sp_grant_share_permissions('${{ env.DATABASE }}')}" + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target + \ No newline at end of file diff --git a/.github/workflows/dbt_run_history.yml b/.github/workflows/dbt_run_history.yml index 3b49221..61de275 100644 --- a/.github/workflows/dbt_run_history.yml +++ b/.github/workflows/dbt_run_history.yml @@ -4,8 +4,10 @@ run-name: dbt_run_history on: workflow_dispatch: schedule: - # Runs daily at 00:00 UTC (see https://crontab.guru) - - cron: '0 0 * * *' + # Runs every 2 hours + # - cron: "0 */2 * * *" + # Runs every hour + - cron: "0 * * * *" env: USE_VARS: "${{ vars.USE_VARS }}" @@ -24,11 +26,30 @@ concurrency: group: ${{ github.workflow }} jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run -s 2+models/silver/streamline/core/history/blocks 2+models/silver/streamline/core/history/collections 2+models/silver/streamline/core/history/transactions 2+models/silver/streamline/core/history/transaction_results --vars '{"STREAMLINE_INVOKE_STREAMS": True, "sql_limit": ${{ vars.SQL_LIMIT_HISTORY }}, "producer_batch_size": ${{ vars.PRODUCER_BATCH_SIZE_HISTORY }}, "worker_batch_size": ${{ vars.WORKER_BATCH_SIZE_HISTORY }}, "batch_call_limit": ${{ vars.BATCH_CALL_LIMIT_HISTORY }}}' - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: | + dbt run -s \ + 2+streamline__get_transaction_results_history_mainnet_14 \ + 2+streamline__get_transaction_results_history_mainnet_15 \ + 2+streamline__get_transaction_results_history_mainnet_16 \ + 2+streamline__get_batch_transaction_results_history_mainnet_17 \ + 2+streamline__get_batch_transaction_results_history_mainnet_18 \ + 2+streamline__get_batch_transaction_results_history_mainnet_19 \ + 2+streamline__get_batch_transaction_results_history_mainnet_22 \ + --vars '{"STREAMLINE_INVOKE_STREAMS": True}' diff --git a/.github/workflows/dbt_run_moments_metadata.yml b/.github/workflows/dbt_run_moments_metadata.yml index 2a4a726..2ea8d97 100644 --- a/.github/workflows/dbt_run_moments_metadata.yml +++ b/.github/workflows/dbt_run_moments_metadata.yml @@ -22,15 +22,35 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run -m silver__allday_moments_metadata_needed_s.sql; - dbt run-operation get_allday_metadata; - dbt run -m silver__nft_allday_metadata_s; - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run -m silver__allday_moments_metadata_needed_s.sql; + dbt run-operation get_allday_metadata; + dbt run -m silver__nft_allday_metadata_s; + dbt run -s tag:topshot; + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_scheduled.yml b/.github/workflows/dbt_run_scheduled.yml index 7be7dd3..f09da4a 100644 --- a/.github/workflows/dbt_run_scheduled.yml +++ b/.github/workflows/dbt_run_scheduled.yml @@ -24,15 +24,34 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; - dbt seed; - dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; + dbt seed; + dbt run -s tag:scheduled_core tag:streamline_complete "flow_models,models/gold" --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/.github/workflows/dbt_run_scheduled_non_core.yml b/.github/workflows/dbt_run_scheduled_non_core.yml index 0a01856..7ba4681 100644 --- a/.github/workflows/dbt_run_scheduled_non_core.yml +++ b/.github/workflows/dbt_run_scheduled_non_core.yml @@ -4,7 +4,7 @@ run-name: dbt_run_scheduled on: workflow_dispatch: schedule: - # Every hour at minute 9 (see https://crontab.guru) + # Every hour at minute 9 (see https://crontab.guru) - cron: "9 * * * *" env: @@ -22,15 +22,35 @@ env: concurrency: group: ${{ github.workflow }} - + jobs: - called_workflow_template: - uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_run_template.yml@main - with: - dbt_command: > - dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; - dbt seed; - dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' - environment: workflow_prod - warehouse: ${{ vars.WAREHOUSE }} - secrets: inherit + dbt: + runs-on: ubuntu-latest + environment: + name: workflow_prod + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + + - name: install dependencies + run: | + pip install -r requirements.txt + dbt deps + + - name: Run DBT Jobs + run: > + dbt run-operation stage_external_sources --vars "ext_full_refresh: true"; + dbt seed; + dbt run -s tag:scheduled_non_core --vars '{"STREAMLINE_START_BLOCK": ${{ vars.STREAMLINE_START_BLOCK }}}' + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target + \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml index c531dcc..23307fe 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_test.yml @@ -5,7 +5,7 @@ on: workflow_dispatch: schedule: # Run at 1600 UTC daily (see https://crontab.guru) - - cron: '0 16 * * *' + - cron: "0 16 * * *" env: SLACK_WEBHOOK_URL: "${{ secrets.SLACK_WEBHOOK_URL }}" @@ -27,19 +27,20 @@ concurrency: jobs: run_dbt_jobs: runs-on: ubuntu-latest - environment: + environment: name: workflow_prod steps: - uses: actions/checkout@v3 - - uses: actions/setup-python@v1 + - uses: actions/setup-python@v4 with: - python-version: "3.7.x" + python-version: "3.10" + cache: "pip" - name: install dependencies run: | - pip3 install dbt-snowflake==${{ env.DBT_VERSION }} cli_passthrough requests click + pip install -r requirements.txt dbt deps - name: Run DBT Jobs @@ -50,3 +51,11 @@ jobs: - name: Log test results run: | python python/dbt_test_alert.py + + - name: Store logs + uses: actions/upload-artifact@v3 + with: + name: dbt-logs + path: | + logs + target diff --git a/Makefile b/Makefile index 5aa4313..07a366e 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ SHELL := /bin/bash # set default target DBT_TARGET ?= dev AWS_LAMBDA_ROLE ?= aws_lambda_flow_api_dev +INVOKE_STREAMS ?= True dbt-console: docker-compose run dbt_console @@ -25,7 +26,7 @@ udfs: udf_2: dbt run-operation create_udf_bulk_grpc_us_east_2 \ - --vars '{"UPDATE_UDFS_AND_SPS":False}' \ + --vars '{"UPDATE_UDFS_AND_SPS":True}' \ --profile flow \ --target $(DBT_TARGET) \ --profiles-dir ~/.dbt/ @@ -83,15 +84,38 @@ tx_history: tx_results_history: dbt run \ --vars '{"STREAMLINE_INVOKE_STREAMS":True, "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ - -m 1+models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet22.sql \ + -m 1+models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql \ --profile flow \ --target $(DBT_TARGET) \ --profiles-dir ~/.dbt +tx_results_batch_history: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS": $(INVOKE_STREAMS), "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql \ + --profile flow \ + --target $(DBT_TARGET) \ + --profiles-dir ~/.dbt + +tx_batch_history: + dbt run \ + --vars '{"STREAMLINE_INVOKE_STREAMS": $(INVOKE_STREAMS), "STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + -m 1+models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql \ + --profile flow \ + --target dev \ + --profiles-dir ~/.dbt + lq_overloads: dbt run \ -s models/deploy/core/ \ --profile flow \ --target $(DBT_TARGET) \ --profiles-dir ~/.dbt \ - --vars '{"UPDATE_EPHEMERAL_UDFS":True}' \ No newline at end of file + --vars '{"UPDATE_EPHEMERAL_UDFS":True}' + +bronze: + dbt run \ + -s bronze__streamline_transaction_results_history \ + --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES": True}' \ + --profiles-dir ~/.dbt \ + --target $(DBT_TARGET) diff --git a/data/seeds__nflad_playoff_rookies.csv b/data/seeds__nflad_playoff_rookies.csv new file mode 100644 index 0000000..04404e2 --- /dev/null +++ b/data/seeds__nflad_playoff_rookies.csv @@ -0,0 +1,62 @@ +wallet,id_0,id_1,id_2,id_3,id_4,id_5,id_6,id_7,id_8,id_9 +0x8b6a462c281aadfb,cf9e61a0-8828-4b1b-ba95-89087cd5509c,72fb8f90-d403-4d03-aa12-16f7b3e82747,5f5beccf-1d7f-420e-8a44-73e98f78c6c4,fcd7f916-e244-4262-8d8d-809094b138d8,ee637cba-8344-43a6-be72-5a3b0a834d75,3c961ef0-2afa-49dc-9c18-72b833bd18fd,b1bbd08a-d2c5-410b-9f72-7ba37e13736f,2e3f4d88-4607-4e96-8595-ad0552c4062d,e8d7b493-ceaf-434d-90c5-dbbe88a1602a,70b41a37-5cec-4b03-a6bd-ad93963b333c +0x4788e734f8491a5b,beafbcb1-5c17-464f-b0c5-943987319d51,c3172f7a-4503-49b0-a503-0f7c2700ffb5,b66f2318-40fb-47a5-91de-2aa4054f5594,77c7c053-e6a6-4e94-ac8b-18c058c25988,e5403ee6-5a76-4782-9e98-a39f35854248,6c105de7-fafc-4a0e-b2ac-11f4a0f00955,7947f3fd-943c-401f-ade9-cf283254e941,f6dda0bf-65c2-46c4-ae5d-0d8de4837212,93128fbe-cefe-4f88-86cc-e1bda4c684f5,232051fe-227f-4ce1-92ea-7d3058e145c7 +0x33fb80905088b044,094dbf5f-8ce1-4b86-afd7-032e4472d98a,6b89f1b3-5cb2-44d5-abf8-a2bb8e0de062,b5f505f6-8b6f-4519-ac34-be85a6182fb7,eb0c4659-ab83-4710-89b2-41be8b23995a,d487eb5a-b43e-433d-ad4a-2098cdceb5ef,cc05f26b-2616-49d3-9237-ceff139a1b9d,8353e889-f80a-4434-9de3-2df1c555c3d3,4db251ec-6b6e-47c4-a6d4-e96b4722dfd4,073f8eec-3e0b-425e-a02f-fb755311bc25,58e76b8e-9e01-4c19-b833-08d134867ac2 +0x19424b8a5400fcb3,8b27c651-7cc2-4767-8d13-901bcf788c2f,19fb3f46-923a-4436-ad8a-b1fd460f299e,6062749d-e54f-4d8e-b4a7-5f3be4cecdc5,0022caa9-a31a-4771-81a8-56b529d1c872,fa6ff877-ea62-4134-81c3-11d8f49e3203,aa4699ad-dee7-447f-a274-0df602874848,09dc9184-6529-401c-bfd8-5f0567155e64,d2651fda-84af-42e4-9c51-f44fdf01f16c,7671965d-dd47-48c9-a7b1-ec752a9d5a34,8d945c5a-bcc0-44b3-b58b-159b9574b78b +0x4ab2327b5e1f3ca1,09a37fbc-1866-4ef0-81a7-062fa04174ea,49f2d1ca-3ba3-4215-b534-39f42de32c38,3162f992-01fe-497a-9ef5-ca8d4935e756,71ea001d-4fed-4e65-9c4d-0ad6ea45e4ac,92fdaa07-fe08-4a6a-975c-ed8ba0059a1c,6884a014-285a-4387-a12b-affba1b2cd5c,f9ea2000-2d0b-49a4-98a3-098af1646bc9,c3575e58-be9e-4088-9ef5-ca7ab4e69e69,52d075ee-c9da-4e2c-aeef-db9b97db56a1,104bf63d-eb4b-45f7-9bdc-3661ca8d335b +0x64e58ffeea9b2a09,32bd33e4-2bf0-4568-a031-2b5d841e19cf,139abca2-0998-4308-a271-0678fa27e9ba,991f2fb1-aa52-4a68-a440-47844b885815,a3911c9e-f3c5-4fb9-9dff-15198c5d17ba,f4ec07ce-5ab7-4490-90c3-b660a099bf7a,55cd11dc-dd44-4deb-b854-a00ef696ca7d,e2540f19-4d3c-4d7e-b43e-e90ec23b6f83,bcd8f5ec-7342-40b1-bd6c-d12470d4ab0f,0277d288-c05b-4901-bbc6-e26a62f81410,1fd7161f-f2ea-4afc-9b69-7da8475fd1a6 +0x7d8f08954f51d0a7,a1861aef-96ad-4882-ae06-cbac00afca4d,95c7d9c9-8482-40b9-8f7b-fcfcdd78c935,3e0b3d41-7c89-4502-adeb-847dd1601d5a,eec73df0-fdf7-4c65-b3a1-a3ebcafc9d09,540ec60d-8d9c-4628-b68e-77c02f2b339d,f27da4aa-9496-4c19-9905-4ddc26d572be,20df9d09-460e-4598-bd07-902e3012bfeb,9effa8c2-1e5d-415f-a94c-d350d72e452d,919951dd-0dd1-43b1-9fcf-12694024ceb8,8e42eddf-5249-41f3-a96d-c3bb788f2f13 +0x9a9e605a98ba8efd,ae028f8b-892c-4858-a5d8-9208822e1ffb,0152b6c1-bdec-4a6b-82c6-e9366776b1ca,176cd541-8ba1-4443-aa83-774bad777bf3,2d702a47-cdb0-43ff-8563-77662a591d6e,92d784e0-20fc-480f-a622-a13c9be95bf8,0a4caa52-0aaa-4f11-a2c9-c53e16a3607d,c06bd6cc-a2a6-4ec8-9d90-3a666597ac41,5255168d-a6fe-4a2b-aae2-f8a119239532,2a5626c0-5f14-4151-a848-a03904394f0b,f96fa495-d83d-443f-a1f7-aeb2ff5fded1 +0xf897de380cb49b64,436105c9-a62f-4b42-bbf8-014fffffdb78,7c43ae0e-de00-4bdd-9542-246ab44040f3,a24fd240-f84c-44e1-a360-386b767ef39a,fcdb7ee1-0882-4355-aeec-ce0136d4cea5,835bce80-a276-494c-ab1e-d53fcfd1f517,cba47569-d5ee-44d8-8c6b-de96415b3290,da132b75-18fd-4f37-bc2b-4a1481325903,ea45a8d1-343f-4dd6-a6e8-ce87e799033a,cd17496c-9ca2-46aa-983a-548f110727ee,a94b1665-328c-433a-babf-e59ac0cf3cc1 +0xa308f76a2b2a7483,089f3c79-ecd2-44b1-97b4-b313e96da57a,e9f58623-f6be-460b-b59d-c2a75ec06ef3,7cb98d53-efef-4b1b-bac5-b2ac57a9e411,0150da18-21ff-4e21-8e45-33aed9425c5e,7dc36cbf-5bd7-4770-a8d2-12e2ed8c273b,b94c169a-c21c-425d-abc5-0cb1a55f7e69,81a0197d-3730-4c1b-bec9-af36eacd9795,9296492a-adf5-4196-bdf6-ec87eddcd634,1e8b6025-4790-43ae-a4c2-77618451999c,b3d75c09-bf7f-4709-aedb-5a05c6f101f6 +0xd96bb5897cf0a54e,86b73f27-a389-4df9-98de-21df674830b3,b01ba84d-e342-4f3b-9bc3-514b519c08c4,e4b536ed-f83b-4f79-a11a-6e03bb683c88,3c6d158e-f38c-4392-8d22-96d1f9c91761,8605d69f-74af-4024-8265-9c93b17df357,d93cf671-e9b8-463d-83b1-bf9e7d9acc0e,75206bb0-3580-4907-807c-ef26d9ed0fdc,cde5666d-47be-486b-90b7-2e9fc45ac717,c7c9fca4-33b7-4f13-9ae1-1f0323dee0e3,edeff610-00f9-4d30-8c6a-3173c34dbc7f +0x235a98cf2b2a2054,e6c7a2d5-2407-496d-9985-934f470ec1ce,c12ba8cc-223d-4a88-82c3-57b4dc068a16,39587363-2181-43e3-9e4c-75183aa4f54c,32386e96-329c-4a80-a151-9b5e48442f1a,aa31e6c0-c613-4569-a25c-dd52a17a585c,d121d809-18af-4888-87e9-31abd04d2d97,427aca16-a5f5-4680-bf77-9e208647e94b,f67c71d8-290a-4de2-a2cb-ce422cd3703e,fa6ff877-ea62-4134-81c3-11d8f49e3203,9341a65e-8915-4f91-a902-b3c1aa558aeb +0xc8bbd22a88b8b467,4a82efea-2c47-4f65-85a1-62267f7d737f,ddd388bb-c29c-44c6-aeb3-cad2d7681100,3a5e6894-497d-4d5c-b65f-0242aa0561fd,0600f5c5-3405-435a-9311-dfc2fe765d7d,611eb69a-c701-427f-8694-1ea298d9eae3,c66c885c-2994-47a1-b1a9-774e5c103bb4,d4b6567b-1cfa-4884-a92c-bee5ab006a53,53ff7a35-5544-45d6-8ccc-b3ee7736578c,2762d66c-b3dc-4112-8948-6c270efa47cf,c86f827f-5803-46a8-8590-6452e6c4f32b +0x7541bafd155b683e,688c10d1-9ed1-4d07-93e3-599d43d438f3,e996e9bc-17b2-445d-a5aa-702971d1bba2,9e47fa6d-575a-4c94-8c17-ba217d0255df,81fdbd9b-6e57-4074-86a3-3304b0627c49,cfcd338e-7f8e-4b22-aed7-65ffb21129cc,7491362f-df19-4992-95e6-545473b4757e,af5498f9-a37d-492d-af20-f90d71707849,2e8c8fe9-b280-4b22-9aa7-cb2667867549,8bdf67b2-e1c1-426a-98ad-7a9c30b2d05f,5faaa80d-16b0-4880-be39-b5147a9f0910 +0x8668bd1d417da9da,02c16fb7-7722-4e3e-90b2-f30c24f0eaac,cdbc4b87-bef2-454c-adb0-e2babc3eff64,51302de3-a592-4cca-9354-30743985a2e5,1f3fcb7f-af77-4978-8e60-80da10ed88aa,e4f1fd45-6b5f-4e37-89bb-d2b9980cf615,96f8f68f-6524-46e9-91c1-73f6b7994668,6cf2ec97-df1f-457a-abdb-7dda44e81aec,5b23deda-f34e-44ad-8c39-3f9ecb61b1fb,f2bfe603-def5-471f-b3c2-36dde76c089d,95f07929-b9d2-46a8-afdf-5f50c12fa14c +0x4e2c842cb5d154ee,cc9bbe8d-085c-42cc-b717-b238c70ec13d,35ac296a-5846-4d91-b2c8-b6b01bc97ce4,10d4b58b-ce41-4a13-bb9b-1d2713d62bcb,bfeb91bd-4913-48b6-8cb9-8db10808572b,be6d1df5-e43a-42da-909d-39c85a817778,1dd73a62-d11f-4532-9286-c0742926005f,56346e37-fa10-4649-89f3-916066470fb8,1e2b8625-c593-4ec8-a988-45f6d0782af9,ea63d5b9-dab0-4221-9cf4-0c2366e3fe02,58758f6c-bcb0-4f25-b7a1-273a5a0412ad +0x4f768fbfc061a5b6,7a79e8c8-a7ed-49ea-b1c8-3f544df1920c,9bc73cbf-bb02-450b-8ca9-92ceab9cadd9,835bf0e3-fa41-477c-99af-45626f4faedc,1268a1ba-31c1-48bd-930c-646554e2b32a,9848cb20-7c41-44c4-8724-d42e3f4ca7a5,458bcc88-dafa-4fb0-b1e9-18c296d351b8,fe439602-2f64-4195-81ed-010a5e76caa0,ac750501-c8ec-4a1e-b8dd-90e2117d41ad,17283ea0-cd52-49a6-a28f-b4e6e01706ea,3269e99d-0109-4519-98cb-ec42d339704a +0xddece960930dea52,bb102abc-cbf6-4475-b00e-cbe95a5a0ac2,8c2bf323-800f-4d20-b415-9ebde43a4a82,a63cebee-5ae3-4ca6-a559-a2c080cfa91b,55bbeabf-389e-41c1-b7d7-9b328cb92a8e,0ea25e64-aaf7-4d1e-86f5-c90c2bd0eb86,b5561594-f821-42ba-96f2-882f83a00697,fd0bfe25-a1d5-418f-8d3d-5159b010d63e,380eb84e-33a3-4c7b-857d-7b14738f61c3,f60b66df-3aa2-4b9f-9979-7f38f370c38a,172ab1c0-793a-4c33-ae71-9aea08cc8ebf +0x3879c4294f92d6e4,90aa873a-0793-47f9-a9e3-c29448bbfc33,e31867ff-cb81-4d41-af0f-892336f30d38,22a2ce31-d14f-44a6-8b73-796b0a557da4,5f5f02db-f863-4955-81f6-2fe01b5c13b4,b8178f6a-70ee-4a32-90f4-8ae092db499d,4ba91d4b-3e10-4dfa-9b89-acd204c949ee,0c09f57a-bfff-4abc-bb0a-fe9acd594dc1,14ac34a6-af04-482e-b4b9-bc16c483a044,54d2b830-fadd-42b4-82e1-2e2c58e471a3,ba2d2b47-ab18-403a-a5e8-2843aaf9783f +0xbee776009d396a51,a84104ab-eb76-4a66-bf90-a1fbf7595fbb,5a0e805d-c725-4241-8a3f-5497f239fba1,0474cafd-9839-411d-a133-6850cfaa7dcb,5f61b4ea-872e-410a-9072-afcaf007a08a,b747d388-35be-4155-b7a6-39a85b2e5dc8,d68d02a9-dd02-459e-9228-1101dfb970cf,fc91ea2e-c0ea-4146-80db-56f04d17292a,0f3bddc3-c46a-4d78-b945-809bb2f24bcd,8cff51cc-08da-4f9b-b8d1-06338767433e,7752967c-61f2-4d07-bdc7-3932741b019e +0x2252abfb10fe4ee3,9101db42-832d-42fa-8e5f-1fd7df0027a0,1d017db6-06ae-4b99-a166-0719b97c0992,c23b0cd4-0883-468c-9fa9-f1577c784d46,c202bb9d-8d56-4aaf-9404-22e7e9a2eb18,ba3b02df-0149-4838-8a33-c614ac5f8be5,278b13f0-b386-4924-849d-56123d910036,00e47aca-2797-445f-ae16-44cab56dd5bf,9510f74c-226d-46b1-bb85-e4e435920979,a0b12f7b-85fd-46c0-8c6a-e2084d94f1b2,048e5ebd-7a70-49ee-8405-928e5b1377b0 +0xb0c3cea14d23d224,e64ccdb7-73d2-4b58-ac6a-7b6ce2b78b6a,149c531b-e96c-495a-beb8-d733a759f2d0,9198d591-cc23-4a5d-9e8c-b410d4f9f436,f2a1bf49-a4b6-4d15-a116-b10f8b8de847,a0f26ec6-6dc1-4d6d-ba8b-85ddaf8e4763,a61999da-d73f-4e3f-9ad4-78e50d909cb1,c8b6bd0c-b75e-4f40-b07e-ad9b8dd2482b,437e8a83-03f2-4b98-8b83-e1d2d419daaa,e71edae3-2c93-4e9d-ae60-9ee9d0290eea,c8509e7a-be5c-414b-8067-49ac0ab7e95f +0x66f4336d1418b0f4,2aa71016-310a-45c1-8852-f5f65cace823,687968c6-1bea-49e7-a5ba-5f901dbec96a,910acbfa-89fc-4d1b-8369-1904a438c2a0,39d2cba9-8521-4222-915e-c4bf9fe19fcb,8e41069b-027d-4cc4-ba34-11206034ae88,4642560a-f268-486e-b2b0-acd342ea7025,8f7ea322-b602-458b-8e12-ecce9bff5e12,9f557432-d4bd-4773-83bc-60b565b2a5a9,40c65226-5dba-45c9-9207-851502be4a72,946dd722-eba7-4cc1-ad20-b21ee655debd +0xed67ca1072a41242,cabc57d2-c06a-41fd-ab15-02040c69ded9,10fef710-786b-4033-87c3-25387a790db9,a450b9de-1b83-48ba-a3ab-ddca210af644,e1d9ade3-3a5e-43ce-8e3d-e1b8b0618f91,bfa5c8cc-6835-4c3b-aebc-c8791178716d,5a20a591-9b3c-46a5-9ccb-56ae0844673d,85ae9eb1-8891-4422-949b-8e4c71e1c99c,edc0df69-ad03-407d-9103-994662f54667,a0eeb7f4-902b-4559-9de9-b131c530510d,9fe122ce-9b37-43c1-b6fa-b4f4e9f89884 +0x6cbd90039bfc4354,b83bb19f-6f57-4a4e-9311-287f713ab4f9,8ac76431-9bb6-49f4-88f3-2e2788a4d8d4,5754aed0-c6bb-4c0c-812b-86eed1a8f107,da732fbb-5bbc-4ced-af63-48bfb816dd63,3c1ccb8c-167d-41b9-a3a7-6c2ce6c9df5c,897099fe-0ce8-4ee4-a5ec-3494268b6f19,7ba73359-8977-43e8-9ca7-936c495fd540,d3313951-206e-4793-8062-4063a5c55e7a,f69a10db-abf7-44ba-8e52-2f25058153d3,57acd488-5967-4f7b-9c3e-70d4570b30d0 +0x760e5312752dc4ae,a5786741-da78-4ea6-babe-36c75f8b1dde,fa784305-c649-498c-8e9a-ffdb1047bee1,7df6bd54-dccf-4c5a-a546-39b4f311cdd8,dac6d5e8-9056-4f4a-9a8c-624a293de877,326eebff-fb4f-40b4-9706-4294c9e59073,d2a30e00-54b5-4988-a63f-a502fd7fde9c,56f037a3-6091-46f9-86c5-125963d5e0ed,e7516f51-0cf4-433d-8b77-7857c1df9e2e,19390d69-b7b0-438a-a3fe-8a1c8d883f85,698774df-ca1e-4028-a272-92675b09b66e +0x3376758a5e4ebe6c,c11e8b6d-8d9b-4ba5-b4da-48c25e3fac7e,83b659d7-b300-4bc9-9a0b-7132d0125265,a42bbe06-d507-4aac-a450-4efdbd40bf00,6268ed69-5d12-48eb-9477-8f090292ab01,e58157c6-55ac-48e3-bfaf-f39450b3cf3c,d1caad86-53f3-4de5-871f-870a5c4b1c38,f905e305-2c2e-49fa-9e0f-7eb859651068,674ceb23-cb65-4226-b4b7-95f056ba84fc,57b8b519-1d10-4414-ba86-366f07db687d,81c7d07a-678d-4853-8064-c3f76122f01b +0x9f74f9896e1df5f1,7e2361ae-996e-4ddc-90a3-dd69c61d23c3,d6a8fc94-b59e-4d6f-8a40-5fbd7b5caafa,6145d337-3be4-4c17-8e71-45e945ce65b8,239cc06f-b985-475f-beca-bba8a823efb2,d15cd2cd-e473-44f3-8f16-6e4e8edf70b3,d4e9a1ce-2fa9-4907-9a36-d6d0e9ee48ec,f36a69d0-a758-4e7c-8a14-e8b2179d7502,7b74058e-82c5-41cb-8e65-7ad9aad20fe2,1bcd3a7c-fbae-4fdd-b197-a1bf81620cfe,0f114755-f908-4c44-b8ec-3ca6b004a7e4 +0xf90cc1a4c029c83c,959aef3a-8583-4b73-812f-032a68351872,8fd0541f-bd6b-4215-87d3-1deab5f78668,38f7d1dd-c215-4185-9dad-4c930670e6a4,56ca08d4-aa45-4a41-b2f7-6fb3cf06eee1,e3dd2608-0eda-4b5a-a482-07c6d8b64ac5,5fb154ed-4585-492b-bd48-2115d7552834,a04e37f4-462a-45cf-a902-9c50c6b65b59,f6464ba0-b9dc-4660-be66-31837c038256,c2d25f8f-c1c9-4e65-9d05-eb05f5f2d84f,2e39c037-4def-4ef3-8030-20eb6f6892f0 +0x9b879eee67d839cc,8dfc2663-252b-4ba4-8e66-de9c3dedf464,d9a82207-cfb8-4158-800e-7d5031c10f8c,3cc47464-431a-40af-be17-45f27b333084,f7b7d937-82b5-4e2d-aef6-dd6bf77b3e0a,dbc0085a-ac00-4ca1-8720-70578e13f8df,64966eb1-df0c-4290-bd0b-261ccdaa510e,ec1aa55e-6c7d-405e-a4ef-1193f538b5eb,70432bad-fe46-415f-81e5-ef009427005f,faa38dc1-cba3-49a2-8831-e1db9eecb04a,b1a2ab4a-1709-4dac-90ff-b4fa4664c132 +0x31aaf244670ceb9d,eeee82c5-821b-482f-af72-75f25ab7c803,5e4aae23-ac42-486d-96d5-987acc472803,a37ca0a3-fab9-4f35-b5c7-e81a529a07d8,fbb0c91a-763b-4a68-a1e9-685949e1515c,eea250c5-57ae-4af0-8e6f-cb87234706c8,4a9fabaa-100a-4acf-9e7c-7408e3383926,2fdfcad5-1eb4-4538-a1e1-d399216ed5d1,8ea8ebe3-4995-4211-abe7-53d3bd931a3e,7cc2eb5f-4458-4d80-b397-09af1f0d877c,95bbde69-13ea-4e3b-939c-fc63e9850100 +0x42e1c4c3843ca7d1,5f6eb9e0-399d-425e-bafc-8b979a221ce8,ee306f97-53d3-4ef8-8108-c5db88cb9942,2f7d550c-2bf3-457d-ad2c-bbfec2d64558,3e5a5ff4-45b1-4ec4-8975-0df1023915cb,baa49878-2bff-400c-a55d-5193533fcbf8,8be56009-f4d7-426a-b0e5-50a528346cd5,ff733121-9d04-470e-949e-9c523869b726,4354c0a7-b1d0-4227-9e77-24c7df4376d9,dbd451ab-646a-4163-88e4-56a934e963f1,ed790d2e-c160-4aa3-ab63-284bf105eef0 +0xcfd9bad75352b43b,4139d1d6-7540-4195-ab64-1d38fd929273,b3818e7c-2797-4327-befd-de2e9489ed1e,5c410b7c-90a5-4be1-83da-ec0a8eb67bd1,45a66ba0-65b9-4d05-946f-b3be410c2f5d,bb77e323-fed1-4ab5-b392-07647d62f277,c143ddcf-f261-487a-8c92-3a59398988ad,5eea72d6-b093-4851-89a4-9346759cd605,1192a375-1c7b-4899-b754-2ffed1fa8425,67564f23-fa9e-45f0-8e21-8da4069af7af,16d54669-74cb-4652-a920-f3ba8bd6e926 +0xccad7ebbc155c12e,6289e565-a00c-4c5b-9541-b5d8b0308ccb,d2f8dd26-3f4c-4e3b-8a40-1387124886cb,39c63ef6-56d8-4e42-a711-c1ff42d9ad0f,31268590-ef5b-462e-b852-08ba61638c88,7a091202-e7ad-4793-a7f3-ceef4b441ad3,9452e1fe-3917-4c2a-8bf5-3607405ecc96,30f1e963-2583-4829-acd5-5ecd65fd6a9a,7ec389b8-b8f9-4e3e-83f1-fa95b230bdf1,607e06dd-92f0-430f-b397-f6ea32cf19e0,bdd4b8ad-f90d-4d05-8d51-59ced0c68542 +0x35b4e6c6be0718c7,791b4fae-3035-4983-9591-965c6b8888e9,dfd20017-0869-473e-86fe-3fcb3b9779de,3a4e7504-bb4d-4b64-a71f-13ca1743579b,d813789f-c76b-4b5b-b147-12a947a46255,83697cd5-074c-49fa-b873-937700d1ce0a,b3bcc583-aa11-449e-83d4-99d25513182f,1a278d01-0e2e-4ffb-845c-bd74baa5d404,165ea854-c805-48e5-a9e8-375becbe9cb9,b9571b57-e000-40de-8197-13483b5a43ef,223146d8-a4ad-4d5f-ab0c-c9fa90cefe94 +0x5b19e38751ca4289,6ec07067-c1d7-4b41-b4a0-8a8286312ebd,38ebb234-da05-473d-9001-b4f7c406881c,18503005-b6f7-4e03-93c1-f4c4b2c85066,18503005-b6f7-4e03-93c1-f4c4b2c85066,653d532d-e7f1-40cd-9fd5-2706cd073764,66edb57d-2615-4a0e-b0d9-2e4ae84f16f0,8a275c48-50f7-424b-917f-2d0cea785086,f78c7a85-a938-45ec-ad55-2188d396d1d2,653d532d-e7f1-40cd-9fd5-2706cd073764,6f3971b7-0545-4bf0-b948-4697b4b5c927 +0x834b160178840864,69549c24-812c-49fd-a931-7ffb312d9aa1,12df6d20-0048-404c-84ca-810361a59056,7e16ed56-c2d8-42f3-b656-27f6791ed346,37e96104-4fe0-4ce1-8195-e52b61db11a9,f3da7af5-40d1-45fb-8622-40319c4e48f1,dd196ffd-b010-4fac-9f6f-f0109ecd5906,ca67e4aa-4817-4c2e-83dd-8a6950fa7410,aa8aa959-1266-4c6b-bc63-98e9f4908cf3,141e9588-66a4-46df-83ec-6663f9d242d5,3a52e3dc-a060-41fa-8710-b8b921432fa3 +0xd6641d89e6372ee1,a05a27a7-ad48-4745-b5ad-52b55aab54e5,0595c170-f7a0-4215-afca-fde1ad6ef161,7fabc696-45be-434e-966f-1a50e0e2360c,66fb4662-3f23-4c20-bb7a-c17929094ab1,a00ff2b6-ccde-429d-ac53-4ed4355b37d2,9238d76a-432e-41d0-b80d-34895bf58565,f67927b3-4fd4-4bef-bc11-5834acc2e676,4f82094c-ffd6-4267-968d-b0624489e177,c996991d-d041-4811-9b2d-864ce3216e19,2ae2479e-9a30-4e4c-a222-4966e59884e8 +0xace9324ff2b33485,aa33911f-290f-4403-972a-8a08dd1efb7f,1331bf9b-325f-47ce-afc8-dc7bc5a7ca5e,d14f5eef-1fae-4091-8326-3908972d2cb4,726701b1-aeca-4ac2-9d6b-bc0fc568a41b,0f17775e-e180-42b9-b75c-3ba52a81d218,a1e84cb4-2e67-40a3-8abf-0a470f34cde6,490394c4-0630-4107-9767-4fbbb3d3c6a6,d61a126c-a6cc-4862-af8f-1a4565b1cad9,fb8242b9-3962-46b4-afa9-19384756c6f8,bec74d49-5fc5-4a85-8882-b085a237938d +0xb7c9e88bc8896107,ab613f76-f372-4e4e-a5a8-7438d8085e0a,5380a249-0bac-43cd-aa1c-cd191171d04f,5380a249-0bac-43cd-aa1c-cd191171d04f,5380a249-0bac-43cd-aa1c-cd191171d04f,0bd85b1c-3a9d-4099-a2a8-10f74e86887a,0bd85b1c-3a9d-4099-a2a8-10f74e86887a,0bd85b1c-3a9d-4099-a2a8-10f74e86887a,5a81bb7c-134d-46e0-b8ba-172e510e5984,5a81bb7c-134d-46e0-b8ba-172e510e5984,5a81bb7c-134d-46e0-b8ba-172e510e5984 +0xc5f2b68d7bf3fadb,d5a2e9c4-938b-4c6b-a1c4-ce909bb4e998,576d29ed-d64b-4d25-8a76-ce4fa5e9a2f3,c0744732-b2c1-48a5-a064-9de765046052,dab8ad84-c50f-4baf-bc66-af6aee19565c,f234f966-91cc-4e5e-bffb-8bd55838e439,8f23408f-39d1-4c99-8d63-03d4566c423b,918a169a-161b-4f63-beff-892c881e6a20,24a9f783-7412-4220-832e-e46ccc41436d,33f952ce-c9aa-4dcb-bfea-668cf9751afb,26622d0e-0eff-4245-b51c-6c65d62981a1 +0x030d66b38b7ab7e3,34ce05db-c1a5-499c-95ee-fb4033772a93,abab864e-11ef-4668-a258-cee4b9b010bd,d3169b4a-83e9-4ce0-aee3-5fcf0069b05c,bde87d9a-3cf3-4b6b-9512-54a079035a82,ad66c18e-b95e-44d1-a6b3-8e624f6a29b6,9f8f43bd-6989-4063-b3c0-c9773a73d6d4,5653986f-2b9f-4215-8629-b323ad90bc5d,49015252-dc79-4646-8a3b-e879693ee522,54991676-cc65-481b-a4cf-7b99f191b0c4,ee6b656c-0e1a-435a-9277-07a948e84726 +0xd0a5910478602d19,6565c118-707b-4899-9781-471aaa83c832,7875fdab-5d79-41d0-be0c-e22b9a98cfcc,d91aae81-4184-4361-a377-520a8ba082e7,4cf209f0-de0b-44a7-9c5d-3205c208dc89,f84115d0-0d0a-447b-9d43-6a4abb7be60a,6cef2ba5-68c4-4e3e-8964-8756ee1c488b,fde655a8-d87f-4e78-8541-daa9ffabc971,96cb81b0-88d6-46ee-b964-6aea005b009c,85f3117a-7583-4b47-9aad-e20bcc9824a7,6a3083c2-e369-4e2a-beb1-f41cac8628bb +0x25c3ed8f6c194bee,7bd6dde9-1d4d-4cf5-aa73-5fff5227bab7,009bf4f6-ade3-455e-8dc8-44ec97e34e7e,9d4c2577-faa0-4d31-8da6-05b26d916167,2d5e4785-8018-4721-a490-114db70dc278,99be1951-21c3-4b77-a475-48e9fb30b215,8059acd4-0eff-4aad-b524-e61a3a57cd5e,e2623042-fe28-4b9f-83df-e416d0a89032,3907ed8d-4f82-4f9c-af8e-dd550b2fbf8c,4d61bfe7-770c-482d-a09c-6cd8936c665b,1ea57835-03ab-4271-8065-3123423b25ef +0xe0680be899be88ed,ebce48a4-fffb-415c-aa35-1b1a23f22814,ec70c345-473e-4a7a-8012-d5b40b06b826,ef276de7-43f8-4c70-bc96-9bea010824bd,75d64136-3b96-46f7-9573-eaabe119801a,bacf5c4b-e95c-4081-b905-bed783456d78,a3f10a9d-6cf2-4d7b-afe2-86e3971d7b57,3fdf1c02-c9d2-4b50-ac51-fdf80139690d,51bebcf1-80a7-43c8-a51f-c17176098dbe,77019f84-ae5b-4c60-88df-b3467a6c9c3f,487890e9-d38e-487a-9d91-f13581c6781f +0x67e7b216af9220a1,9bb62484-91c2-4696-a0ec-553ed1fa297b,31998c92-8124-4c15-b434-6aaa820b9d52,34aeb8db-d589-4150-bd85-09339859dc0a,5f50bd63-8c89-4f2b-92fb-5adf085be3b3,5971a015-3e08-4c2f-b891-f3e01670919d,f79005c9-97c7-4f5f-ae6d-9211360a982e,b2a72821-fb61-4928-9ec0-ccb9707b5bb6,fec74010-474f-47b0-a2ed-0817e16d4b78,6c823f36-8904-47f7-8647-fb9f35620e40,d3b26ee7-fa3a-461d-a294-b46c69698032 +0x0443bb06b96ba03f,967cfea0-6458-4883-96c6-e4ba31323e65,fb561fed-d586-400b-ad08-d906e273a625,90b31335-1173-4e74-8f35-be88d23bf4d1,f12e9157-4d3a-400b-9098-bc0caab1bfbc,daa65928-532f-407c-9ee0-5a92e253ef36,d1346416-bf44-4b03-87bf-cafcaf59754e,b04239a5-faf8-4463-8872-356da4eba566,8571723d-4c63-43a9-a0dc-08807cd2b23b,d0e37196-b359-4315-bc63-aa3b4992dc68,f0ea2d1d-6d7d-46fd-b744-8a29d7745bf5 +0x5bd5385a428ef5b2,07205827-74c3-4687-9186-265ef4cb010b,17e78e1d-c163-4371-82a2-ebdd71107e15,eedea377-7d6f-43d2-9a9e-2d1aa3e47a88,f48de566-f7e4-4edf-851e-a977c1a130c3,1b32e45d-cb20-4b87-b714-025ab123492a,e1f1276e-3a8c-4959-aa9a-f0144b38be0f,2ef87988-7aca-4d97-b7f4-21e8d07e68c5,919a6878-4cc5-4dea-bcb3-75a7e91b8be4,2072da07-a4db-4e67-b162-1a71ee4b40b0,b15c78f8-9354-4a28-b648-512cedc6d383 +0x115561ebcaec1594,8380b66f-9009-481c-b662-69867e8d9b9c,c9c0b849-8d64-456c-a254-1b8a62ca1385,ae3c9c2f-355d-4ffd-910c-b31d97e919a8,431571f9-9c37-4e4c-a032-fafa8d8e6814,f591f4bf-8e1e-4ffb-a8be-0e36e94f9c50,064724ef-da18-4b29-9095-f1a6310c4425,4d7232ac-afab-4b4c-a1e4-ff5e05c86703,0822f543-93aa-49dd-afbd-5fa9c511bb4f,d1272345-da9c-43bc-a318-99977dbdcf39,3c46ed0b-6dd1-4864-a801-701f68cd8f26 +0x813eecae0ea74db8,a071708c-026d-4aa3-8002-20e9b5e724b9,d43ef994-88f3-42ae-90aa-837f0765b4b4,6be6cb48-1089-43f6-897c-5f77edafde87,0eb399a1-69eb-48f2-a541-b3e1f4890e28,d816e634-046f-4bdf-b653-12ee1e669123,04c7350e-dc1a-432c-af4c-c7d35d93f5af,cb212f92-991c-40c9-b900-20618cab545e,2482d622-8ee0-4377-8bb3-1071c6bc3d93,32e16342-d767-4cc1-9518-e43cd504c428,b3d21524-4c52-43f1-9e5e-7ff64106a3f9 +0x04845feb89c563b8,f055be97-6e96-472e-825e-1647a70596e1,214d4293-e98e-4942-aaea-c6194feddbca,a5e1f1d4-1a77-4c4e-97f6-7cc74eb6e2a2,214d4293-e98e-4942-aaea-c6194feddbca,ccc68674-c4b0-4c93-b0cc-2333934b71c9,88b26d8d-5e57-470e-b93e-e832f839c96d,88b26d8d-5e57-470e-b93e-e832f839c96d,ccc68674-c4b0-4c93-b0cc-2333934b71c9,df9d7ccf-697d-48ea-b3dc-995ada142c18,df9d7ccf-697d-48ea-b3dc-995ada142c18 +0xb64f9203a5c5ac68,d371c094-1f79-49f0-837a-55765d9f85db,f5220e57-929f-4b60-8013-f863bb7301c3,fde65319-256f-4108-99f7-4ccb35d91352,9dc121ef-bee2-4d96-9108-e3d8b3d0b831,950be74b-9412-4bdb-8c11-1a4eb03823f3,8a1c69c1-ee0e-45f7-971d-a1a167698622,e780ccac-ed73-4f80-816b-d28d533cb88b,14d513f2-b9cc-4618-b77e-c684b845e8e7,bcad154f-4365-4b10-9445-012e0645d920,98a8d821-69ea-46bc-b7cc-1c70cad468fd +0xfad196beb00c807c,c95af050-35d0-4876-b787-42a941bcea0a,994a4b7d-95cf-476b-98f9-5a16b5663350,d46c90c2-20ef-48a6-982d-69f8ecc5952a,e2ddefdf-a89d-4257-b294-00eaa69f81c8,d5ac28d4-8e92-4e94-b063-e232b01ddd38,c0e125b8-53f4-46e1-991d-8fb213ad52fe,9fac6d0f-ff8b-463a-ad4d-68ae5c4e8cc6,9ffccbb1-9d8a-4f09-bf19-e63bccd90c8b,bbe6d76c-2396-41ef-b067-2697c4a66dce,cdb4365a-0be8-4ed7-b9f8-e5b01a654057 +0x32c66e4278279976,47fb87de-b4fe-4c6c-9287-39045833944e,1f5d96a3-dfbf-4572-aa4c-3e6adf689012,d6a79448-97f0-4599-91c8-740060bdf766,4b3977bc-8bf8-4d7a-97dc-8d1fc9a13f15,36722c49-0eeb-4295-ae96-ec77143279e8,36174172-a08b-47b2-b0e8-d9d831d5a693,41b3d369-f443-4994-808e-6c780140fce9,ab5c03c6-0f1d-4a0a-a137-00f79a84f274,6cc73251-d006-47df-8809-eff623426c79,7424ad3f-6264-46d1-8ebe-e8a7a0aa1c99 +0xfe5007fe4ddf8f1f,b3423016-652f-4d65-b12c-0e729a5c585a,2530c717-ff16-4cf2-bdaa-dfdf9ae6f567,1233f88f-ce75-470d-bede-222319f41d92,e8babeb7-957c-4c9f-bd02-e782e9e2aa2a,c6173d43-6a77-466b-af53-33a178e53688,43924abe-dd79-425e-93e5-40236ffdf9a3,f2cdf37c-2f37-45de-8679-0200050098d9,0ac965ce-bb35-4459-853c-459bb5041bd5,3fd4d86d-a403-448d-9c4c-8e4466ac2dc2,f95fe474-cf57-4d3c-bf93-5c2b65a7a6ae +0xf91acb323cb9df06,37350ce5-ce18-467d-afa0-275e6754e95f,5ad290b4-f3b3-4526-9485-515316ecdc33,62e171fa-45f7-4060-a6a5-1b7a7891cf22,3eae2644-8253-4fa2-8757-7a26fca150c3,8eebee5b-c678-477d-b11c-55ec0d5c3536,c9b4510a-ec9b-4d16-8211-00fd9c349e1f,7a4ca60b-50f2-4f80-879a-59a4bf5629b1,18f5b8ab-c488-48fd-8eea-73605dfa64a8,6fb91fe3-98f6-4ce4-8566-22e0e483f7c0,f8ac29ae-ed18-4bfe-a5ad-fe280bd5df02 +0x2f58577dd7194d85,fff69832-27d9-465a-9701-d56392f623f6,d3735925-ad7b-4a72-868d-9a36ece8a53c,c5c18be4-29c2-4807-a16d-b5a6243b196d,1303ada7-a896-4db0-be01-af31eb23fa17,59fbc3a1-1521-4f12-9db8-25220b525eb7,c19742ba-2a67-480b-a477-3fbcd19a8d3c,9c116ba1-ab69-4f96-90da-818fcd8e0c74,d00d392a-8d4c-49e7-aaeb-1a1951173943,0814a1f7-a6c9-4239-a1f3-9e5e427743bc,9de0d688-aa36-4ff9-ae22-763808d07a0e +0xcc2563b2fe07f312,edac5a33-4c73-4609-8409-894ed5be5659,6e843d65-28a2-47b8-9624-0860160f1ffa,97bd8286-1545-47c2-8d53-b76526e6c72f,a3186f33-179e-4fb8-86fb-a779cae07a43,0cbce21c-67d8-4ddf-9126-f7337bb0efcc,03da74d5-dacf-480c-8c8e-4fc1c9851da1,ee9e6682-8252-4431-a584-b82bce3b4ab6,24d86405-3f22-4c36-b444-93b713008b5e,3e9913b3-fcac-4cbf-9d03-46bbea066e07,2d549e68-cbee-4e75-9f1d-186603cb59fe +0x3fbf6fa2d72851ea,755f1c40-fdd6-4673-9f2b-5541525791e2,cfe4babd-1160-4590-9c03-82b886497403,6c4f2c4b-a5b3-4fe0-a1bc-30e70c6469ad,c06bd942-1e5f-41db-ac60-0e9830746011,6227b450-3777-4904-a6bf-49872703d0cb,4dd6961e-ff01-499d-b51d-fa899b42ae89,3ab3bb86-5424-4676-89aa-27088deb3ce0,471719f1-ff5e-46e7-96c1-d63c1384ea1e,90c35677-8741-4409-9822-f9d39d8a3a08,4bb6670e-2657-4780-95c6-102531ab6cad +0x31007f0d4dc6f5fc,66145dbc-d978-4206-bc21-a9b0ea333297,f4dcc505-12e6-4b0c-ac5d-f06bf78d49f7,308e749d-c489-414e-9434-0f733d3881ea,407b52ad-8250-4113-a194-58e6da2018ec,52b4d8d2-09e5-47c7-adee-8a634eb23b88,3a62058d-64df-4bd1-8e37-b4602a460cbf,175efb3a-b0bf-4d67-a41b-4b35ffbf0730,21094d5f-2977-43ec-ab35-5c9313a454c1,68ef8822-e537-44ae-a58f-21a2c804cbe1,fc1edb57-1ae5-4939-8c4e-49cdc89f9b5a +0xb76e43c87818681f,c98f65c5-a25e-47f9-ae65-34ce038daadf,ff4d585d-fb86-406e-8af8-d7617e13d8a7,ad350ea8-ab84-4682-9419-8d9fc93e0ed3,51a97864-43c3-49a7-8409-93457d126b6d,97ec1585-cba8-4e5e-a24e-fa8c2833647d,5ed76c4c-a518-4e8c-8453-b66d917edd64,17e04799-cccd-4826-ba13-57406d85de3c,7ff65714-e580-4dd7-b620-48d09d0e2e07,e13cc0f5-c7e4-4dcc-ad36-d343b24f8892,597a3cc9-a4a5-479c-8781-c224c11e2bf7 \ No newline at end of file diff --git a/data/seeds__nflad_playoff_vets.csv b/data/seeds__nflad_playoff_vets.csv new file mode 100644 index 0000000..2cf4d16 --- /dev/null +++ b/data/seeds__nflad_playoff_vets.csv @@ -0,0 +1,28 @@ +wallet,id_0,id_1,id_2,id_3,id_4,id_5,id_6,id_7,id_8,id_9 +0xace9324ff2b33485,aa33911f-290f-4403-972a-8a08dd1efb7f,1331bf9b-325f-47ce-afc8-dc7bc5a7ca5e,d14f5eef-1fae-4091-8326-3908972d2cb4,726701b1-aeca-4ac2-9d6b-bc0fc568a41b,a1e84cb4-2e67-40a3-8abf-0a470f34cde6,490394c4-0630-4107-9767-4fbbb3d3c6a6,0f17775e-e180-42b9-b75c-3ba52a81d218,d61a126c-a6cc-4862-af8f-1a4565b1cad9,fb8242b9-3962-46b4-afa9-19384756c6f8,bec74d49-5fc5-4a85-8882-b085a237938d +0x173b9231273cc349,2eeb2a05-8dd9-4606-a008-33a635bd5ff0,811dc290-8b9e-40cb-9578-696ce2ad7d2a,bd0dbbca-996d-4155-872e-0b4104328227,dcef4c4b-948d-4d50-a97a-75498d98a7fb,002e85b4-e8fb-4d24-b265-519c04517a08,cccef914-2983-413f-9dbc-88987289a856,ae53bab4-14fa-4021-82f1-e4ea278c9c45,05913398-fd9d-41b8-b93c-a17a948db27a,b1feb65d-78fb-42ac-bead-ec9b5199f0b3,f43e4ad2-fd72-4277-b57f-f2d2a9be4d2d +0x0443bb06b96ba03f,0a53ede5-bf07-474b-90f3-cb5a4948d24c,b13c0042-e76e-4a7b-9cfc-7036193ddd16,962f416a-d165-446e-aee1-c882dbfcff94,b67915ae-bda7-424c-9efa-74402bb775cd,39193d76-3b6b-4e8b-9ce4-e46fd5fdab09,00e0c7ea-fdbf-4691-a3bc-8727ec0e75fc,1ac71755-a13c-492f-ba91-2d97a6ef860f,c4c4b40c-6d31-455b-aba3-24e9ce517492,2aaf8138-d1ee-441d-9ade-6ef2b2ffe861,649cd9c0-83fe-4e66-a5c7-f00d0fc5c405 +0x7541bafd155b683e,2c24479e-3c9d-484d-a954-fa87c26d0b22,a32dbacd-5e8f-473f-8e38-f92a816bc775,e30ccda6-28c4-4977-8faf-f76aaad7c263,1096b957-9238-4bf9-8689-1e5f85bffbdf,92cd7488-eaf1-4b67-952b-434884fed0c1,86ebe2e1-40b5-432a-87a4-e69b60bb4618,a54b48e6-7acd-4bc1-abc0-3f76c4224343,c5f8dba3-f3e4-475d-a645-4c0c456fd30b,a2dfb3c6-e86a-471c-a39c-f63ab24ebb35,c66e8969-c861-4e7f-bfd1-4ba9a51798de +0xec7bfbf14f264fcf,5b4db7be-5273-4d18-a0bc-41c29f2b2f79,f4b9cacd-7029-49e8-ad3b-a76b82f633ab,f753243f-01bd-4c81-8ef2-d4df59c92869,9683f3bf-0c4e-4cdd-be54-d90a7d06a4c6,df5c056d-d22e-4251-9ee8-0196294be75f,8dae1f3e-162b-4ce8-a17a-41848eee9374,f98fc698-bed0-42e6-a7ba-bdcbe49495d0,94f4810e-7661-4441-9747-e42a55dc5654,970deca5-e7f1-4657-af33-3688b6a7c8ba,e37d67b7-2aac-4605-96a4-ce0e6175bf99 +0xfc7e83400aec87d4,2660b923-1104-4a83-88d9-9372376010ed,56ba58af-00bb-4845-b47e-5ae525812b84,4e5ad7dd-9d9e-4352-99a6-9ddcd778a13b,e4d56a8a-83d8-453c-93a4-2136ad2458ff,469323e2-97ca-49ab-b7f8-523ba028a052,f24cfc95-1390-42c4-9e80-b9c4811f1b34,3361069a-8cdd-413f-929c-c70601d013fc,4d036cb1-1335-4750-a443-6844111f137e,81047553-5d70-477b-8b90-f09a77d85b57,2f806d17-976a-4e45-8d28-a4c643788299 +0x7e378ec7ee8e14e8,21929db1-c8c5-4cde-bbb4-145abc2f57fe,70f74ddf-a9fd-4a1e-9933-0cc5e2d0dcb7,96bef61e-ecfe-40fe-9c38-77ed61323ba9,58daa3bf-529b-4cea-9e7c-fcb4b800f789,12b96fdf-efb2-4f74-b72e-80e42f6d3bd8,0e76d1c6-dbe2-4ac4-8c08-b7e3d9d91e13,1222c92b-5532-4fac-b641-f7b7bfcf1dba,97e377c0-b98c-4e6b-8eec-00ac1fe89159,26006bb1-d1ed-40e7-8bb2-57ec41aac5df,1e29c34c-006d-42f1-be86-619a4b7c0d33 +0xe5d6e4db7b16dd40,364ee38f-6c8e-4b50-b1e4-18ad90605bd4,3e6324e2-06ec-4f9d-a24f-4c78a732cb28,5ad7e6dc-8b55-41fa-ad6c-379a1f3d78c8,6f12d5ce-40ac-4d4c-8bbc-9e57d116a116,f1f9f7fd-d162-4d00-8be2-735417d0bc7d,db4843b4-b103-48fb-a0e8-a151adb40122,2c0ed8bb-22dc-4dd1-be18-5b24c5a47f0d,b8488524-d896-49a5-bcae-037f460390ee,e00860aa-a7ec-4e2e-8fef-64f12e4e4252,6e72fdde-dd74-474e-bff6-6c01a1e6e4f5 +0x4e2c842cb5d154ee,54654272-c5c2-4f1c-96b3-7486035b628a,cccf0df0-d020-473e-9153-8e5bfd239340,86867220-202b-4362-beb1-0d006297f712,5df54205-eaf3-4a77-9566-7e79581d99c2,9a25faa3-c31a-4d14-a60f-5c69dc456891,89acf312-ef5e-40cd-a6fb-031c5b5c003c,56c138e2-a1fc-4db0-94c4-853cae295613,2f3cf1b9-08f9-478d-8757-96a2b9c9f10f,d3912492-c054-47b7-bf94-d39f19c432c3,34ca7c36-aebb-401a-96bb-4bc8b6bc06d7 +0xf90cc1a4c029c83c,959aef3a-8583-4b73-812f-032a68351872,8fd0541f-bd6b-4215-87d3-1deab5f78668,38f7d1dd-c215-4185-9dad-4c930670e6a4,56ca08d4-aa45-4a41-b2f7-6fb3cf06eee1,e3dd2608-0eda-4b5a-a482-07c6d8b64ac5,5fb154ed-4585-492b-bd48-2115d7552834,a04e37f4-462a-45cf-a902-9c50c6b65b59,f6464ba0-b9dc-4660-be66-31837c038256,c2d25f8f-c1c9-4e65-9d05-eb05f5f2d84f,2e39c037-4def-4ef3-8030-20eb6f6892f0 +0x4f768fbfc061a5b6,0b26ee74-9db8-4f41-a176-a8de8326d105,ec114b2a-30e3-4f2f-8dc8-95a43cc80898,e992dd1c-d527-402f-aca5-d752385d77be,cbf05869-f1f4-4faa-9f36-d23401142222,6f56c515-c08b-4451-9267-41a60cbc161e,15835a0c-b024-41f3-8f74-4e41bac9af21,d952a97d-7be1-461f-abb5-cf091a41407e,34bd662f-c563-4c5e-b153-75f41ef48a58,4a3e06ab-01ea-4b89-964f-5d4e82ee3f26,07de7dbe-c1fb-4e3c-b7b2-1481e1470768 +0xa5a13301c0ba6d59,ee94b3cc-5133-436c-8883-48f153e08d0d,8eaf42b2-dc7b-4559-8e99-3af0cfd77173,922925e7-b4df-475c-8c15-eab410890251,a94b5129-2ee9-4b6e-9252-f4c4ec4139b7,f592b613-b5b7-4a5c-8bfa-c2e0f462ab2b,6b6e0437-ace8-4d82-b89c-ba05434339c4,fcf47ce2-4755-4d2b-84fb-141735596861,544bc77a-11b6-459d-bddb-351555a346a9,742fd7a7-2470-440f-9e7c-0bd4112b1fc7,df0bcb06-90e7-46be-b630-38abdff79bf4 +0xd41ad8e9e886c87b,69b99a58-02cc-448d-ad6f-d4b798ea0eec,635f2b68-8232-41a1-831c-5b67d15bc45d,9de8638a-8bc9-46ae-9241-39ab252e590c,285d8cb9-6b0d-4a1e-80a6-e4f19ca04a7e,e7285e1e-6673-472b-b281-757cde637209,0060aaf3-ab8f-420c-9e35-ee333b1d735d,4090c32b-d044-4751-a75b-6e580b1fbac3,f47503de-707f-4c77-9a76-53c074fe49e6,793efd9e-941d-4962-9959-803bb43c3c4e,34cec5df-11aa-4fae-9380-0345d4a0a44c +0xbee776009d396a51,ec3131c4-a41e-4af6-a72a-dbccb4aea45d,227fb823-eba8-4d0b-ae82-703551e1eedc,261a4562-6031-4a53-9b3a-458e0aa0aca4,6a77d90f-e3ce-4df4-aefe-55e10af261a4,709feba2-f99d-4863-93a7-6a1be26d4a3a,5e7242e4-74ca-4dfa-9c63-493f1ef0dac0,054b1031-da47-4e26-9e18-afd62fd12771,f6ffdcb9-388c-4f6f-bb42-1fc23fa0297c,df0ccee5-7ad8-4981-b712-f9ad56231d45,e7aa6646-be68-43de-89bc-98eb6a4ebea9 +0x760e5312752dc4ae,372d4129-ad8a-42b5-ac47-b32b3d29a63b,2ce55b5d-5a8a-4ac8-bf03-b597648c6850,ee60a3f4-57a4-4214-a276-244cc893e42c,ccaa2c99-7a1d-4cc9-977e-5894923f3db3,26bcb7fa-96e8-4d34-9626-bd1dc1aba7eb,6d6f5fe9-cff8-495e-87cc-87a7d92d4e97,0cc3457c-12a3-480b-a166-b3f26656d81d,d925c4e5-6891-4e45-982b-9a46eb9c2992,023d092e-7479-48dc-aa74-8c9545cc05e4,21874c65-51e7-4fed-9f01-8b1437136aa3 +0xa4838affeea62764,78bf941a-caca-4720-85eb-801e1e6f30b9,2add2b0f-cece-4461-a9eb-f5fbf08fc175,6eb164ec-66b0-4e10-8918-9d3746d47e7f,149c71af-534d-4e85-982a-37c7e791f48c,dafdf13f-426a-43bb-935f-e1a85115b11f,553a8460-ad9d-4730-81c2-08da15840139,923b8adb-b783-4a5f-8ee3-98538e1dc699,d09c9ce0-7667-498f-aa4e-574575e3a4b4,628781cb-9992-4d01-b103-4eae53c092a4,c366f0ea-56fd-46bd-a28b-b63eadd7e925 +0x2252abfb10fe4ee3,a1864fec-4f24-4d33-96fd-93004552d9ce,f0ba5448-ee62-4708-8b4e-9e546d92852e,ec1a0c04-74dd-4f1d-9353-f2ae10c429f8,33722d52-27e3-45f5-9b2b-9e775090b49c,5ecd1ed3-ca84-41fd-80a5-65969144b484,383d978b-ae7c-4eb4-bf07-3354bd66b627,f52176b4-c73d-4b35-85c2-2a2dd8ac4094,90582ebe-9e5b-4564-a6a4-3d3082d4cb63,9aa5e37e-ed8b-4f10-bc5b-4bd6b873f287,f58bab21-bc2b-494a-ae24-e173c7236281 +0x3376758a5e4ebe6c,104f309a-a304-4e3e-84e2-4b839a76f34a,ae21dd58-3932-430f-8825-127414d7c157,46edffe9-b0e3-4114-b03e-1940b25aec81,6022db61-4c21-40ed-b78a-b3cb963288ee,7996faf5-eaf2-4a5e-8fd7-ae4d0adad82c,f9665537-69b4-4001-94bf-6161bb4a937e,1d43d72d-e533-4eec-a218-af6d58f66628,b4ff9126-dbf4-47f6-bdf3-451261b183be,fd7cba15-90fd-40ce-bb66-a44cf085228d,8687808f-e5b9-4178-b64b-629959d63746 +0xce609693f76f1467,2b49511d-bd47-4278-810c-27f0b35fc07f,2ce36831-3213-489e-9e87-53844696fa88,dae1740a-b465-400d-9955-d5054642b0c3,fb821c60-67d4-4ad3-a519-6530c388c22d,d4846a67-0030-4ace-94f3-70cb4b760185,e786cc09-310d-4db7-9ace-22bfca73ae72,4045c143-5e0c-4753-93fe-3329bc444da6,488fe010-2e31-4d75-999d-ff7890d622ca,00e0721d-f762-4f34-ae8c-cb548a1c9193,90b72019-91d9-4e96-bfcc-1611cbdbec93 +0x9b879eee67d839cc,f4de468e-8a30-49d4-b0e6-506d9474c381,4f5a0ae1-b6b5-41aa-b3bd-d8e5caca9fb6,62486154-fb61-436a-a86d-d57c61d0a9ce,654583e0-ad08-42b4-a078-db1549357089,5ebd22f0-e43b-4519-8fa3-1236cf61f81b,bcfe6fda-d9a5-4859-8f04-e6e6e574cfe5,bfe9395b-e3e8-4518-a2c6-8775e270bfcd,0af1cb79-83bf-4d6e-be12-ed9f1ed4e920,c949e453-21bb-4e19-95f8-aa6a1214ee85,c269f632-24ef-44b9-88e2-f42e9e8da534 +0x31aaf244670ceb9d,367dd4fe-8788-4862-afe0-69a282a6e8f6,83eb6ae0-ae02-4fef-a3d5-ef61ed761d64,efba77f5-0a34-4384-8d3f-e2f430e12e67,94277a34-e4aa-4cde-9f52-16fbf53e2479,e26dc40b-f8e8-4bb1-90cf-aecfe78a3027,c8152fb5-a1d3-4ee6-a3e4-057d26f40078,687a634e-55b4-4a41-8d20-31fecd1f7666,c3d7b003-c62f-4838-b33b-3a4370451ef4,18c42e3a-3495-4001-bc97-e420f0b95ab2,6f5467c8-9aa2-41a2-b475-823f320c638b +0x42e1c4c3843ca7d1,09b7520b-7559-42d4-b9b4-5236b5f03768,ee306f97-53d3-4ef8-8108-c5db88cb9942,2f7d550c-2bf3-457d-ad2c-bbfec2d64558,ee306f97-53d3-4ef8-8108-c5db88cb9942,96c2f72d-5740-4bcf-b391-00968045590d,1945bfc2-7c51-492e-b34e-2ec69598b612,f83e337e-6e3c-4ab5-963f-7d22daeaf10e,26a76e1e-ec04-43ef-b1bb-66eb27aba739,7936ce06-f286-4d44-bdd2-94441084211d,4e8f2f44-431b-4497-9975-555323f64553 +0xb0c3cea14d23d224,9126c8d3-0296-4539-85c7-f87157adaf30,0c32e373-ed48-45ec-8b85-9a3e6b9c394c,58df1592-2761-4c5b-8561-79e30eafde8e,8ca1194b-d6f7-491c-be11-9d64949e8c73,88d512d0-8742-4dd1-9230-e651cedc5cdc,45edb88f-1845-4787-a934-c9216f6b4b7e,d03964ed-5847-41ce-831c-16f7cc325556,f261889c-2faa-461b-a43f-80d073215734,0306db11-f3eb-44e5-894e-14fe82592f2d,e83de93a-52dd-4bd5-ba5d-64c8c4e0f9da +0xccad7ebbc155c12e,784c9c23-e9e8-43b1-afe6-5a57525b9e23,31e0080e-2d61-44c7-998f-46877986b2b1,77ab3e4d-06f5-4364-9fd1-ed3f9f2cea44,aa60ef08-a7d6-4077-bf0c-7cd2a30a624b,931d7d5c-4b8b-461a-9b48-56402c5e274d,8037b193-1b6e-4a4b-bade-2773738ecb4f,3bc4bb77-6869-4287-87ca-aaa0269e1356,2e4c6e20-78e9-4c52-8509-0f215b512123,a6d7d85b-027b-4fa8-b373-d5957ee77afc,c9ebcc85-38c0-4970-a128-043f3c9d21eb +0x7b70673057616601,d70f4c39-e976-40c3-9ec0-0f5b0713bd4a,d9ad5b83-4db5-4334-9336-97920356761e,b653a900-2ca2-4d0c-ba44-ad866c9d6289,389202bf-7788-4bc2-bc6c-43dfa7e24dd6,f46f036a-0859-44af-87ee-8a14b2df5ef5,0045fe51-bece-4dda-be52-f7722f2c0eed,ad0bf3e5-fd9c-4019-955b-21457fb24f2b,4710d615-2c3c-45b9-bd3c-264cd34bcaab,f49b19a6-65e6-4a9a-9cff-cea0545d937e,979b3a96-5443-444c-9743-8a8e470a9119 +0xc5f2b68d7bf3fadb,a79371bf-8fb0-4386-b727-49c5371fc14c,2b474ecf-5e16-48ac-92fd-022f82abe128,3ebc7164-57d3-4b8d-a976-66f833cde8e3,45646a4b-d8e4-4cbb-9b0a-79f2b871931f,e90cae5e-c72a-4b52-8609-0ea135236a07,f234f966-91cc-4e5e-bffb-8bd55838e439,e102165d-0c6b-4155-be43-1e481ee73da5,41a204de-595a-45f2-a255-2de04bf734b7,a0fa1fa6-3972-4ba6-9443-e71575f87822,9a005460-9758-4543-bc15-1b6d2736bb77 +0x31007f0d4dc6f5fc,8ff03e2a-872e-4e54-85c1-ab19729167a4,6b5ad88d-ec04-411b-b183-fb9e60ad2256,5287a4b5-2896-41fd-9407-8ecd26d61f61,7c20be46-083e-4712-a513-4eb748ff0a60,0ad83fa1-33c6-4c7e-a0b1-cd4660c3016c,6ce6d3a5-4c6e-403d-a6bf-aa643263eb07,7180ebf9-7680-49b3-af50-24e42d1dc842,9fb4d93b-a0d2-4f58-8158-eaba98b64d85,01ce2edf-366e-4446-a58f-9049fb33a15c,fa4edd13-d556-4bda-9ac6-c6ae3ec5424c \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index 420f164..e03185c 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,10 +2,10 @@ # and underscores. A good package name should reflect your organization's # name or the intended use of these models name: "flow_models" -version: "1.2.0" +version: "1.7.0" config-version: 2 -require-dbt-version: ">=1.4.0" +require-dbt-version: ">=1.7.0" # This setting configures which "profile" dbt uses for this project. profile: "flow" @@ -31,8 +31,7 @@ on-run-start: - "{{create_udfs()}}" on-run-end: - - '{{ apply_meta_as_tags(results) }}' - + - "{{ apply_meta_as_tags(results) }}" # Configuring models # Full documentation: https://docs.getdbt.com/docs/configuring-models @@ -74,7 +73,6 @@ dispatch: - flow-models - dbt_snowflake_query_tags - dbt - # query-comment: # comment: '{{ dbt_snowflake_query_tags.get_query_comment(node) }}' # append: true # Snowflake removes prefixed comments. diff --git a/macros/api_udf/get_allday_metadata.sql b/macros/api_udf/get_allday_metadata.sql index dab1041..2fb9af4 100644 --- a/macros/api_udf/get_allday_metadata.sql +++ b/macros/api_udf/get_allday_metadata.sql @@ -168,7 +168,7 @@ CREATE OR REPLACE PROCEDURE {{ target.database }}.bronze_api.allday_metadata() create_temp_table_command += ` SELECT - {{ target.database }}.live.udf_api('GET', CONCAT('https://nflallday.com/consumer/graphql?query=','${query}' ), {'Accept-Encoding': 'gzip', 'Content-Type': 'application/json', 'Accept': 'application/json','Connection': 'keep-alive'},{}) AS res + {{ target.database }}.live.udf_api('GET', CONCAT('https://nflallday.com/consumer/graphql?query=','${query}' ), {'User-Agent': 'Flipside_Flow_metadata/0.1','Accept-Encoding': 'gzip', 'Content-Type': 'application/json', 'Accept': 'application/json','Connection': 'keep-alive'},{}) AS res `; diff --git a/macros/streamline/models.sql b/macros/streamline/models.sql index c8cca92..7d5514e 100644 --- a/macros/streamline/models.sql +++ b/macros/streamline/models.sql @@ -101,7 +101,7 @@ FROM TABLE( information_schema.external_table_file_registration_history( - start_time => DATEADD('day', -3, CURRENT_TIMESTAMP()), + start_time => DATEADD('hour', -36, SYSDATE()), table_name => '{{ source( "bronze_streamline", table_name ) }}') ) A ), diff --git a/macros/streamline/streamline_udfs.sql b/macros/streamline/streamline_udfs.sql index 441b180..84d77ae 100644 --- a/macros/streamline/streamline_udfs.sql +++ b/macros/streamline/streamline_udfs.sql @@ -36,7 +36,7 @@ {% if target.name == "prod" %} aws_flow_api_prod_us_east_2 AS 'https://78rpbojpue.execute-api.us-east-2.amazonaws.com/prod/udf_bulk_grpc' {% elif target.name == "dev" %} - aws_flow_api_dev_3 AS 'https://ul6x832e8l.execute-api.us-east-1.amazonaws.com/dev/udf_bulk_grpc' + aws_flow_api_dev_3 AS 'https://j6qalrfe69.execute-api.us-east-2.amazonaws.com/dev/udf_bulk_grpc' {% elif target.name == "sbx" %} {{ log("Creating sbx udf_bulk_grpc", info=True) }} aws_flow_api_sbx AS 'https://bc5ejedoq8.execute-api.us-east-1.amazonaws.com/sbx/udf_bulk_grpc' diff --git a/models/bronze/bronze__labels.sql b/models/bronze/bronze__labels.sql index 06fa634..db0e93a 100644 --- a/models/bronze/bronze__labels.sql +++ b/models/bronze/bronze__labels.sql @@ -14,11 +14,13 @@ WITH labels AS ( label_type, label_subtype, address_name, - project_name + project_name, + _is_deleted, + labels_combined_id FROM {{ source( - 'crosschain_core', - 'dim_labels' + 'crosschain_silver', + 'labels_combined' ) }} WHERE blockchain = 'flow' diff --git a/models/descriptions/__overview__.md b/models/descriptions/__overview__.md index 259e55e..16457ad 100644 --- a/models/descriptions/__overview__.md +++ b/models/descriptions/__overview__.md @@ -3,10 +3,12 @@ # Welcome to the Flipside Crypto FLOW Models Documentation ## **What does this documentation cover?** + The documentation included here details the design of the FLOW - tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/flow-models/) +tables and views available via [Flipside Crypto.](https://flipsidecrypto.xyz/) For more information on how these models are built, please see [the github repository.](https://github.com/flipsideCrypto/flow-models/) ## **How do I use these docs?** + The easiest way to navigate this documentation is to use the Quick Links below. These links will take you to the documentation for each table, which contains a description, a list of the columns, and other helpful information. If you are experienced with dbt docs, feel free to use the sidebar to navigate the documentation, as well as explore the relationships between tables and the logic building them. @@ -16,10 +18,13 @@ There is more information on how to use dbt docs in the last section of this doc ## **Quick Links to Table Documentation** **Click on the links below to jump to the documentation for each schema.** + ### Beta Tables (`FLOW`.`BETA`.``) + - [ez_nft_topshot_packs](#!/model/model.flow_models.beta__ez_nft_topshot_packs) ### Core Tables (`flow`.`CORE`.``) + - [dim_contract_labels](#!/model/model.flow_models.core__dim_contract_labels) - [ez_token_transfers](#!/model/model.flow_models.core__ez_token_transfers) - [fact_blocks](#!/model/model.flow_models.core__fact_blocks) @@ -27,29 +32,36 @@ There is more information on how to use dbt docs in the last section of this doc - [fact_transactions](#!/model/model.flow_models.core__fact_transactions) ### DeFi Tables (`flow`.`DEFI`.``) + - [dim_swap_pool_labels](#!/model/model.flow_models.defi__dim_swap_pool_labels) - [ez_bridge_transactions](#!/model/model.flow_models.defi__ez_bridge_transactions) - [ez_swaps](#!/model/model.flow_models.defi__ez_swaps) ### Governance Tables (`flow`.`GOV`.``) + - [dim_validator_labels](#!/model/model.flow_models.gov__dim_validator_labels) - [ez_staking_actions](#!/model/model.flow_models.gov__ez_staking_actions) ### NFT Tables (`flow`.`NFT`.``) + - [dim_allday_metadata](#!/model/model.flow_models.nft__dim_allday_metadata) - [dim_moment_metadata](#!/model/model.flow_models.nft__dim_moment_metadata) - [dim_topshot_metadata](#!/model/model.flow_models.nft__dim_topshot_metadata) - [ez_nft_sales](#!/model/model.flow_models.nft__ez_nft_sales) ### Price Tables (`flow`.`PRICE`.``) + - [fact_hourly_prices](#!/model/model.flow_models.price__fact_hourly_prices) - [fact_prices](#!/model/model.flow_models.price__fact_prices) +### Stats Tables (flow.stats) + +- [ez_core_metrics_hourly](#!/model/model.flow_models.stats__ez_core_metrics_hourly) ## **Data Model Overview** The FLOW - models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** +models are built a few different ways, but the core fact tables are built using three layers of sql models: **bronze, silver, and gold (or core).** - Bronze: Data is loaded in from the source as a view - Silver: All necessary parsing, filtering, de-duping, and other transformations are done here @@ -57,16 +69,17 @@ The FLOW The dimension tables are sourced from a variety of on-chain and off-chain sources. -Convenience views (denoted ez_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. +Convenience views (denoted ez\_) are a combination of different fact and dimension tables. These views are built to make it easier to query the data. ## **Using dbt docs** + ### Navigation -You can use the ```Project``` and ```Database``` navigation tabs on the left side of the window to explore the models in the project. +You can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models in the project. ### Database Tab -This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are *not* shown in this interface, as they do not exist in the database. +This view shows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown in this interface, as they do not exist in the database. ### Graph Exploration @@ -74,16 +87,16 @@ You can click the blue icon on the bottom-right corner of the page to view the l On model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the Expand button at the top-right of this lineage pane, you'll be able to see all of the models that are used to build, or are built from, the model you're exploring. -Once expanded, you'll be able to use the ```--models``` and ```--exclude``` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). +Once expanded, you'll be able to use the `--models` and `--exclude` model selection syntax to filter the models in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax). Note that you can also right-click on models to interactively filter and explore the graph. - ### **More information** + - [Flipside](https://flipsidecrypto.xyz/) - [Velocity](https://app.flipsidecrypto.com/velocity?nav=Discover) - [Tutorials](https://docs.flipsidecrypto.com/our-data/tutorials) - [Github](https://github.com/FlipsideCrypto/flow-models) - [What is dbt?](https://docs.getdbt.com/docs/introduction) -{% enddocs %} \ No newline at end of file +{% enddocs %} diff --git a/models/descriptions/label.md b/models/descriptions/label.md new file mode 100644 index 0000000..623a063 --- /dev/null +++ b/models/descriptions/label.md @@ -0,0 +1,5 @@ +{% docs label %} + +The label for this address. + +{% enddocs %} diff --git a/models/descriptions/platform.md b/models/descriptions/platform.md new file mode 100644 index 0000000..ddaa5e1 --- /dev/null +++ b/models/descriptions/platform.md @@ -0,0 +1,5 @@ +{% docs platform %} + +Protocol on which the event (dex swap, nft trade) occurred, where able to determine. + +{% enddocs %} diff --git a/models/descriptions/stats_core.md b/models/descriptions/stats_core.md new file mode 100644 index 0000000..b0c6ba3 --- /dev/null +++ b/models/descriptions/stats_core.md @@ -0,0 +1,71 @@ +{% docs ez_core_metrics_hourly_table_doc %} + +A convenience table that aggregates block and transaction related metrics using various aggregate functions such as SUM, COUNT, MIN and MAX from the fact_transactions table, on an hourly basis. Stats for the current hour will be updated as new data arrives. + +{% enddocs %} + +{% docs block_timestamp_hour %} + +The hour of the timestamp of the block. + +{% enddocs %} + +{% docs block_number_min %} + +The minimum block number in the hour. + +{% enddocs %} + +{% docs block_number_max %} + +The maximum block number in the hour. + +{% enddocs %} + +{% docs block_count %} + +The number of blocks in the hour. + +{% enddocs %} + +{% docs transaction_count %} + +The number of transactions in the hour. + +{% enddocs %} + +{% docs transaction_count_success %} + +The number of successful transactions in the hour. + +{% enddocs %} + +{% docs transaction_count_failed %} + +The number of failed transactions in the hour. + +{% enddocs %} + +{% docs unique_from_count %} + +The number of unique proposer from addresses in the hour. + +{% enddocs %} + +{% docs unique_payer_count %} + +The number of unique payers to addresses in the hour. + +{% enddocs %} + +{% docs total_fees_native %} + +The sum of all fees in the hour, in the native fee currency. + +{% enddocs %} + +{% docs total_fees_usd %} + +The sum of all fees in the hour, in USD. + +{% enddocs %} diff --git a/models/descriptions/table_dim_labels.md b/models/descriptions/table_dim_labels.md new file mode 100644 index 0000000..ac3266c --- /dev/null +++ b/models/descriptions/table_dim_labels.md @@ -0,0 +1,5 @@ +{% docs table_dim_labels %} + +The labels table is a store of one-to-one address identifiers, or an address name. Labels are broken out into a "type" (such as cex, dex, dapp, games, etc.) and a "subtype" (ex: contract_deployer, hot_wallet, token_contract, etc.) in order to help classify each address name into similar groups. Our labels are sourced from many different places, but can primarily be grouped into two categories: automatic and manual. Automatic labels are continuously labeled based on certain criteria, such as a known contract deploying another contract, behavior based algorithms for finding deposit wallets, and consistent data pulls of custom protocol APIs. Manual labels are done periodically to find addresses that cannot be found programmatically such as finding new protocol addresses, centralized exchange hot wallets, or trending addresses. Labels can also be added by our community by using our add-a-label tool (https://science.flipsidecrypto.xyz/add-a-label/) or on-chain with near (https://near.social/lord1.near/widget/Form) and are reviewed by our labels team. A label can be removed by our labels team if it is found to be incorrect or no longer relevant; this generally will only happen for mislabeled deposit wallets. + +{% enddocs %} diff --git a/models/gold/beta/beta__dim_nflad_playoff_rosters.sql b/models/gold/beta/beta__dim_nflad_playoff_rosters.sql new file mode 100644 index 0000000..d4f95cc --- /dev/null +++ b/models/gold/beta/beta__dim_nflad_playoff_rosters.sql @@ -0,0 +1,28 @@ +{{ config( + materialized = 'view' +) }} + +WITH rookies AS ( + + SELECT + *, + 'rookie' AS campaign + FROM + {{ ref('seeds__nflad_playoff_rookies') }} +), +vets AS ( + SELECT + *, + 'vet' AS campaign + FROM + {{ ref('seeds__nflad_playoff_vets') }} +) +SELECT + * +FROM + rookies +UNION ALL +SELECT + * +FROM + vets diff --git a/models/gold/beta/beta__dim_nflad_playoff_rosters.yml b/models/gold/beta/beta__dim_nflad_playoff_rosters.yml new file mode 100644 index 0000000..b25ae3e --- /dev/null +++ b/models/gold/beta/beta__dim_nflad_playoff_rosters.yml @@ -0,0 +1,6 @@ +version: 2 + +models: + - name: beta__dim_nflad_playoff_rosters + description: |- + A temporary table containing official rosters for the Flipside x NFL AllDay Playoff Challenge. diff --git a/models/gold/beta/beta__ez_moment_player_ids.sql b/models/gold/beta/beta__ez_moment_player_ids.sql new file mode 100644 index 0000000..0fd3c13 --- /dev/null +++ b/models/gold/beta/beta__ez_moment_player_ids.sql @@ -0,0 +1,86 @@ +{{ config( + materialized = 'view' +) }} + +WITH espn AS ( + + SELECT + CASE + -- slight name mismatches + WHEN A.full_name = 'Patrick Mahomes' THEN 'Patrick Mahomes II' + WHEN A.full_name = 'Joshua Palmer' THEN 'Josh Palmer' + WHEN A.full_name = 'Gabe Davis' THEN 'Gabriel Davis' + ELSE A.full_name + END AS player, + t.display_name AS team, + A.id AS espn_player_id, + SPLIT( + SPLIT(TRY_PARSE_JSON(A.team) :"$ref", 'http://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2023/teams/') [1], + '?' + ) [0] :: INT AS espn_team_id, + try_parse_json(status):type::string = 'active' as is_active -- note, this may depend on time of data pull from ESPN. Includes IR status. Update as needed. + FROM + {{ ref('beta__dim_nfl_athletes') }} A + LEFT JOIN {{ ref('beta__dim_nfl_teams') }} + t + ON espn_team_id = t.id + WHERE + SPLIT( + SPLIT(TRY_PARSE_JSON(POSITION) :parent :"$ref", 'http://sports.core.api.espn.com/v2/sports/football/leagues/nfl/positions/') [1], + '?') [0] :: INT = 70 -- offense only + ), + allday AS ( + SELECT + nft_id, + nflallday_id, + serial_number, + moment_tier, + CASE + -- Some moments use DJ others D.J. + WHEN player = 'D.J. Moore' THEN 'DJ Moore' + ELSE player + END AS player, + team, + season, + play_type, + moment_stats_full :metadata :playerPosition :: STRING AS POSITION + FROM + {{ ref('nft__dim_allday_metadata') }} + ad + WHERE + classification = 'PLAYER_GAME' + AND season >= 2018 + AND POSITION IN ( + 'QB', + 'WR', + 'RB', + 'TE' + ) + ), + FINAL AS ( + SELECT + nft_id, + nflallday_id, + serial_number, + moment_tier, + ad.player, + ad.team, + season, + play_type, + POSITION, + espn_player_id, + espn_team_id, + is_active + FROM + allday ad + LEFT JOIN espn + ON TRIM(LOWER( + ad.player + )) = TRIM(LOWER( + espn.player + )) + ) + SELECT + * + FROM + FINAL diff --git a/models/gold/core/core__dim_labels.sql b/models/gold/core/core__dim_labels.sql new file mode 100644 index 0000000..0db2f75 --- /dev/null +++ b/models/gold/core/core__dim_labels.sql @@ -0,0 +1,18 @@ +{{ config( + materialized = 'view', + tags = ['scheduled'] +) }} + +SELECT + blockchain, + creator, + address, + address_name, + label_type, + label_subtype, + project_name AS label, + labels_id AS dim_labels_id, + inserted_timestamp, + modified_timestamp +FROM + {{ ref('silver__labels') }} diff --git a/models/gold/core/core__dim_labels.yml b/models/gold/core/core__dim_labels.yml new file mode 100644 index 0000000..b97fdb9 --- /dev/null +++ b/models/gold/core/core__dim_labels.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: core__dim_labels + description: '{{ doc("table_dim_labels") }}' + columns: + - name: BLOCKCHAIN + description: '{{ doc("blockchain") }}' + + - name: CREATOR + description: '{{ doc("creator") }}' + + - name: ADDRESS + description: '{{ doc("address") }}' + + - name: ADDRESS_NAME + description: '{{ doc("address_name") }}' + + - name: LABEL_TYPE + description: '{{ doc("label_type") }}' + + - name: LABEL_SUBTYPE + description: '{{ doc("label_subtype") }}' + + - name: LABEL + description: '{{ doc("label") }}' + + - name: DIM_LABELS_ID + description: "{{doc('id')}}" + + - name: INSERTED_TIMESTAMP + description: "{{doc('inserted_timestamp')}}" + + - name: MODIFIED_TIMESTAMP + description: "{{doc('modified_timestamp')}}" diff --git a/models/gold/defi/defi__ez_swaps.sql b/models/gold/defi/defi__ez_swaps.sql index 48c3460..a682a37 100644 --- a/models/gold/defi/defi__ez_swaps.sql +++ b/models/gold/defi/defi__ez_swaps.sql @@ -1,7 +1,7 @@ {{ config ( materialized = 'view', tags = ['ez', 'scheduled'], - meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }}} + meta ={ 'database_tags':{ 'table':{ 'PURPOSE': 'SWAPS' }} } ) }} WITH chainwalkers AS ( @@ -20,7 +20,6 @@ WITH chainwalkers AS ( token_in_destination, token_in_contract, token_in_amount, - _inserted_timestamp, NULL AS inserted_timestamp, NULL AS modified_timestamp FROM @@ -32,7 +31,7 @@ WITH chainwalkers AS ( ), streamline AS ( SELECT - swaps_id, + swaps_final_id AS swaps_id, tx_id, block_timestamp, block_height, @@ -45,11 +44,10 @@ streamline AS ( token_in_destination, token_in_contract, token_in_amount, - _inserted_timestamp, inserted_timestamp, modified_timestamp FROM - {{ ref('silver__swaps_s') }} + {{ ref('silver__swaps_final') }} WHERE block_height >= {{ var( 'STREAMLINE_START_BLOCK' @@ -83,15 +81,9 @@ SELECT swaps_id, {{ dbt_utils.generate_surrogate_key(['tx_id', 'swap_index']) }} ) AS ez_swaps_id, - COALESCE ( - inserted_timestamp, - _inserted_timestamp - ) AS inserted_timestamp, - COALESCE ( - modified_timestamp, - _inserted_timestamp - ) AS modified_timestamp + COALESCE(inserted_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS inserted_timestamp, + COALESCE(modified_timestamp, '2000-01-01' :: TIMESTAMP_NTZ) AS modified_timestamp FROM FINAL WHERE - token_in_destination IS NOT NULL + token_in_contract IS NOT NULL diff --git a/models/gold/defi/defi__ez_swaps.yml b/models/gold/defi/defi__ez_swaps.yml index 214ec44..fd4f0ca 100644 --- a/models/gold/defi/defi__ez_swaps.yml +++ b/models/gold/defi/defi__ez_swaps.yml @@ -38,8 +38,6 @@ models: - name: TOKEN_OUT_SOURCE description: "{{ doc('token_out_source') }}" - tests: - - not_null - name: TOKEN_OUT_AMOUNT description: "{{ doc('token_out_amount') }}" @@ -53,8 +51,6 @@ models: - name: TOKEN_IN_DESTINATION description: "{{ doc('token_in_destination') }}" - tests: - - not_null - name: TOKEN_IN_AMOUNT description: "{{ doc('token_in_amount') }}" diff --git a/models/gold/nft/nft__dim_allday_metadata.sql b/models/gold/nft/nft__dim_allday_metadata.sql index 0deec7b..d85cbfc 100644 --- a/models/gold/nft/nft__dim_allday_metadata.sql +++ b/models/gold/nft/nft__dim_allday_metadata.sql @@ -106,7 +106,7 @@ SELECT COALESCE ( nft_unique_id, {{ dbt_utils.generate_surrogate_key(['nft_id']) }} - ) AS dim_allday_metadata, + ) AS dim_allday_metadata_id, COALESCE ( inserted_timestamp, _inserted_timestamp diff --git a/models/gold/nft/nft__dim_allday_metadata.yml b/models/gold/nft/nft__dim_allday_metadata.yml index 8899a20..8f5dafa 100644 --- a/models/gold/nft/nft__dim_allday_metadata.yml +++ b/models/gold/nft/nft__dim_allday_metadata.yml @@ -9,56 +9,95 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - nft_id + severity: error + error_if: ">10" + warn_if: "<10" columns: - name: NFT_ID description: "{{ doc('nft_id') }}" tests: - - not_null - - unique + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" + - unique: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR + severity: error + error_if: ">10" + warn_if: "<10" - name: NFT_COLLECTION description: "{{ doc('nft_collection') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR + severity: error + error_if: ">10" + warn_if: "<10" - name: NFLALLDAY_ID description: "{{ doc('nflallday_id') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING - VARCHAR + severity: error + error_if: ">10" + warn_if: "<10" - name: SERIAL_NUMBER description: "{{ doc('serial_number') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER + severity: error + error_if: ">10" + warn_if: "<10" - name: TOTAL_CIRCULATION description: "{{ doc('total_circulation') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - NUMBER + severity: error + error_if: ">10" + warn_if: "<10" - name: MOMENT_DESCRIPTION description: "{{ doc('moment_description') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -67,7 +106,10 @@ models: - name: PLAYER description: "{{ doc('player') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -76,7 +118,10 @@ models: - name: TEAM description: "{{ doc('team') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -85,7 +130,10 @@ models: - name: SEASON description: "{{ doc('season') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -94,7 +142,10 @@ models: - name: WEEK description: "{{ doc('week') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -103,7 +154,10 @@ models: - name: CLASSIFICATION description: "{{ doc('classification') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -112,7 +166,10 @@ models: - name: PLAY_TYPE description: "{{ doc('play_type') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -121,7 +178,10 @@ models: - name: MOMENT_DATE description: "{{ doc('moment_date') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - TIMESTAMP_NTZ @@ -129,7 +189,10 @@ models: - name: SERIES description: "{{ doc('series') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -138,7 +201,10 @@ models: - name: SET_NAME description: "{{ doc('set_name') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - STRING @@ -147,7 +213,10 @@ models: - name: VIDEO_URLS description: "{{ doc('video_urls') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - ARRAY @@ -155,13 +224,16 @@ models: - name: MOMENT_STATS_FULL description: "{{ doc('moment_stats_full') }}" tests: - - not_null + - not_null: + severity: error + error_if: ">10" + warn_if: "<10" - dbt_expectations.expect_column_values_to_be_in_type_list: column_type_list: - OBJECT - VARIANT - - name: DIM_ALLDAY_METADATA + - name: DIM_ALLDAY_METADATA_ID description: "{{ doc('pk_id') }}" - name: INSERTED_TIMESTAMP diff --git a/models/gold/nft/nft__ez_nft_sales.yml b/models/gold/nft/nft__ez_nft_sales.yml index 0386897..ba4f038 100644 --- a/models/gold/nft/nft__ez_nft_sales.yml +++ b/models/gold/nft/nft__ez_nft_sales.yml @@ -13,6 +13,9 @@ models: - buyer - nft_collection - nft_id + severity: error + error_if: ">10" + warn_if: "<10" columns: - name: TX_ID diff --git a/models/gold/price/price__dim_asset_metadata.sql b/models/gold/price/price__dim_asset_metadata.sql index 98d2393..f79ccad 100644 --- a/models/gold/price/price__dim_asset_metadata.sql +++ b/models/gold/price/price__dim_asset_metadata.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tag = ['scheduled'] + tag = ['scheduled'], persist_docs ={ "relation": true, "columns": true } ) }} diff --git a/models/gold/price/price__ez_asset_metadata.sql b/models/gold/price/price__ez_asset_metadata.sql index d771b77..e38a762 100644 --- a/models/gold/price/price__ez_asset_metadata.sql +++ b/models/gold/price/price__ez_asset_metadata.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - tag = ['scheduled'] + tag = ['scheduled'], persist_docs ={ "relation": true, "columns": true } ) }} diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.sql b/models/gold/stats/stats__ez_core_metrics_hourly.sql new file mode 100644 index 0000000..b2c581e --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.sql @@ -0,0 +1,59 @@ +{{ config ( + materialized = 'view', + tags = ['scheduled_non_core'] +) }} + + +WITH txs AS ( + + SELECT + block_timestamp_hour, + transaction_count, + transaction_count_success, + transaction_count_failed, + unique_from_count, + total_fees AS total_fees_native, + LAST_VALUE( + p.price ignore nulls + ) over ( + ORDER BY + block_timestamp_hour rows unbounded preceding + ) AS imputed_close, + core_metrics_hourly_id AS ez_core_metrics_hourly_id, + s.inserted_timestamp AS inserted_timestamp, + s.modified_timestamp AS modified_timestamp + FROM + {{ ref('silver_stats__core_metrics_hourly') }} + s + LEFT JOIN {{ ref('silver__complete_token_prices') }} + p + ON s.block_timestamp_hour = p.hour + AND p.asset_id = 'Flow' +) +SELECT + A.block_timestamp_hour, + A.block_number_min, + A.block_number_max, + A.block_count, + b.transaction_count, + b.transaction_count_success, + b.transaction_count_failed, + b.unique_from_count, + b.total_fees_native, + ROUND( + b.total_fees_native * b.imputed_close, + 2 + ) AS total_fees_usd, + A.core_metrics_block_hourly_id AS ez_core_metrics_hourly_id, + GREATEST( + A.inserted_timestamp, + b.inserted_timestamp + ) AS inserted_timestamp, + GREATEST( + A.modified_timestamp, + b.modified_timestamp + ) AS modified_timestamp +FROM + {{ ref('silver_stats__core_metrics_block_hourly') }} A + JOIN txs b + ON A.block_timestamp_hour = b.block_timestamp_hour \ No newline at end of file diff --git a/models/gold/stats/stats__ez_core_metrics_hourly.yml b/models/gold/stats/stats__ez_core_metrics_hourly.yml new file mode 100644 index 0000000..8b5ac9b --- /dev/null +++ b/models/gold/stats/stats__ez_core_metrics_hourly.yml @@ -0,0 +1,34 @@ +version: 2 +models: + - name: stats__ez_core_metrics_hourly + description: '{{ doc("ez_core_metrics_hourly_table_doc") }}' + + columns: + - name: BLOCK_TIMESTAMP_HOUR + description: '{{ doc("block_timestamp_hour") }}' + - name: BLOCK_NUMBER_MIN + description: '{{ doc("block_number_min") }}' + - name: BLOCK_NUMBER_MAX + description: '{{ doc("block_number_max") }}' + - name: BLOCK_COUNT + description: '{{ doc("block_count") }}' + - name: TRANSACTION_COUNT + description: '{{ doc("transaction_count") }}' + - name: TRANSACTION_COUNT_SUCCESS + description: '{{ doc("transaction_count_success") }}' + - name: TRANSACTION_COUNT_FAILED + description: '{{ doc("transaction_count_failed") }}' + - name: UNIQUE_FROM_COUNT + description: '{{ doc("unique_from_count") }}' + - name: UNIQUE_PAYER_COUNT + description: '{{ doc("unique_payer_count") }}' + - name: TOTAL_FEES_NATIVE + description: '{{ doc("total_fees_native") }}' + - name: TOTAL_FEES_USD + description: '{{ doc("total_fees_usd") }}' + - name: EZ_CORE_METRICS_HOURLY_ID + description: '{{ doc("pk_id") }}' + - name: INSERTED_TIMESTAMP + description: '{{ doc("inserted_timestamp") }}' + - name: MODIFIED_TIMESTAMP + description: '{{ doc("modified_timestamp") }}' diff --git a/models/silver/core/silver__streamline_events.sql b/models/silver/core/silver__streamline_events.sql index 775b3aa..faac045 100644 --- a/models/silver/core/silver__streamline_events.sql +++ b/models/silver/core/silver__streamline_events.sql @@ -80,53 +80,60 @@ attributes AS ( SELECT event_id, OBJECT_AGG( - VALUE :name :: variant, - COALESCE( - VALUE :value :value :fields, - VALUE :value :value :staticType, - VALUE :value :value :value :value :: STRING, - VALUE :value :value :value :: STRING, - VALUE :value :value :: STRING, - 'null' - ) :: variant - ) AS event_data - FROM - flatten_events, - LATERAL FLATTEN ( - COALESCE( - decoded_payload :value :fields :: variant, - event_values :value :fields :: variant - ) + data_key, + IFF(IS_ARRAY(TRY_PARSE_JSON(data_value)) OR IS_OBJECT(TRY_PARSE_JSON(data_value)), PARSE_JSON(data_value)::VARIANT, data_value::VARIANT) + ) AS event_data + FROM + ( + SELECT + event_id, + VALUE :name :: variant AS data_key, + COALESCE( + VALUE :value :value :fields, + VALUE :value :value :staticType, + VALUE :value :value :value :value :: STRING, + VALUE :value :value :value :: STRING, + VALUE :value :value :: STRING, + 'null' + ) AS data_value + FROM + flatten_events, + LATERAL FLATTEN ( + COALESCE( + decoded_payload :value :fields :: variant, + event_values :value :fields :: variant + ) + ) + ) + GROUP BY + 1 + ), + FINAL AS ( + SELECT + e.tx_id, + e.block_height, + e.block_timestamp, + e.event_id, + e.event_index, + e.events_count, + e.payload, + e.event_contract, + e.event_type, + A.event_data, + e.tx_succeeded, + e._inserted_timestamp, + e._partition_by_block_id, + {{ dbt_utils.generate_surrogate_key( + ['event_id'] + ) }} AS streamline_event_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id + FROM + flatten_events e + LEFT JOIN attributes A USING (event_id) ) - GROUP BY - 1 -), -FINAL AS ( SELECT - e.tx_id, - e.block_height, - e.block_timestamp, - e.event_id, - e.event_index, - e.events_count, - e.payload, - e.event_contract, - e.event_type, - A.event_data, - e.tx_succeeded, - e._inserted_timestamp, - e._partition_by_block_id, - {{ dbt_utils.generate_surrogate_key( - ['event_id'] - ) }} AS streamline_event_id, - SYSDATE() AS inserted_timestamp, - SYSDATE() AS modified_timestamp, - '{{ invocation_id }}' AS _invocation_id + * FROM - flatten_events e - LEFT JOIN attributes A USING (event_id) -) -SELECT - * -FROM - FINAL + FINAL diff --git a/models/silver/defi/silver__swaps_aggregator.sql b/models/silver/defi/silver__swaps_aggregator.sql new file mode 100644 index 0000000..835e826 --- /dev/null +++ b/models/silver/defi/silver__swaps_aggregator.sql @@ -0,0 +1,104 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['inserted_timestamp::DATE'], + unique_key = 'swaps_aggregator_id', + tags = ['scheduled_non_core'] +) }} + +WITH events AS ( + + SELECT + block_height, + block_timestamp, + tx_id, + event_index, + event_contract, + event_type, + event_data, + _inserted_timestamp, + modified_timestamp AS _modified_timestamp + FROM + {{ ref('silver__streamline_events') }} + +{% if is_incremental() %} +WHERE + _modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +parse_swap_log AS ( + SELECT + block_height, + block_timestamp, + tx_id, + event_type, + ROW_NUMBER() over ( + PARTITION BY tx_id + ORDER BY + event_index + ) - 1 AS swap_index, + event_data :poolAddress :: STRING AS pool_address, + event_data :poolSource :: STRING AS pool_source, + event_data :tokenInAmount :: FLOAT AS token_in_amount, + event_data :tokenInKey :: STRING AS token_in_contract, + event_data :tokenOutAmount :: FLOAT AS token_out_amount, + event_data :tokenOutKey :: STRING AS token_out_contract + FROM + events + WHERE + event_contract = 'A.e876e00638d54e75.LogEntry' + AND event_type = 'PoolSwapInAggregator' + AND block_height >= 67100587 +), +transactions AS ( + SELECT + tx_id, + authorizers, + modified_timestamp AS _modified_timestamp + FROM + {{ ref('silver__streamline_transactions_final') }} + WHERE + tx_id IN ( + SELECT + DISTINCT tx_id + FROM + parse_swap_log + ) + AND block_height >= 67100587 + +{% if is_incremental() %} +AND _modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +) +SELECT + block_height, + block_timestamp, + s.tx_id, + swap_index, + pool_address, + pool_source, + t.authorizers [0] :: STRING AS trader, + token_in_amount, + token_in_contract, + token_out_amount, + token_out_contract, + {{ dbt_utils.generate_surrogate_key( + ['tx_id','swap_index'] + ) }} AS swaps_aggregator_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS invocation_id +FROM + parse_swap_log s + LEFT JOIN transactions t USING (tx_id) diff --git a/models/silver/defi/silver__swaps_aggregator.yml b/models/silver/defi/silver__swaps_aggregator.yml new file mode 100644 index 0000000..8a3184f --- /dev/null +++ b/models/silver/defi/silver__swaps_aggregator.yml @@ -0,0 +1,119 @@ +version: 2 + +models: + - name: silver__swaps_aggregator + description: |- + This table records asset swaps on the Flow blockchain parsing the log output of swaps through aggregators. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - tx_id + - trader + - swap_index + + columns: + - name: tx_id + description: "{{ doc('tx_id') }}" + tests: + - not_null + + - name: block_timestamp + description: "{{ doc('block_timestamp') }}" + tests: + - not_null + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_NTZ + + - name: block_height + description: "{{ doc('block_height') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: pool_address + description: "{{ doc('swap_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: pool_source + description: "{{ doc('platform') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: swap_index + description: "{{ doc('swap_index') }}" + + - name: trader + description: "{{ doc('trader') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: token_out_amount + description: "{{ doc('token_out_amount') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: token_out_contract + description: "{{ doc('token_out_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: token_in_amount + description: "{{ doc('token_in_amount') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + + - name: token_in_contract + description: "{{ doc('token_in_contract') }}" + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - STRING + - VARCHAR + + - name: _inserted_timestamp + description: "{{ doc('_inserted_timestamp') }}" + + - name: swaps_aggregator_id + description: "{{ doc('pk_id') }}" + + - name: INSERTED_TIMESTAMP + description: "{{ doc('inserted_timestamp') }}" + + - name: MODIFIED_TIMESTAMP + description: "{{ doc('modified_timestamp') }}" + + - name: _INVOCATION_ID + description: "{{ doc('invocation_id') }}" diff --git a/models/silver/swaps/silver__swaps_events_s.sql b/models/silver/defi/silver__swaps_events_s.sql similarity index 100% rename from models/silver/swaps/silver__swaps_events_s.sql rename to models/silver/defi/silver__swaps_events_s.sql diff --git a/models/silver/swaps/silver__swaps_events_s.yml b/models/silver/defi/silver__swaps_events_s.yml similarity index 100% rename from models/silver/swaps/silver__swaps_events_s.yml rename to models/silver/defi/silver__swaps_events_s.yml diff --git a/models/silver/defi/silver__swaps_final.sql b/models/silver/defi/silver__swaps_final.sql new file mode 100644 index 0000000..61fda66 --- /dev/null +++ b/models/silver/defi/silver__swaps_final.sql @@ -0,0 +1,100 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + cluster_by = ['block_timestamp::DATE'], + unique_key = 'swaps_final_id', + tags = ['scheduled_non_core'] +) }} + +WITH swaps_from_aggregator AS ( + + SELECT + block_height, + block_timestamp, + tx_id, + swap_index, + pool_address AS swap_contract, + pool_source AS platform, + trader, + token_in_amount, + token_in_contract, + NULL AS token_in_destination, + token_out_amount, + token_out_contract, + NULL AS token_out_source, + modified_timestamp AS _modified_timestamp, + 0 AS _priority + FROM + {{ ref('silver__swaps_aggregator') }} + +{% if is_incremental() %} +WHERE + _modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} + ) +{% endif %} +), +swaps AS ( + SELECT + block_height, + block_timestamp, + tx_id, + swap_index, + swap_contract, + NULL AS platform, + trader, + token_in_amount, + token_in_contract, + token_in_destination, + token_out_amount, + token_out_contract, + token_out_source, + modified_timestamp AS _modified_timestamp, + 1 AS _priority + FROM + {{ ref('silver__swaps_s') }} + WHERE + tx_id NOT IN ( + SELECT + DISTINCT tx_id + FROM + swaps_from_aggregator + ) + +{% if is_incremental() %} +AND _modified_timestamp >= ( + SELECT + MAX(modified_timestamp) + FROM + {{ this }} +) +{% endif %} +), +swaps_union AS ( + SELECT + * + FROM + swaps_from_aggregator + UNION ALL + SELECT + * + FROM + swaps +) {# Note - curr prices pipeline does not include token address data, making the join difficult and +inaccurate.NEW prices models DO have this so will +ADD + price fields WITH may RELEASE.#} +SELECT + *, + {{ dbt_utils.generate_surrogate_key( + ['tx_id', 'swap_index'] + ) }} AS swaps_final_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + swaps_union diff --git a/models/silver/swaps/silver__swaps_s.sql b/models/silver/defi/silver__swaps_s.sql similarity index 100% rename from models/silver/swaps/silver__swaps_s.sql rename to models/silver/defi/silver__swaps_s.sql diff --git a/models/silver/swaps/silver__swaps_s.yml b/models/silver/defi/silver__swaps_s.yml similarity index 100% rename from models/silver/swaps/silver__swaps_s.yml rename to models/silver/defi/silver__swaps_s.yml diff --git a/models/silver/swaps/silver__swaps_single_trade_s.sql b/models/silver/defi/silver__swaps_single_trade_s.sql similarity index 100% rename from models/silver/swaps/silver__swaps_single_trade_s.sql rename to models/silver/defi/silver__swaps_single_trade_s.sql diff --git a/models/silver/swaps/silver__swaps_single_trade_s.yml b/models/silver/defi/silver__swaps_single_trade_s.yml similarity index 100% rename from models/silver/swaps/silver__swaps_single_trade_s.yml rename to models/silver/defi/silver__swaps_single_trade_s.yml diff --git a/models/silver/labels/silver__labels.sql b/models/silver/labels/silver__labels.sql index 019f112..a4dfdb2 100644 --- a/models/silver/labels/silver__labels.sql +++ b/models/silver/labels/silver__labels.sql @@ -1,7 +1,10 @@ {{ config( - materialized = 'table', - cluster_by = ['address'], - unique_key = 'event_id', + materialized = 'incremental', + unique_key = 'labels_id', + cluster_by = 'modified_timestamp::DATE', + incremental_strategy = 'merge', + merge_exclude_columns = ["inserted_timestamp"], + post_hook = "ALTER TABLE {{ this }} ADD SEARCH OPTIMIZATION ON EQUALITY(address); DELETE FROM {{ this }} WHERE _is_deleted = TRUE;", tags = ['scheduled', 'streamline_scheduled', 'scheduled_non_core'] ) }} @@ -16,14 +19,24 @@ WITH labels AS ( label_subtype, address_name, project_name, - {{ dbt_utils.generate_surrogate_key( - ['blockchain','label_type','label_subtype'] - ) }} AS labels_id, + _is_deleted, + labels_combined_id AS labels_id, SYSDATE() AS inserted_timestamp, SYSDATE() AS modified_timestamp, '{{ invocation_id }}' AS _invocation_id FROM {{ ref('bronze__labels') }} + + {% if is_incremental() %} + WHERE modified_timestamp >= ( + SELECT + MAX( + modified_timestamp + ) + FROM + {{ this }} + ) + {% endif %} ) SELECT * diff --git a/models/silver/nft/metadata/livequery/livequery__allday_moments_metadata_needed.sql b/models/silver/nft/metadata/livequery/allday/livequery__allday_moments_metadata_needed.sql similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__allday_moments_metadata_needed.sql rename to models/silver/nft/metadata/livequery/allday/livequery__allday_moments_metadata_needed.sql diff --git a/models/silver/nft/livequery/silver__allday_moments_metadata_needed_s.sql b/models/silver/nft/metadata/livequery/allday/silver__allday_moments_metadata_needed_s.sql similarity index 53% rename from models/silver/nft/livequery/silver__allday_moments_metadata_needed_s.sql rename to models/silver/nft/metadata/livequery/allday/silver__allday_moments_metadata_needed_s.sql index a8c29e6..06f1bc4 100644 --- a/models/silver/nft/livequery/silver__allday_moments_metadata_needed_s.sql +++ b/models/silver/nft/metadata/livequery/allday/silver__allday_moments_metadata_needed_s.sql @@ -4,8 +4,10 @@ tags = ['livequery', 'allday', 'moment_metadata'], ) }} -{% set table_exists = check_table_exists('SILVER', 'NFT_ALLDAY_METADATA_S') %} - +{% set table_exists = check_table_exists( + 'SILVER', + 'NFT_ALLDAY_METADATA_S' +) %} WITH mints AS ( SELECT @@ -49,9 +51,35 @@ FROM {{ ref('silver__nft_allday_metadata') }} {% if table_exists | trim == '"True"' %} EXCEPT - SELECT - nft_collection AS event_contract, - nft_id AS moment_id - FROM - {{ ref('silver__nft_allday_metadata_s') }} -{% endif %} \ No newline at end of file +SELECT + nft_collection AS event_contract, + nft_id AS moment_id +FROM + {{ ref('silver__nft_allday_metadata_s') }} +EXCEPT +( + WITH try_number AS ( + SELECT + 'A.e4cf4bdc1751c65d.AllDay' AS EVENT_CONTRACT, + FLATTENED.VALUE AS REQUESTED_ID, + COUNT(*) AS REQUEST_COUNT + + FROM + {{ source( + 'bronze_api', + 'allday_metadata' + ) }} AS mf, LATERAL FLATTEN( + input => mf.requested_ids + ) AS flattened + WHERE contract = 'A.e4cf4bdc1751c65d.AllDay' + GROUP BY FLATTENED.VALUE + ) +SELECT + event_contract, + REQUESTED_ID AS moment_id +FROM + try_number +WHERE + request_count > 10 +) +{% endif %} diff --git a/models/silver/nft/metadata/livequery/livequery__null_moments_metadata.sql b/models/silver/nft/metadata/livequery/topshot/livequery__null_moments_metadata.sql similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__null_moments_metadata.sql rename to models/silver/nft/metadata/livequery/topshot/livequery__null_moments_metadata.sql diff --git a/models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.py b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py similarity index 80% rename from models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.py rename to models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py index 20afa54..8775767 100644 --- a/models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.py +++ b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.py @@ -44,7 +44,13 @@ def model(dbt, session): # define params for UDF_API method = "POST" - headers = {"Content-Type": "application/json"} + headers = { + "Accept": "application/json", + "Accept-Encoding": "gzip", + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "Flipside_Flow_metadata/0.1", + } url = topshot_gql_params[0][0] # gql query passed with the post request @@ -52,34 +58,29 @@ def model(dbt, session): # metadata request requires moment_id, defined in a separate view # number of moment_ids to request set by .limit(), timeout experienced at 4000 - inputs = ( dbt.ref("livequery__topshot_moments_metadata_needed") .select("EVENT_CONTRACT", "MOMENT_ID") - .limit(1000) + .limit(600) ) + # Note prior limit of 3500 leads to 429 error / rate limit by system + # Per Dapper team, 50 reqs per 10 seconds. If exceeded, blocked for 30s. # register the udf_construct_data function udf_construct_data = register_udf_construct_data() - try: - requests = F.call_udf( - "flow.live.udf_api", - method, - url, - headers, - udf_construct_data(F.lit(data), F.col("MOMENT_ID")), - ) - - except: - requests = F.lit({"error": F.col("MOMENT_ID")}) - # use with_columns to source moment_id from the input_df and call multiple udf_api calls at once # columns defined in the array will be appended to the input dataframe response = inputs.with_columns( ["DATA", "_INSERTED_DATE", "_INSERTED_TIMESTAMP", "_RES_ID"], [ - requests, + F.call_udf( + "flow.live.udf_api", + method, + url, + headers, + udf_construct_data(F.lit(data), F.col("MOMENT_ID")), + ), F.sysdate().cast(T.DateType()), F.sysdate(), F.md5(F.concat(F.col("EVENT_CONTRACT"), F.col("MOMENT_ID"))), diff --git a/models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.yml b/models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.yml similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__request_topshot_metadata.yml rename to models/silver/nft/metadata/livequery/topshot/livequery__request_topshot_metadata.yml diff --git a/models/silver/nft/metadata/livequery/livequery__topshot_moments_metadata_needed.sql b/models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql similarity index 100% rename from models/silver/nft/metadata/livequery/livequery__topshot_moments_metadata_needed.sql rename to models/silver/nft/metadata/livequery/topshot/livequery__topshot_moments_metadata_needed.sql diff --git a/models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.sql b/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql similarity index 100% rename from models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.sql rename to models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.sql diff --git a/models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.yml b/models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.yml similarity index 100% rename from models/silver/nft/metadata/livequery/silver__nft_topshot_metadata.yml rename to models/silver/nft/metadata/livequery/topshot/silver__nft_topshot_metadata.yml diff --git a/models/silver/nft/metadata/livequery/streamline__null_moments_metadata.sql b/models/silver/nft/metadata/livequery/topshot/streamline__null_moments_metadata.sql similarity index 100% rename from models/silver/nft/metadata/livequery/streamline__null_moments_metadata.sql rename to models/silver/nft/metadata/livequery/topshot/streamline__null_moments_metadata.sql diff --git a/models/silver/nft/silver__nft_allday_metadata_s.sql b/models/silver/nft/silver__nft_allday_metadata_s.sql index b43dadd..75b94e9 100644 --- a/models/silver/nft/silver__nft_allday_metadata_s.sql +++ b/models/silver/nft/silver__nft_allday_metadata_s.sql @@ -37,7 +37,7 @@ FLATTEN_RES AS ( FROM api_call, LATERAL FLATTEN(input => api_call.res:data:data:searchMomentNFTsV2:edges) as flattened_array WHERE api_call.res:status_code = 200 - AND data IS NOT NULL + AND data IS NOT NULL ), FINAL AS ( @@ -73,8 +73,6 @@ FINAL AS ( '{{ invocation_id }}' AS _invocation_id FROM FLATTEN_RES - qualify - row_number() over (partition by nft_allday_metadata_s_id order by _inserted_timestamp desc) = 1 ) SELECT * diff --git a/models/silver/price/silver__complete_native_asset_metadata.sql b/models/silver/price/silver__complete_native_asset_metadata.sql index c3b0eb9..8c7e9a5 100644 --- a/models/silver/price/silver__complete_native_asset_metadata.sql +++ b/models/silver/price/silver__complete_native_asset_metadata.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_native_asset_metadata_id', - tags = ['non_realtime'] + tags = ['scheduled_non_core'] ) }} SELECT diff --git a/models/silver/price/silver__complete_native_prices.sql b/models/silver/price/silver__complete_native_prices.sql index 6ccf4fd..3978dc0 100644 --- a/models/silver/price/silver__complete_native_prices.sql +++ b/models/silver/price/silver__complete_native_prices.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_native_prices_id', - tags = ['non_realtime'] + tags = ['scheduled_non_core'] ) }} SELECT diff --git a/models/silver/price/silver__complete_provider_asset_metadata.sql b/models/silver/price/silver__complete_provider_asset_metadata.sql index a22da44..b9132ed 100644 --- a/models/silver/price/silver__complete_provider_asset_metadata.sql +++ b/models/silver/price/silver__complete_provider_asset_metadata.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_provider_asset_metadata_id', - tags = ['non_realtime'] + tags = ['scheduled_non_core'] ) }} SELECT diff --git a/models/silver/price/silver__complete_provider_prices.sql b/models/silver/price/silver__complete_provider_prices.sql index 60f5593..6fd9cda 100644 --- a/models/silver/price/silver__complete_provider_prices.sql +++ b/models/silver/price/silver__complete_provider_prices.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_provider_prices_id', - tags = ['non_realtime'] + tags = ['scheduled_non_core'] ) }} SELECT diff --git a/models/silver/price/silver__complete_token_asset_metadata.sql b/models/silver/price/silver__complete_token_asset_metadata.sql index 0de57e6..ea05d0c 100644 --- a/models/silver/price/silver__complete_token_asset_metadata.sql +++ b/models/silver/price/silver__complete_token_asset_metadata.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_token_asset_metadata_id', - tags = ['non_realtime'] + tags = ['scheduled_non_core'] ) }} SELECT diff --git a/models/silver/price/silver__complete_token_prices.sql b/models/silver/price/silver__complete_token_prices.sql index 98b3f62..a2b4655 100644 --- a/models/silver/price/silver__complete_token_prices.sql +++ b/models/silver/price/silver__complete_token_prices.sql @@ -2,7 +2,7 @@ materialized = 'incremental', incremental_strategy = 'delete+insert', unique_key = 'complete_token_prices_id', - tags = ['non_realtime'] + tags = ['scheduled_non_core'] ) }} SELECT diff --git a/models/silver/price/silver__prices_swaps_s.sql b/models/silver/price/silver__prices_swaps_s.sql index c736691..a59af63 100644 --- a/models/silver/price/silver__prices_swaps_s.sql +++ b/models/silver/price/silver__prices_swaps_s.sql @@ -37,7 +37,9 @@ flow_price AS ( recorded_hour as _timestamp, open as price_usd FROM - {{ ref('silver__prices_hourly') }} + {{ ref('silver__complete_token_prices') }} + WHERE + token = 'Flow' ), stable_out AS ( SELECT diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.sql b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql new file mode 100644 index 0000000..30bafef --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.sql @@ -0,0 +1,57 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['curated', 'scheduled_non_core'] +) }} +/* run incremental timestamp value first then use it as a static value */ +{% if execute %} + +{% if is_incremental() %} +{% set query %} + +SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour +FROM + {{ ref('core__fact_blocks') }} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) - INTERVAL '2 hours' + FROM + {{ this }} + ) {% endset %} + {% set min_block_timestamp_hour = run_query(query).columns [0].values() [0] %} +{% endif %} +{% endif %} +SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + MIN(block_height) :: INT AS block_number_min, + MAX(block_height) :: INT AS block_number_max, + COUNT( + 1 + ) AS block_count, + MAX(inserted_timestamp) AS _inserted_timestamp, + {{ dbt_utils.generate_surrogate_key( + ['block_timestamp_hour'] + ) }} AS core_metrics_block_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + {{ ref('core__fact_blocks') }} +WHERE + block_timestamp_hour < DATE_TRUNC('hour', systimestamp()) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + block_timestamp +) >= '{{ min_block_timestamp_hour }}' +{% endif %} +GROUP BY + 1 diff --git a/models/silver/stats/silver_stats__core_metrics_block_hourly.yml b/models/silver/stats/silver_stats__core_metrics_block_hourly.yml new file mode 100644 index 0000000..8d91718 --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_block_hourly.yml @@ -0,0 +1,41 @@ +version: 2 +models: + - name: silver_stats__core_metrics_block_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + columns: + - name: BLOCK_TIMESTAMP_HOUR + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: BLOCK_NUMBER_MIN + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_NUMBER_MAX + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: BLOCK_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: _INSERTED_TIMESTAMP + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.sql b/models/silver/stats/silver_stats__core_metrics_hourly.sql new file mode 100644 index 0000000..fd1085c --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.sql @@ -0,0 +1,116 @@ +{{ config( + materialized = 'incremental', + incremental_strategy = 'delete+insert', + unique_key = "block_timestamp_hour", + cluster_by = ['block_timestamp_hour::DATE'], + tags = ['curated', 'scheduled_non_core'] +) }} +/* run incremental timestamp value first then use it as a static value */ +{% if execute %} + +{% if is_incremental() %} +{% set query %} + +SELECT + MIN(DATE_TRUNC('hour', block_timestamp)) block_timestamp_hour +FROM + {{ ref('core__fact_transactions') }} +WHERE + modified_timestamp >= ( + SELECT + MAX(modified_timestamp) - INTERVAL '2 hours' + FROM + {{ this }} + ) {% endset %} + {% set min_block_timestamp_hour = run_query(query).columns [0].values() [0] %} +{% endif %} +{% endif %} + +WITH fees AS ( + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + SUM( + event_data :amount :: FLOAT + ) AS total_fees + FROM + {{ ref('core__fact_events') }} + -- TODO: change this to silver when the backfill is done + WHERE + event_type = 'FeesDeducted' + AND block_timestamp_hour < DATE_TRUNC( + 'hour', + CURRENT_TIMESTAMP + ) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + block_timestamp +) >= '{{ min_block_timestamp_hour }}' +{% endif %} +GROUP BY + 1 +), +transactions AS ( + SELECT + DATE_TRUNC( + 'hour', + block_timestamp + ) AS block_timestamp_hour, + COUNT( + DISTINCT tx_id + ) AS transaction_count, + COUNT( + DISTINCT CASE + WHEN tx_succeeded THEN tx_id + END + ) AS transaction_count_success, + COUNT( + DISTINCT CASE + WHEN NOT tx_succeeded THEN tx_id + END + ) AS transaction_count_failed, + COUNT( + DISTINCT proposer + ) AS unique_from_count, + COUNT( + payer + ) AS unique_payer_count, + MAX(inserted_timestamp) AS _inserted_timestamp + FROM + {{ ref('core__fact_transactions') }} AS tx + WHERE + block_timestamp_hour < DATE_TRUNC( + 'hour', + CURRENT_TIMESTAMP + ) + +{% if is_incremental() %} +AND DATE_TRUNC( + 'hour', + block_timestamp +) >= '{{ min_block_timestamp_hour }}' +{% endif %} +GROUP BY + 1 +) +SELECT + tx.*, + COALESCE( + total_fees, + 0 + ) AS total_fees, + -- As we are missing data, we miss the fee events. We need to coalesce to 0 + {{ dbt_utils.generate_surrogate_key( + ['tx.block_timestamp_hour'] + ) }} AS core_metrics_hourly_id, + SYSDATE() AS inserted_timestamp, + SYSDATE() AS modified_timestamp, + '{{ invocation_id }}' AS _invocation_id +FROM + transactions AS tx + LEFT JOIN fees + ON tx.block_timestamp_hour = fees.block_timestamp_hour diff --git a/models/silver/stats/silver_stats__core_metrics_hourly.yml b/models/silver/stats/silver_stats__core_metrics_hourly.yml new file mode 100644 index 0000000..68d716c --- /dev/null +++ b/models/silver/stats/silver_stats__core_metrics_hourly.yml @@ -0,0 +1,56 @@ +version: 2 +models: + - name: silver_stats__core_metrics_hourly + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - BLOCK_TIMESTAMP_HOUR + columns: + - name: BLOCK_TIMESTAMP_HOUR + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - TIMESTAMP_LTZ + - TIMESTAMP_NTZ + - name: TRANSACTION_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_SUCCESS + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TRANSACTION_COUNT_FAILED + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: UNIQUE_FROM_COUNT + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - NUMBER + - FLOAT + - name: TOTAL_FEES + tests: + - not_null + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: + - DECIMAL + - FLOAT + - NUMBER + - name: _INSERTED_TIMESTAMP + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 1 diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql index 46620ab..91c8cf8 100644 --- a/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_transaction_results_history.sql @@ -2,11 +2,29 @@ materialized = 'view' ) }} -{% set +{# + Full array to table names, keeping commented for posterity. + {% set table_names = [ 'TRANSACTION_RESULTS_CANDIDATE_07', 'TRANSACTION_RESULTS_CANDIDATE_08', 'TRANSACTION_RESULTS_CANDIDATE_09', 'TRANSACTION_RESULTS_MAINNET_01', 'TRANSACTION_RESULTS_MAINNET_02', 'TRANSACTION_RESULTS_MAINNET_03', 'TRANSACTION_RESULTS_MAINNET_04', 'TRANSACTION_RESULTS_MAINNET_05', 'TRANSACTION_RESULTS_MAINNET_06', 'TRANSACTION_RESULTS_MAINNET_07', 'TRANSACTION_RESULTS_MAINNET_08', 'TRANSACTION_RESULTS_MAINNET_09', 'TRANSACTION_RESULTS_MAINNET_10', 'TRANSACTION_RESULTS_MAINNET_11', 'TRANSACTION_RESULTS_MAINNET_12', 'TRANSACTION_RESULTS_MAINNET_13', 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_20', 'TRANSACTION_RESULTS_MAINNET_21', 'TRANSACTION_RESULTS_MAINNET_22' ] +%} #} +{# + TODO - below array includes NVs with 99.99% coverage. For example Mainnet 5 is missing 532 txs. + These are edge cases to be investigated after the final sporks get to near-complete status. +{% set + table_names = + [ + 'TRANSACTION_RESULTS_MAINNET_05', 'TRANSACTION_RESULTS_MAINNET_06', 'TRANSACTION_RESULTS_MAINNET_09', 'TRANSACTION_RESULTS_MAINNET_10', 'TRANSACTION_RESULTS_MAINNET_11', 'TRANSACTION_RESULTS_MAINNET_12', 'TRANSACTION_RESULTS_MAINNET_13', 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_20', 'TRANSACTION_RESULTS_MAINNET_22' + ] +%} #} + +{% set + table_names = + [ + 'TRANSACTION_RESULTS_MAINNET_14', 'TRANSACTION_RESULTS_MAINNET_15', 'TRANSACTION_RESULTS_MAINNET_16', 'TRANSACTION_RESULTS_MAINNET_17', 'TRANSACTION_RESULTS_MAINNET_18', 'TRANSACTION_RESULTS_MAINNET_19', 'TRANSACTION_RESULTS_MAINNET_22' + ] %} {{ streamline_multiple_external_table_query( diff --git a/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql b/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql index ff0a376..1094e58 100644 --- a/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql +++ b/models/silver/streamline/bronze/core/history/bronze__streamline_transactions_history.sql @@ -1,12 +1,29 @@ {{ config ( materialized = 'view' ) }} - +{# + Full array to table names, keeping commented for posterity. {% set table_names = [ 'TRANSACTIONS_CANDIDATE_07', 'TRANSACTIONS_CANDIDATE_08', 'TRANSACTIONS_CANDIDATE_09', 'TRANSACTIONS_MAINNET_01', 'TRANSACTIONS_MAINNET_02', 'TRANSACTIONS_MAINNET_03', 'TRANSACTIONS_MAINNET_04', 'TRANSACTIONS_MAINNET_05', 'TRANSACTIONS_MAINNET_06', 'TRANSACTIONS_MAINNET_07', 'TRANSACTIONS_MAINNET_08', 'TRANSACTIONS_MAINNET_09', 'TRANSACTIONS_MAINNET_10', 'TRANSACTIONS_MAINNET_11', 'TRANSACTIONS_MAINNET_12', 'TRANSACTIONS_MAINNET_13', 'TRANSACTIONS_MAINNET_14', 'TRANSACTIONS_MAINNET_15', 'TRANSACTIONS_MAINNET_16', 'TRANSACTIONS_MAINNET_17', 'TRANSACTIONS_MAINNET_18', 'TRANSACTIONS_MAINNET_19', 'TRANSACTIONS_MAINNET_20', 'TRANSACTIONS_MAINNET_21', 'TRANSACTIONS_MAINNET_22' ] +%} #} +{# + TODO - below array includes NVs with 99.99% coverage. For example Mainnet 5 is missing 532 txs. + These are edge cases to be investigated after the final sporks get to near-complete status. +{% set + table_names = + [ + 'TRANSACTIONS_MAINNET_05', 'TRANSACTIONS_MAINNET_06', 'TRANSACTIONS_MAINNET_09', 'TRANSACTIONS_MAINNET_10', 'TRANSACTIONS_MAINNET_11', 'TRANSACTIONS_MAINNET_12', 'TRANSACTIONS_MAINNET_13', 'TRANSACTIONS_MAINNET_14', 'TRANSACTIONS_MAINNET_15', 'TRANSACTIONS_MAINNET_16', 'TRANSACTIONS_MAINNET_17', 'TRANSACTIONS_MAINNET_18', 'TRANSACTIONS_MAINNET_19' + ] +%} #} + +{% set + table_names = + [ + 'TRANSACTIONS_MAINNET_18', 'TRANSACTIONS_MAINNET_19' + ] %} {{ streamline_multiple_external_table_query( diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql new file mode 100644 index 0000000..17be0eb --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_17.sql @@ -0,0 +1,59 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet17.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_17', 'sql_limit', '750000', 'producer_batch_size', '50000', 'worker_batch_size', '500', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '800'))", + target = "streamline.{{this.identifier}}" + ) + ) +}} + +WITH blocks AS ( +-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 14 + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 27341470 + AND 31735954 + EXCEPT + SELECT + distinct block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results_history") }} + WHERE + block_number BETWEEN 27341470 + AND 31735954 +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 27341470 + AND 31735954 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_results_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql new file mode 100644 index 0000000..3a1a6f8 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_18.sql @@ -0,0 +1,59 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet18.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_18', 'sql_limit', '2000000', 'producer_batch_size', '30000', 'worker_batch_size', '300', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '770'))", + target = "streamline.{{this.identifier}}" + ) + ) +}} + +WITH blocks AS ( +-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 31735955 + AND 35858810 + EXCEPT + SELECT + distinct block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results_history") }} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_results_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql new file mode 100644 index 0000000..953abb1 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_19.sql @@ -0,0 +1,60 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet19.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_19', 'sql_limit', '5000000', 'producer_batch_size', '30000', 'worker_batch_size', '300', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '750'))", + target = "streamline.{{this.identifier}}" + ) + ) +}} + + +WITH blocks AS ( +-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 35858811 + AND 40171633 + EXCEPT + SELECT + distinct block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results_history") }} + WHERE + block_number BETWEEN 35858811 + AND 40171633 +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 35858811 + AND 40171633 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_results_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql new file mode 100644 index 0000000..5d3f915 --- /dev/null +++ b/models/silver/streamline/core/history/transaction_results/batch/streamline__get_batch_transaction_results_history_mainnet_22.sql @@ -0,0 +1,59 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet22.nodes.onflow.org:9000', 'external_table', 'transaction_results_mainnet_22', 'sql_limit', '750000', 'producer_batch_size', '50000', 'worker_batch_size', '500', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '800'))", + target = "streamline.{{this.identifier}}" + ) + ) +}} + +WITH blocks AS ( +-- CTE to identify blocks that doesn't have tx_results ingested for mainnet 18 + SELECT + block_height + FROM + {{ ref("streamline__blocks") }} + WHERE + block_height BETWEEN 47169687 + AND 55114466 + EXCEPT + SELECT + distinct block_number AS block_height + FROM + {{ ref("streamline__complete_get_transaction_results_history") }} + WHERE + block_number BETWEEN 47169687 + AND 55114466 +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 47169687 + AND 55114466 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transaction_results_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql index 3e3861a..b698fcb 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_14.sql @@ -1,8 +1,8 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_14', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" + func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet14.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_14', 'sql_limit', '750000', 'producer_batch_size','50000', 'worker_batch_size','500', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "streamline.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql index 52120e3..557ac97 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_15.sql @@ -1,8 +1,8 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_15', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" + func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet15.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_15','sql_limit', '750000', 'producer_batch_size','50000', 'worker_batch_size','500', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "streamline.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql index 0f43df6..b30f08c 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_16.sql @@ -1,8 +1,8 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_16', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" + func = "streamline.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet16.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_16', 'sql_limit', '3000000', 'producer_batch_size','30000', 'worker_batch_size','300', 'batch_call_limit', {{var('batch_call_limit','1')}}))", + target = "streamline.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql index 718c4d9..2ea397a 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_17.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_17', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet17.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_17', 'sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql index 14032bc..3a2e2e0 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_18.sql @@ -1,8 +1,17 @@ {{ config ( materialized = "view", - post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", - target = "{{this.schema}}.{{this.identifier}}" + + post_hook = fsc_utils.if_data_call_function_v2( + func = 'udf_bulk_grpc_us_east_2', + target = "{{this.schema}}.{{this.identifier}}", + params = { + "node_url":"access-001.mainnet18.nodes.onflow.org:9000", + "external_table": "transaction_results_mainnet_18", + "sql_limit": "25000", + "producer_batch_size": "1000", + "worker_batch_size": "200", + "sql_source": "{{this.identifier}}" + } ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql index 9de1992..e0b9b8f 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_19.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_19','sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql index b3bf9ab..7220632 100644 --- a/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql +++ b/models/silver/streamline/core/history/transaction_results/streamline__get_transaction_results_history_mainnet_22.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_22', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet22.nodes.onflow.org:9000','external_table', 'transaction_results_mainnet_22', 'sql_limit', '6000000', 'producer_batch_size','1000', 'worker_batch_size','2', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql new file mode 100644 index 0000000..dcb2c35 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_18.sql @@ -0,0 +1,68 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet18.nodes.onflow.org:9000', 'external_table', 'transactions_mainnet_18', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '850'))", + target = "streamline.{{this.identifier}}" + ) + ) +}} + +WITH collection_transactions AS ( + + SELECT + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id + FROM + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 31735955 + AND 35858810 +), +-- CTE to identify transactions that haven't been ingested yet +blocks AS ( + SELECT + distinct(block_height) + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 31735955 + AND 35858810 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transactions_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql new file mode 100644 index 0000000..ab51d29 --- /dev/null +++ b/models/silver/streamline/core/history/transactions/batch/streamline__get_batch_transactions_mainnet_19.sql @@ -0,0 +1,68 @@ +{{ config ( + materialized = "view", + post_hook = if_data_call_function( + func = "streamline.udf_bulk_grpc(object_construct('node_url','access-001.mainnet19.nodes.onflow.org:9000', 'external_table', 'transactions_mainnet_19', 'sql_limit', '188000', 'producer_batch_size', '14000', 'worker_batch_size', '100', 'sql_source', '{{this.identifier}}', 'concurrent_requests', '800'))", + target = "streamline.{{this.identifier}}" + ) + ) +}} + +WITH collection_transactions AS ( + + SELECT + block_number AS block_height, + TRANSACTION.value :: STRING AS transaction_id + FROM + {{ ref('streamline__complete_get_collections_history') }} + cch, + LATERAL FLATTEN( + input => cch.data :transaction_ids + ) AS TRANSACTION + WHERE + block_height BETWEEN 35858811 + AND 40171633 +), +-- CTE to identify transactions that haven't been ingested yet +blocks AS ( + SELECT + distinct(block_height) + FROM + collection_transactions ct + LEFT JOIN {{ ref("streamline__complete_get_transactions_history") }} + t + ON ct.transaction_id = t.id + WHERE + t.id IS NULL +), +block_ids AS ( +-- CTE to generate the block_ids for the missing block transactions + SELECT + data:id::STRING as block_id, + block_number + FROM + {{ ref("streamline__complete_get_blocks_history")}} + WHERE + block_number BETWEEN 35858811 + AND 40171633 +) +-- Generate the requests based on the missing block transactions +SELECT + OBJECT_CONSTRUCT( + 'grpc', + 'proto3', + 'method', + 'get_transactions_by_block_i_d', + 'block_height', + block_height :: INTEGER, + 'method_params', + OBJECT_CONSTRUCT( + 'block_id', + block_id + ) + ) AS request +FROM + blocks +JOIN + block_ids on blocks.block_height = block_ids.block_number +ORDER BY + block_height ASC diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql index e368e63..a9883d3 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_18.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transactions_mainnet_18', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet18.nodes.onflow.org:9000','external_table', 'transactions_mainnet_18', 'sql_limit', '660000', 'producer_batch_size','30000', 'worker_batch_size', '300', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql index ca42f2d..60ce12c 100644 --- a/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql +++ b/models/silver/streamline/core/history/transactions/streamline__get_transactions_history_mainnet_19.sql @@ -1,7 +1,7 @@ {{ config ( materialized = "view", post_hook = if_data_call_function( - func = "{{this.schema}}.udf_bulk_grpc_us_east_2(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transactions_mainnet_19', 'sql_limit', {{var('sql_limit','500000')}}, 'producer_batch_size', {{var('producer_batch_size','10000')}}, 'worker_batch_size', {{var('worker_batch_size','1000')}}, 'batch_call_limit', {{var('batch_call_limit','1')}}))", + func = "{{this.schema}}.udf_bulk_grpc(object_construct('sql_source', '{{this.identifier}}','node_url','access-001.mainnet19.nodes.onflow.org:9000','external_table', 'transactions_mainnet_19', 'sql_limit', '660000', 'producer_batch_size','30000', 'worker_batch_size', '300', 'batch_call_limit', {{var('batch_call_limit','1')}}))", target = "{{this.schema}}.{{this.identifier}}" ) ) }} diff --git a/models/silver_cw/price/silver__prices_swaps.sql b/models/silver_cw/price/silver__prices_swaps.sql index 25309ff..71bac6b 100644 --- a/models/silver_cw/price/silver__prices_swaps.sql +++ b/models/silver_cw/price/silver__prices_swaps.sql @@ -37,7 +37,7 @@ flow_price AS ( recorded_hour as _timestamp, open as price_usd FROM - {{ ref('silver__prices_hourly') }} + {{ ref('silver__complete_token_prices') }} WHERE token = 'Flow' ), diff --git a/models/sources.yml b/models/sources.yml index 344f18a..61f66d0 100644 --- a/models/sources.yml +++ b/models/sources.yml @@ -8,11 +8,11 @@ sources: - name: flow_blocks - name: flow_txs - - name: crosschain_core + - name: crosschain_silver database: crosschain - schema: core + schema: silver tables: - - name: dim_labels + - name: labels_combined - name: bronze_streamline database: streamline diff --git a/package-lock.yml b/package-lock.yml new file mode 100644 index 0000000..2de10b9 --- /dev/null +++ b/package-lock.yml @@ -0,0 +1,16 @@ +packages: +- package: calogica/dbt_expectations + version: 0.8.0 +- package: dbt-labs/dbt_external_tables + version: 0.8.0 +- package: dbt-labs/dbt_utils + version: 1.0.0 +- git: https://github.com/FlipsideCrypto/fsc-utils.git + revision: ec7703e90ad70dfd997abe57717a07400aae2f17 +- package: get-select/dbt_snowflake_query_tags + version: 2.3.3 +- package: calogica/dbt_date + version: 0.7.2 +- git: https://github.com/FlipsideCrypto/livequery-models.git + revision: bca494102fbd2d621d32746e9a7fe780678044f8 +sha1_hash: bb079c5efcbe9e6e5f17498d0c787df276e58f67 diff --git a/packages.yml b/packages.yml index 97f06d3..5c945f5 100644 --- a/packages.yml +++ b/packages.yml @@ -6,6 +6,6 @@ packages: - package: dbt-labs/dbt_utils version: 1.0.0 - git: https://github.com/FlipsideCrypto/fsc-utils.git - revision: "v1.9.3" + revision: "v1.17.2" - package: get-select/dbt_snowflake_query_tags version: [">=2.0.0", "<3.0.0"] diff --git a/profiles.yml b/profiles.yml index 8c2adbc..96d8884 100644 --- a/profiles.yml +++ b/profiles.yml @@ -3,17 +3,27 @@ flow: outputs: dev: type: snowflake - account: "{{ env_var('SF_ACCOUNT') }}" - # User/password auth - user: "{{ env_var('SF_USERNAME') }}" + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" authenticator: externalbrowser - role: "{{ env_var('SF_ROLE') }}" - schema: "{{ env_var('SF_SCHEMA') }}" - region: "{{ env_var('SF_REGION') }}" - database: "{{ env_var('SF_DATABASE') }}" - warehouse: "{{ env_var('SF_WAREHOUSE') }}" - threads: 4 + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 + client_session_keep_alive: False + prod: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + user: "{{ env_var('USER') }}" + authenticator: externalbrowser + role: "{{ env_var('ROLE') }}" + schema: "{{ env_var('SCHEMA') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + threads: 8 client_session_keep_alive: False - query_tag: flow_curator config: send_anonymous_usage_stats: False diff --git a/python/espn/README.md b/python/espn/README.md new file mode 100644 index 0000000..2eb9b7a --- /dev/null +++ b/python/espn/README.md @@ -0,0 +1,16 @@ +# Log ESPN Player IDs +Python script to query data from the ESPN API and store it in Snowflake for prod use. Presently, the player endpoint is what we will likely need to run again before the season starts. + +The teams endpoint can be set by uncommenting lines 115-122 and commenting out 124-132. The target table is presently set in-line in the script. + + +## .env +Script will load the following config variables to create a Snowflake connection. Write access to prod db is required, as the data ingested from the ESPN API is uploaded to a bronze table. +``` +USERNAME=@flipsidecrypto.com +ROLE= +ACCOUNT= +WAREHOUSE=<> +DATABASE=<> +SCHEMA=<> +``` diff --git a/python/espn/get_espn.py b/python/espn/get_espn.py new file mode 100644 index 0000000..71355f9 --- /dev/null +++ b/python/espn/get_espn.py @@ -0,0 +1,141 @@ +import snowflake +import pandas as pd +import requests as res +import re +import json +import os +from datetime import datetime, timezone +from dotenv import load_dotenv +from snowflake.connector.pandas_tools import write_pandas + + +def upload_to_snowflake(conn, data, table_name): + + # Convert the data to a pandas DataFrame + df = pd.DataFrame(data) + df.columns = [camel_to_snake(col) for col in df.columns] + + # Add a TIMESTAMP column + df['_INSERTED_TIMESTAMP'] = datetime.now(timezone.utc) + + df.rename(columns={'$REF': 'REF'}, inplace=True) + + # Map pandas data types to Snowflake data types + dtype_mapping = { + 'object': 'STRING', + 'int64': 'NUMBER', + 'float64': 'FLOAT', + 'datetime64[ns]': 'FLOAT', # loaded as unix timestamp. Can cast in SF using :: TIMESTAMP_NTZ + 'bool': 'BOOLEAN', + 'dict': 'VARIANT', + # Add more mappings if needed + } + + # Generate the CREATE TABLE statement + # TODO should change this to IF NOT EXISTS ? Or leave as-is because future pulls overwrite instead of append + create_table_stmt = 'CREATE OR REPLACE TABLE {} (\n'.format(table_name) + for col, dtype in df.dtypes.items(): + snowflake_dtype = dtype_mapping.get(str(dtype), 'STRING') + create_table_stmt += ' "{}" {},\n'.format(col, snowflake_dtype) + create_table_stmt = create_table_stmt.rstrip(',\n') + '\n);' + + # Execute the CREATE TABLE statement + conn.cursor().execute(create_table_stmt) + + # Upload the DataFrame to Snowflake + write_pandas(conn, df, table_name) + + +def get_espn(base_url, params): + # paginate through endpoint until no items returned to get urls for each entity + response = [] + page = 1 + while True: + r = res.get(f"{base_url}?page={page}", params=params) + if len(r.json()['items']) == 0: + break + response.extend(r.json()['items']) + page += 1 + + # log completion of url request + print(f"Retrieved {len(response)} items from {base_url}") + + data = [] + errors = [] + for i, url in enumerate(response): + try: + r = res.get(url['$ref']) + + # if success, append to data + if r.status_code == 200: + data.append(r.json()) + else: + print(f"Error getting {url['$ref']}: {r.status_code}") + # append error to list of errors + errors.append(url['$ref']) + + # log every 100 successful requests + if i % 100 == 0: + print(f"Retrieved {i} entities") + + except Exception as e: + print(f"Error getting {url['$ref']}: {e}") + # append error to list of errors + errors.append(url['$ref']) + + + # if any errors, write to file + if len(errors) > 0: + with open('errors.txt', 'w') as f: + f.write('\n'.join(errors)) + + return data + + +def camel_to_snake(name): + # Convert camel case to snake case for column naming + + name = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).upper() + + +def main(): + load_dotenv() + # Establish connection to Snowflake + conn = snowflake.connector.connect( + user=os.getenv('USERNAME'), + authenticator= "externalbrowser", + account=os.getenv('ACCOUNT'), + warehouse=os.getenv('WAREHOUSE'), + database=os.getenv('DATABASE'), + schema=os.getenv('SCHEMA'), + role=os.getenv('ROLE'), + ) + + # teams_url = "https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/seasons/2023/teams" + # params = { + # "limit": 50 + # } + # table_name = "ESPN_NFL_TEAMS" + + # data = get_espn(teams_url, params) + # upload_to_snowflake(conn, data, table_name) + + athletes_url = "https://sports.core.api.espn.com/v2/sports/football/leagues/nfl/athletes" + params = { + "limit": 1000, + "active": "true" + } + table_name = "ESPN_NFL_ATHLETES" + + data = get_espn(athletes_url, params) + upload_to_snowflake(conn, data, table_name) + + # Close the connection + conn.close() + + return True + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 5464821..aacfb7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -dbt-snowflake~=1.4 +dbt-snowflake>=1.7,<1.8 +protobuf==4.25.3 diff --git a/tests/tests__block_tx_count_recency.sql b/tests/tests__block_tx_count_recency.sql index e37eecd..4e383ba 100644 --- a/tests/tests__block_tx_count_recency.sql +++ b/tests/tests__block_tx_count_recency.sql @@ -1,6 +1,7 @@ {{ config( severity = 'error', - tags = ['observability'] + tags = ['observability'], + enabled = False ) }} WITH check_lag AS (