flip/solana-models
1
0
Fork 0
mirror of https://github.com/FlipsideCrypto/solana-models.git synced 2026-02-06 11:27:00 +00:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

An 4090/force paralell execution (#402)

Browse Source 
* force paralell execution for each udf request

* increase number of commands per workflow, decrease schedule frequency
This commit is contained in:
desmond-hui 2023-11-15 09:51:03 -08:00 committed by GitHub
parent b2f31fc4c9
commit d32599e851
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/dbt_run_get_nft_compressed_backfill.yml vendored
View File

@ -41,3 +41,12 @@ jobs:
- name: Run DBT Jobs
run: |
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql

2
data/github_actions__workflows.csv
View File

@ -1,5 +1,5 @@
workflow_name,workflow_schedule
dbt_run_get_nft_compressed_backfill,"*/5 * * * *"
dbt_run_get_nft_compressed_backfill,"*/10 * * * *"
dbt_run_incremental,"1,16,31,46 * * * *"
dbt_run_incremental_non_core,"25,50 * * * *"
dbt_test_tasks,"0,30 * * * *"

1 workflow_name workflow_schedule
2 dbt_run_get_nft_compressed_backfill */5 * * * * */10 * * * *
3 dbt_run_incremental 1,16,31,46 * * * *
4 dbt_run_incremental_non_core 25,50 * * * *
5 dbt_test_tasks 0,30 * * * *

6
macros/streamline/bulk_parse_compressed_nft_mints/udf_bulk_parse_compressed_nft_mints.sql
View File

@ -1,6 +1,10 @@
{% macro udf_bulk_parse_compressed_nft_mints() %}
CREATE
OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_parse_compressed_nft_mints("JSON" ARRAY) returns ARRAY api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%}
OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_parse_compressed_nft_mints("JSON" ARRAY)
returns ARRAY
api_integration = aws_solana_api_dev
max_batch_rows = 1
AS {% if target.database == 'SOLANA' -%}
'https://cpzzn7ohu0.execute-api.us-east-1.amazonaws.com/prod/parse'
{% else %}
'https://rd7pddtgl9.execute-api.us-east-1.amazonaws.com/dev/parse'

9
models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql
View File

@ -92,7 +92,6 @@
-- AND e.block_timestamp :: DATE = C.block_timestamp :: DATE
-- AND ii_program_id = 'noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV'
-- )
{% for i in range(1,6) %}
SELECT
ARRAY_AGG(request) AS batch_request,
streamline.udf_bulk_parse_compressed_nft_mints(batch_request) AS responses,
@ -106,10 +105,6 @@ SELECT
FROM
{{ source('bronze_api_prod','parse_compressed_nft_mints_requests') }}
WHERE
gn = {{ min_gn }}+{{ i }}
gn between {{ min_gn }}+1 and {{ min_gn }}+10
GROUP BY
gn
{% if not loop.last %}
UNION ALL
{% endif %}
{% endfor %}
gn