diff --git a/.github/workflows/dbt_run_get_nft_compressed_backfill.yml b/.github/workflows/dbt_run_get_nft_compressed_backfill.yml index b8c98e5c..5829ef59 100644 --- a/.github/workflows/dbt_run_get_nft_compressed_backfill.yml +++ b/.github/workflows/dbt_run_get_nft_compressed_backfill.yml @@ -41,3 +41,12 @@ jobs: - name: Run DBT Jobs run: | dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql + dbt run -s models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql \ No newline at end of file diff --git a/data/github_actions__workflows.csv b/data/github_actions__workflows.csv index f4327b6a..e280f4d4 100644 --- a/data/github_actions__workflows.csv +++ b/data/github_actions__workflows.csv @@ -1,5 +1,5 @@ workflow_name,workflow_schedule -dbt_run_get_nft_compressed_backfill,"*/5 * * * *" +dbt_run_get_nft_compressed_backfill,"*/10 * * * *" dbt_run_incremental,"1,16,31,46 * * * *" dbt_run_incremental_non_core,"25,50 * * * *" dbt_test_tasks,"0,30 * * * *" diff --git a/macros/streamline/bulk_parse_compressed_nft_mints/udf_bulk_parse_compressed_nft_mints.sql b/macros/streamline/bulk_parse_compressed_nft_mints/udf_bulk_parse_compressed_nft_mints.sql index daa729e7..677e1a0c 100644 --- a/macros/streamline/bulk_parse_compressed_nft_mints/udf_bulk_parse_compressed_nft_mints.sql +++ b/macros/streamline/bulk_parse_compressed_nft_mints/udf_bulk_parse_compressed_nft_mints.sql @@ -1,6 +1,10 @@ {% macro udf_bulk_parse_compressed_nft_mints() %} CREATE - OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_parse_compressed_nft_mints("JSON" ARRAY) returns ARRAY api_integration = aws_solana_api_dev AS {% if target.database == 'SOLANA' -%} + OR REPLACE EXTERNAL FUNCTION streamline.udf_bulk_parse_compressed_nft_mints("JSON" ARRAY) + returns ARRAY + api_integration = aws_solana_api_dev + max_batch_rows = 1 + AS {% if target.database == 'SOLANA' -%} 'https://cpzzn7ohu0.execute-api.us-east-1.amazonaws.com/prod/parse' {% else %} 'https://rd7pddtgl9.execute-api.us-east-1.amazonaws.com/dev/parse' diff --git a/models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql b/models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql index 54cb6fcc..e92dc506 100644 --- a/models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql +++ b/models/bronze/bronze_api/bronze_api__parse_compressed_nft_mints.sql @@ -92,7 +92,6 @@ -- AND e.block_timestamp :: DATE = C.block_timestamp :: DATE -- AND ii_program_id = 'noopb9bkMVfRPU8AsbpTUg8AQkHtKwMYZiFUjNRtMmV' -- ) -{% for i in range(1,6) %} SELECT ARRAY_AGG(request) AS batch_request, streamline.udf_bulk_parse_compressed_nft_mints(batch_request) AS responses, @@ -106,10 +105,6 @@ SELECT FROM {{ source('bronze_api_prod','parse_compressed_nft_mints_requests') }} WHERE - gn = {{ min_gn }}+{{ i }} + gn between {{ min_gn }}+1 and {{ min_gn }}+10 GROUP BY - gn -{% if not loop.last %} -UNION ALL -{% endif %} -{% endfor %} \ No newline at end of file + gn \ No newline at end of file