.

.github/workflows/dbt_run_streamline_realtime.yml

@@ -6,8 +6,8 @@ on:
     branches:
       - "main"
   schedule:
-    - cron: '17 * * * *'
-    # Run every hour at 7 minutes past the hour
+    - cron: '17 3,8,13,18,23 * * *'
+    # Run every 5 hours at 17 minutes past the hour
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

models/streamline/core/realtime/streamline__checkpoints_realtime.sql

@@ -4,7 +4,7 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"checkpoints",
-        "sql_limit" :"800000",
+        "sql_limit" :"5000000",
         "producer_batch_size" :"200000",
         "worker_batch_size" :"50000",
         "async_concurrent_requests" :"25",

models/streamline/core/realtime/streamline__transactions_realtime.sql

@@ -4,7 +4,7 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"transactions",
-        "sql_limit" :"400000",
+        "sql_limit" :"2000000",
         "producer_batch_size" :"100000",
         "worker_batch_size" :"5000",
         "async_concurrent_requests" :"10",