.

.github/workflows/dbt_run_streamline_realtime.yml

@@ -6,8 +6,8 @@ on:
     branches:
       - "main"
   schedule:
-    - cron: '17 3,7,11,15,19,23 * * *'
-    # Run every 4 hours at 17 minutes past the hour
+    - cron: '17 2,5,8,11,14,17,20,23 * * *'
+    # Run every 3 hours at 17 minutes past the hour
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

models/streamline/core/realtime/streamline__checkpoints_realtime.sql

@@ -4,7 +4,7 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"checkpoints",
-        "sql_limit" :"4000000",
+        "sql_limit" :"3000000",
         "producer_batch_size" :"200000",
         "worker_batch_size" :"50000",
         "async_concurrent_requests" :"25",

models/streamline/core/realtime/streamline__transactions_realtime.sql

@@ -4,7 +4,7 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"transactions",
-        "sql_limit" :"2200000",
+        "sql_limit" :"1500000",
         "producer_batch_size" :"100000",
         "worker_batch_size" :"5000",
         "async_concurrent_requests" :"10",