sched

.github/workflows/dbt_run_streamline_realtime.yml

@@ -6,8 +6,8 @@ on:
     branches:
       - "main"
   schedule:
-    - cron: '7,22,37,52 * * * *'
-    # Run every hour at 7, 22, 37, and 52 minutes past the hour
+    - cron: '7 * * * *'
+    # Run every hour at 7 minutes past the hour
     
 env:
   DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}"

models/streamline/core/realtime/streamline__checkpoints_realtime.sql

@@ -4,9 +4,10 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"checkpoints",
-        "sql_limit" :"35000",
-        "producer_batch_size" :"35000",
-        "worker_batch_size" :"35000",
+        "sql_limit" :"10000000",
+        "producer_batch_size" :"200000",
+        "worker_batch_size" :"50000",
+        "async_concurrent_requests" :"25",
         "sql_source" :"{{this.identifier}}",
         "order_by_column": "checkpoint_number DESC" }
     ),

models/streamline/core/realtime/streamline__transactions_realtime.sql

@@ -4,9 +4,10 @@
         func = 'streamline.udf_bulk_rest_api_v2',
         target = "{{this.schema}}.{{this.identifier}}",
         params ={ "external_table" :"transactions",
-        "sql_limit" :"18000",
-        "producer_batch_size" :"18000",
-        "worker_batch_size" :"9000",
+        "sql_limit" :"2000000",
+        "producer_batch_size" :"100000",
+        "worker_batch_size" :"5000",
+        "async_concurrent_requests" :"10",
         "sql_source" :"{{this.identifier}}",
         'exploded_key': '["result"]',
         "order_by_column": "checkpoint_number DESC" }