(20250108) Tick-In script handles the buffer better.

background/finstitutions/trading/tick_in_stateful.py

@@ -44,6 +44,7 @@ import os
 from utils_v2.string import json
 from utils_v2.string import regex
 from utils_v2.system import files
+from utils_v2.date_time import date_time
 from utils_v2.database.async_mongo_v2 import AsyncMongo
 from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
 from utils_v2.queue.kafka.controllers.kafka import ProducerKafka, ConsumerKafka
@@ -158,15 +159,16 @@ def ticks_to_kafka(ticks: List[TradingTick]) -> int:
     # Push out the ticks to the queue:
     for tick in ticks:
 
-        # Flush the existing messages if needed:
+        # Flush the existing messages (if needed):
         TOTAL_TICK_COUNT += 1
         TICKS_SINCE_FLUSH += 1
-        if TICKS_SINCE_FLUSH > 5_000:
-            kafka_producer.flush()
+        if TICKS_SINCE_FLUSH > 50_000:
+            kafka_producer.flush(timeout = 0.0)
             TICKS_SINCE_FLUSH = 0
 
         # Push this one tick to the queue:
         success = kafka_producer.produce(value = tick.summary)
+        kafka_producer.client.poll(0)
         if success: success_count += 1
         else: failure_count += 1
 
@@ -211,7 +213,8 @@ def on_zerodha_kite_ticks(ws, ticks) -> None:
     # Model the raw input ticks:
     ticks = TradingTick.from_zerodha_kite(
         ticks = ticks,
-        instrument_lookup = ZERODHA_INSTRUMENT_LOOKUP
+        instrument_lookup = ZERODHA_INSTRUMENT_LOOKUP,
+        received_ts = date_time.get_current_utc_date_time(as_string = False)
     )
 
     # Send the ticks to Kafka:
@@ -428,7 +431,10 @@ def init(
             client_id = f"{SERVER_HOSTNAME}_{TOKEN_KEY}",
             acks = producer_creds["config"].get("acks", 1),
             retries = producer_creds["config"].get("retries", 1),
-            linger_ms = producer_creds["config"].get("lingerMs", 0)
+            linger_ms = producer_creds["config"].get("lingerMs", 0),
+            misc_json = {
+                "queue.buffering.max.messages": 2_00_000
+            }
         ),
         topic = producer_creds["topic"],
         serializer = JSONSerializer(),