(20241226) Date-time management for Zerodha's instrument list.

playground/socketio/to_kafka.py

@@ -88,17 +88,22 @@ kafka_producer = ProducerKafka(
     topic = "tickers",
     config = create_config(
         bootstrap_servers = "del.ditscentre.in:9092",
-        buffer_size = 1,
+        # buffer_memory = 3_35_54_432,
         security_protocol = "SSL",
-        # ca_file = r"../../creds/kafka/cert_authority.pem",
-        # cert_file = r"../../creds/kafka/fullchain.pem",
-        # key_file = r"../../creds/kafka/privkey.pem"
-        ca_file = os.path.join(parent_dir, "creds", "kafka", "cert_authority.pem"),
-        cert_file = os.path.join(parent_dir, "creds", "kafka", "fullchain.pem"),
-        key_file = os.path.join(parent_dir, "creds", "kafka", "privkey.pem")
-    )
+        ca_file = r"../../creds/kafka/cert_authority.pem",
+        cert_file = r"../../creds/kafka/fullchain.pem",
+        key_file = r"../../creds/kafka/privkey.pem"
+        # ca_file = os.path.join(parent_dir, "creds", "kafka", "cert_authority.pem"),
+        # cert_file = os.path.join(parent_dir, "creds", "kafka", "fullchain.pem"),
+        # key_file = os.path.join(parent_dir, "creds", "kafka", "privkey.pem")
+    ),
+    debug = False
 )
 
+# For metrics:
+tick_count = 0
+ticks_since_flush = 0
+
 
 #   *****************************************************************************************************************
 #  *****                                                                                                          ****
@@ -107,12 +112,30 @@ kafka_producer = ProducerKafka(
 #   *****************************************************************************************************************
 
 
+def flush_kafka():
+    print("FLUSHING!")
+    kafka_producer.flush()
+
+
 def to_kafka(tick: TradingTick) -> bool:
 
+    global tick_count
+    global ticks_since_flush
+
+    tick_count += 1
+    ticks_since_flush += 1
+    if ticks_since_flush >= 50_000:
+        flush_kafka()
+        ticks_since_flush = 0
+
     success = False
     summary = tick.summary
     summary["messageType"] = "ticks"
     success = kafka_producer.produce(value = summary)
+    if not success:
+        print("ERROR ON TICK NO.:", tick_count)
+        flush_kafka()
+
     return success
 
 
@@ -141,7 +164,7 @@ def on_ticks(ws, ticks):
     # print("TICK SAMPLE:", json.to_string(ticks[0].summary, default=str))
     results = [to_kafka(tick) for tick in ticks]
     success = sum(results)
-    print(f"TICKS: {len(ticks): <4}    |    PRODUCED: {success: <4}{'    |    FAILURE(S)!' if success < len(results)else ''}")
+    print(f"TICKS: {len(ticks): <6,}    |    PRODUCED: {success: <6,}    |    TOTAL: {tick_count: >10,}{'    |    FAILURE(S)!' if success < len(results)else ''}")
 
 
 # ---------------------------------------------------------------------------------------------------------------------
@@ -154,8 +177,8 @@ def main():
     global INSTRUMENT_LOOKUP
 
     # Load Zerodha credentials:
-    # creds = json.from_file(r"../../creds/zerodha/api.json")
-    creds = json.from_file(os.path.join(parent_dir, "creds", "zerodha", "api.json"))
+    creds = json.from_file(r"../../creds/zerodha/api.json")
+    # creds = json.from_file(os.path.join(parent_dir, "creds", "zerodha", "api.json"))
     api_key = creds["apiKey"]
     access_token = creds["accessToken"]
 
@@ -173,7 +196,7 @@ def main():
     instruments += kite.instruments(exchange = "BCD")
 
     # Pick the instruments of interest:
-    instruments = instruments[:1000]
+    instruments = instruments[:1250]
     instruments = [TradingSymbol.from_zerodha_kite(i) for i in instruments]
 
     # Create the lookup: