(20241231) Too many thing happening...

playground/socketio/to_kafka.py

@@ -43,7 +43,7 @@ import os
 # My utils:
 from utils_v2.string import json
 from utils_v2.system import files
-from utils_v2.queue.kafka import ProducerKafka, create_config
+from utils_v2.queue.kafka.controllers.kafka import ProducerKafka
 
 # To make HTTP calls:
 import httpx
@@ -86,7 +86,7 @@ cwd = files.get_cwd()
 parent_dir = cwd
 kafka_producer = ProducerKafka(
     topic = "tickers",
-    config = create_config(
+    config = ProducerKafka.create_config(
         bootstrap_servers = "del.ditscentre.in:9092",
         # buffer_memory = 3_35_54_432,
         security_protocol = "SSL",
@@ -130,9 +130,10 @@ def to_kafka(tick: TradingTick) -> bool:
 
     success = False
     summary = tick.summary
-    print(json.to_string(summary))
-    print(json.to_string(tick.model_dump(), default=str))
+    # print(json.to_string(summary, default=str))
+    # print(json.to_string(tick.model_dump(), default=str))
     summary["messageType"] = "ticks"
+    summary = json.from_string(json.to_string(summary, default=str))
     success = kafka_producer.produce(value = summary)
     if not success:
         print("ERROR ON TICK NO.:", tick_count)
@@ -187,8 +188,16 @@ def main():
     # Get the instruments of interest:
     response = httpx.post(url = r"https://api.thecaoffice.com/markets/watchlist/distincts")
     instruments_of_interest = response.json()["data"]["rs0"]
-    print("TOTAL INSTR. OF INTEREST:", len(instruments_of_interest))
-    symbols_of_interest = [i["symbol"] for i in instruments_of_interest]
+    symbols_of_interest = []
+    broker_tokens_of_interest = []
+    for i in instruments_of_interest:
+        symbol = i["symbol"]
+        broker_token = i["broker_token"]
+        if broker_token is not None and i["source"] == "zerodha":
+            symbols_of_interest.append(symbol)
+            broker_tokens_of_interest.append(broker_token)
+    print("TOTAL INSTR. OF INTEREST:", f"{len(broker_tokens_of_interest)}/{len(instruments_of_interest)}")
+    print(broker_tokens_of_interest)
 
     # Create an instance of Zerodha's Kite connection:
     kite = KiteConnect(api_key = api_key)
@@ -196,29 +205,31 @@ def main():
 
     # Get the entire list of instruments:
     instruments = []
-    # instruments += kite.instruments(exchange = "NSE")
-    # instruments += kite.instruments(exchange = "NFO")
-    # instruments += kite.instruments(exchange = "BSE")
-    # instruments += kite.instruments(exchange = "BFO")
+    instruments += kite.instruments(exchange = "NSE")
+    instruments += kite.instruments(exchange = "NFO")
+    instruments += kite.instruments(exchange = "BSE")
+    instruments += kite.instruments(exchange = "BFO")
     instruments += kite.instruments(exchange = "MCX")
-    # instruments += kite.instruments(exchange = "CDS")
-    # instruments += kite.instruments(exchange = "BCD")
+    instruments += kite.instruments(exchange = "CDS")
+    instruments += kite.instruments(exchange = "BCD")
 
-    # # Pick the instruments of interest:
+    # Pick the instruments of interest:
     # instruments = [TradingSymbol.from_zerodha_kite(i) for i in instruments[:1000]]
-    # instruments = [
-    #     TradingSymbol.from_zerodha_kite(i) for i in instruments
-    #     if i["tradingsymbol"] in symbols_of_interest or i["name"] in symbols_of_interest
-    # ]
     instruments = [
         TradingSymbol.from_zerodha_kite(i) for i in instruments
-        if i["instrument_token"] in [109760007]
+        if str(i["instrument_token"]) in broker_tokens_of_interest
     ]
+    # instruments = [
+    #     TradingSymbol.from_zerodha_kite(i) for i in instruments
+    #     if i["instrument_token"] in [109760007]
+    # ]
+    print("SELECTED INSTRUMENTS:", len(instruments))
 
     # Create the lookup:
     for i in instruments:
         INSTRUMENT_TOKENS.append(i.brokerToken)
         INSTRUMENT_LOOKUP[i.brokerToken] = i.model_dump()
+    print("LOOK-UP READY!")
 
     # Start the websocket with Zerodha:
     kite_ws = KiteTicker(
@@ -232,6 +243,7 @@ def main():
 
     # If you choose to go threaded, you will need to work purely with callbacks.
     # You will need to have an infinite loop in the main thread.
+    print("STARTING WS...")
     kite_ws.connect(threaded = True)