khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

master

Browse Source
This commit is contained in:
khushal
2024-12-30 18:17:33 +05:30
parent de46a55353
commit fcc3d4b2e8
10 changed files with 6626 additions and 5765 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils_v2/queue/async_kafka.py
+7 -39
View File
@@ -473,7 +473,7 @@ if __name__ == "__main__":
import asyncio
import time
from data_models.kafka_message import KafkaMessage
# from data_models.kafka_message import KafkaMessage
ssl_ctx = get_ssl_context(
ca_file = r"/home/developer/PycharmProjects/utils/cred/kafka/cert_authority.pem",
@@ -497,13 +497,13 @@ if __name__ == "__main__":
while True:
messages = await consumer.consume(count = 1)
if len(messages) > 0: print("MESSAGE:", json.to_string(messages[0], default=str))
if len(messages) > 0: print("MESSAGE:", json.to_string(messages[0], default = str))
await asyncio.sleep(1.0)
async def producer_test():
producer = ProducerKafka(
topic = "kft_file_upload",
topic = "tickers",
bootstrap_servers = "del.ditscentre.in:9092",
security_protocol = "SSL",
ssl_context = ssl_ctx
@@ -512,43 +512,11 @@ if __name__ == "__main__":
print("READY!")
while True:
my_msg = KafkaMessage(
data = {
"accepted": False,
"reason": "low resolution"
},
media = {
"name": "pikachu_poster.jpg",
"ext": "jpg",
"url": "https://nexcom.ditscentre.in/utils/files/small/download/66ded1c1c1c05139a618b5ff",
"attr": {
"user": "SarangKabir",
"project": "ACE-PGP",
"id": 173,
"campaignActivityId": "25",
"idCampaign": 49,
"phoneNo": "7977821877"
}
},
appId = "aceWockhardt",
proc = {
"name": "_assessImg",
"attr": {
"blurThreshold": 0.25,
"clarityThreshold": 0.65,
"nsfwThreshold": 0.25,
"minWidth": 512,
"minHeight": 512
}
},
ack = None
)
success = await producer.produce(my_msg.model_dump())
print("produced...")
time.sleep(1.0)
break
success = await producer.produce({"name": "Bhopli", "color": "orange"})
print("PRODUCED:", success)
await asyncio.sleep(0.25)
await producer.close()
asyncio.run(consumer_test())
asyncio.run(producer_test())
utils_v2/queue/kafka.py
+264 -238
View File
@@ -81,7 +81,7 @@ import ssl
# *****************************************************************************************************************
def create_config(
def create_producer_config(
bootstrap_servers: str | List[str],
group_id: str | None = None,
security_protocol: Literal["PLAINTEXT", "SSL"] = "PLAINTEXT",
@@ -142,8 +142,8 @@ class ProducerKafka:
def __init__(
self,
topic: str,
config: dict,
topic: str,
serializer = None,
debug: bool = True,
debug_prefix = "Kafka (P) | "
@@ -151,8 +151,8 @@ class ProducerKafka:
"""
Create a Kafka Producer.
:param topic: The topic to produce on.
:param config: The configuration as expected by Confluent-Kafka library.
:param topic: The topic to produce on.
:param serializer: The serializer to use.
:param debug: Whether, or not, you want to print the debug strings.
:param debug_prefix: The prefix to use while debugging.
@@ -175,6 +175,46 @@ class ProducerKafka:
def disable_debug(self):
self.__printer.disable()
@staticmethod
def create_config(
bootstrap_servers: str | List[str],
security_protocol: Literal["PLAINTEXT", "SSL"] = "PLAINTEXT",
ca_file: str | None = None,
cert_file: str | None = None,
key_file: str | None = None,
client_id: str | int | None = None
):
"""
Creates the config required for Confluent-Kafka's library.
:param bootstrap_servers: The addresses of the Kafka brokers.
:param security_protocol: What sort of security protocol to use.
:param ca_file: Needed for 'SSL' security protocol.
:param cert_file: Needed for 'SSL' security protocol.
:param key_file: Needed for 'SSL' security protocol.
:param client_id: An identifier for one producer. Useful for debugging later.
:return: The dictionary that needs to be passed as the 'conf' param when creating the producer.
"""
# Start with the bare minimum:
if not isinstance(bootstrap_servers, list): bootstrap_servers = [bootstrap_servers]
config = {
"bootstrap.servers": ",".join(bootstrap_servers),
"security.protocol": security_protocol
}
# Add the SSL security details:
if security_protocol == "SSL":
config["ssl.ca.location"] = ca_file
config["ssl.certificate.location"] = cert_file
config["ssl.key.location"] = key_file
# Add an identifier for debugging:
if client_id: config["client.id"] = client_id
# Done here:
return config
def connect(self) -> bool:
"""
@@ -270,242 +310,204 @@ class ProducerKafka:
# ---------------------------------------------------------------------------------------------------------------------
# class ConsumerKafka:
#
# def __init__(
# self,
# topic,
# serializer = None,
# debug = True,
# debug_prefix = "Kafka (C) | ",
# **kwargs
# ):
#
# """
# Create a Kafka Consumer.
# :param topic: The topic to consumer on.
# :param serializer: The serializer to use.
# :param debug: Whether, or not, you want to print the debug strings.
# :param debug_prefix: The prefix to use while debugging.
# :param kwargs: Any configuration parameters for the Kafka instances.
# """
#
# # Initialize the debugger:
# self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
# if not debug: self.__printer.disable()
#
# # initialize the Kafka producer:
# self.__topic = topic
# self.__kwargs = kwargs
# self.__consumer = None
# self.__connected = False
# self.__serializer = serializer or JSONSerializer()
#
# # For establishing connection:
# self.__exclusive_semaphore = asyncio.Semaphore(1)
#
# def enable_debug(self):
# self.__printer.enable()
#
# def disable_debug(self):
# self.__printer.disable()
#
# async def connect(self):
#
# """
# Connects to the Kafka server if not connected.
# :return: True or False based on the success of the operation.
# """
#
# async with self.__exclusive_semaphore:
# if not self.__connected:
# try:
# self.__consumer = AIOKafkaConsumer(self.__topic, **self.__kwargs)
# await self.__consumer.start()
# self.__connected = True
# except Exception as exception: self.__printer(exception)
# return self.__connected
#
# async def ensure_connection(self):
#
# """
# Connects to the Kafka server if not connected.
# :return: True or False based on the success of the operation.
# """
#
# if not self.__connected: await self.connect()
# return self.__connected
#
# async def close(self):
#
# """
# Terminates the connection.
# :return: None.
# """
#
# if self.__connected:
# try:
# await self.__consumer.stop()
# self.__printer("Consumer closed!")
# self.__connected = False
# except Exception as exception: self.__printer(exception)
#
# async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"):
#
# """
# Get messages from the Kafka server.
# :param count: The number of messages to get from the Kafka server.
# :param timeout: The time in seconds to wait for retrieval.
# :param encoding: The encoding to use.
# :return: The messages that were received. If no messages are available, an empty list will be returned.
# """
#
# # Ensure connectivity to the server.
# # If not connected, return with failure immediately.
# if not await self.ensure_connection(): return []
#
# # Make a variable that will hold the final results:
# messages = []
#
# try:
#
# # Read some messages:
# results = await self.__consumer.getmany(
# max_records = max(1, count),
# timeout_ms = int(timeout * 1_000)
# )
#
# # Format the received messages:
# if results:
# for topic_partition, records in results.items():
# for record in records:
# record_dict = record.__dict__
# record_dict["value"] = self.__serializer.deserialize(
# data = record_dict["value"],
# encoding = encoding
# )
# messages.append(record_dict)
#
# # Debugging print if something went wrong:
# except Exception as exception: self.__printer(exception)
#
# # Done here:
# return messages
class ConsumerKafka:
def __init__(
self,
config: dict,
topic: str,
serializer = None,
debug = True,
debug_prefix = "Kafka (C) | ",
**kwargs
):
# ---------------------------------------------------------------------------------------------------------------------
"""
Create a Kafka Consumer.
:param topic: the topic to consume on.
:param serializer: The serializer to use.
:param debug: Whether, or not, you want to print the debug strings.
:param debug_prefix: The prefix to use while debugging.
:param kwargs: Any configuration parameters for the Kafka instances.
"""
# Initialize the debugger:
self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self.__printer.disable()
# class BidirectionalKafka:
#
# # The 'roles' that the instance can take.
# # The master talks on the channel (topic) that the slave listens on and vice versa.
# # Master-Slave is only for deciding who talks on which channel and who listens on which.
# # In a two-party system, one must be the master, the other must be the slave.
# # There are no extra privileges that the master enjoys. The naming convention was borrowed from common protocols
# # used in electronics (like I2C).
# ROLE_MASTER = 1
# ROLE_SLAVE = 0
#
# def __init__(
# self,
# role,
# topic,
# ack_topic: str = None,
# group: str = None,
# serializer = None,
# debug = True,
# debug_prefix = "Kafka (B) | ",
# **kwargs
# ):
#
# """
# Creates a walkie-talkie type setup to use Kafka in a bidirectional manner. Fo more information on all the
# individual methods, please read through the doc-strings of the component classes 'ProducerKafka', and
# 'ConsumerKafka'.
# :param role: Select from "ROLE_MASTER" and "ROLE_SLAVE". Between the two parties that are talking, one will be
# the master and the other will be the slave. The channel that the master uses to speak will the one the slave
# uses to listen, and vice versa.
# :param topic: The topic to communicate on. Will be the same between the master and the slave.
# :param ack_topic: Explicitly provide this for the second channel, or it will be created from the name of the
# topic itself. Will be the same between the master and the slave.
# :param group: The group to assign the instance to.
# :param debug: Whether, or not, you want to print the debug strings.
# :param debug_prefix: The prefix to use while debugging.
# :param kwargs: Any configuration parameters for the Kafka instances.
# """
#
# # Not down the basic variables:
# self.__role = role
# self.__topic = topic
# self.__ack_topic = ack_topic or topic + "Ack"
# self.__group = group
#
# # In case the current instance is the master,
# # it will talk on "topic", and listen on "ack_topic":
# if self.__role == self.ROLE_MASTER:
# self.__producer_kwargs = kwargs.copy()
# self.__producer = ProducerKafka(
# topic = self.__topic,
# serializer = serializer,
# debug = debug,
# debug_prefix = debug_prefix.strip() + " (P) | ",
# **self.__producer_kwargs
# )
# self.__consumer_kwargs = kwargs.copy()
# self.__consumer_kwargs["group_id"] = self.__group
# self.__consumer = ConsumerKafka(
# topic = self.__ack_topic,
# group = group,
# serializer = serializer,
# debug = debug,
# debug_prefix = debug_prefix.strip() + " (C) | ",
# **self.__consumer_kwargs
# )
#
# # On the other hand, if the current instance is a slave,
# # It will listen on "topic", and talk on "ack_topic":
# else:
# self.__producer_kwargs = kwargs.copy()
# self.__producer = ProducerKafka(
# topic = self.__ack_topic,
# serializer = serializer,
# debug = debug,
# debug_prefix = debug_prefix.strip() + " (P) | ",
# **self.__producer_kwargs
# )
# self.__consumer_kwargs = kwargs.copy()
# self.__consumer_kwargs["group_id"] = self.__group
# self.__consumer = ConsumerKafka(
# topic = self.__topic,
# group = group,
# serializer = serializer,
# debug = debug,
# debug_prefix = debug_prefix.strip() + " (C) | ",
# **self.__consumer_kwargs
# )
#
# def enable_debug(self):
# self.__producer.enable_debug()
# self.__consumer.enable_debug()
#
# def disable_debug(self):
# self.__producer.disable_debug()
# self.__consumer.disable_debug()
#
# async def ensure_connection(self):
# await self.__producer.ensure_connection()
# await self.__consumer.ensure_connection()
#
# async def close(self):
# await self.__producer.close()
# await self.__consumer.close()
#
# async def produce(self, message, encoding = "utf-8"):
# return await self.__producer.produce(message, encoding = encoding)
#
# async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"):
# return await self.__consumer.consume(count = count, timeout = timeout, encoding = encoding)
# initialize the Kafka consumer:
self.__config = config
self.__topic = topic
self.__kwargs = kwargs
self.__consumer = None
self.__connected = False
self.__serializer = serializer or JSONSerializer()
def enable_debug(self):
self.__printer.enable()
def disable_debug(self):
self.__printer.disable()
@staticmethod
def create_config(
bootstrap_servers: str | List[str],
group_id: str,
auto_offset_reset: Literal["latest", "earliest"] = "latest",
security_protocol: Literal["PLAINTEXT", "SSL"] = "PLAINTEXT",
ca_file: str | None = None,
cert_file: str | None = None,
key_file: str | None = None,
client_id: str | int | None = None
):
"""
Creates the config required for Confluent-Kafka's library.
:param bootstrap_servers: The addresses of the Kafka brokers.
:param group_id: When a set of consumers are working on one topic in a group such that you want only one of them
to read a particular message.
:param auto_offset_reset: Use this to influence the behaviour of how the Kafka consumer will read messages when
it first connects to the broker. It could either want to read the earliest (oldest) messages or the latest
(newest) messages from the queue.
:param security_protocol: What sort of security protocol to use.
:param ca_file: Needed for 'SSL' security protocol.
:param cert_file: Needed for 'SSL' security protocol.
:param key_file: Needed for 'SSL' security protocol.
:param client_id: An identifier for one producer. Useful for debugging later.
:return: The dictionary that needs to be passed as the 'conf' param when creating the producer.
"""
# Start with the bare minimum:
if not isinstance(bootstrap_servers, list): bootstrap_servers = [bootstrap_servers]
config = {
"bootstrap.servers": ",".join(bootstrap_servers),
"security.protocol": security_protocol
}
# Add the SSL security details:
if security_protocol == "SSL":
config["ssl.ca.location"] = ca_file
config["ssl.certificate.location"] = cert_file
config["ssl.key.location"] = key_file
# Add an identifier for debugging:
if client_id: config["client.id"] = client_id
# Consumer-specific:
config["group.id"] = group_id
config["auto.offset.reset"] = auto_offset_reset
# Done here:
return config
def connect(self):
"""
Connects to the Kafka server if not connected.
:return: True or False based on the success of the operation.
"""
if not self.__connected:
try:
if not isinstance(self.__topic, list): self.__topic = [self.__topic]
self.__consumer = Consumer(self.__config)
self.__consumer.subscribe(self.__topic)
self.__connected = True
except Exception as exception: self.__printer(exception)
return self.__connected
def ensure_connection(self):
"""
Connects to the Kafka server if not connected.
:return: True or False based on the success of the operation.
"""
if not self.__connected: self.connect()
return self.__connected
def close(self):
"""
Terminates the connection.
:return: None.
"""
if self.__connected:
try:
self.__consumer.close()
self.__printer("Consumer closed!")
self.__connected = False
except Exception as exception: self.__printer(exception)
def __consume_one(
self,
timeout = 0.05,
encoding = "utf-8"
) -> dict | None:
"""
Here we consume exactly one message from the broker. If we need multiple messages (a batch of messages), we call
this method as many times as needed.
:param timeout: The time in seconds to wait for retrieval.
:param encoding: The encoding to use.
:return: A dict that holds the details of the message. Null if no message was fetched.
"""
message = self.__consumer.poll(timeout = timeout)
if message is None or message.error(): return None
else:
message_value = message.value()
if message_value is not None: message_value = self.__serializer.deserialize(message_value, encoding = encoding)
return {
"topic": message.topic(),
"partition": message.partition(),
"offset": message.offset(),
"key": message.key().decode("utf-8") if message.key() else None,
"value": message_value,
"timestamp": message.timestamp()[1],
"headers": message.headers()
}
def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"):
"""
Get messages from the Kafka broker.
:param count: The number of messages to get from the Kafka server.
:param timeout: The time in seconds to wait for retrieval.
:param encoding: The encoding to use.
:return: The messages that were received. If no messages are available, an empty list will be returned.
"""
# Make some variables:
start_time = time.time()
messages = []
# Ensure connectivity to the server.
# If not connected, return with failure immediately.
if not self.ensure_connection(): return messages
# Get into an indefinite loop.
while True:
# If timed-out, break out of the loop:
elapsed_time = time.time() - start_time
if elapsed_time > timeout: break
# Get the next message:
message = self.__consume_one(
timeout = timeout - elapsed_time,
encoding = encoding
)
# If a message was fetched in the timeout, append it to the list of messages.
# Break out of the loop if you have reached the needed no. of messages:
if message:
messages.append(message)
if len(messages) >= count: break
# Done here:
return messages
# *****************************************************************************************************************
@@ -528,12 +530,13 @@ if __name__ == "__main__":
# Create and connect the producer:
producer = ProducerKafka(
topic = "kft_file_upload",
config = create_config(
config = ProducerKafka.create_config(
bootstrap_servers = "del.ditscentre.in:9092",
security_protocol = "SSL",
ca_file = r"../../creds/kafka/cert_authority.pem",
cert_file = r"../../creds/kafka/fullchain.pem",
key_file = r"../../creds/kafka/privkey.pem"
key_file = r"../../creds/kafka/privkey.pem",
client_id = 123
)
)
producer.connect()
@@ -557,6 +560,29 @@ if __name__ == "__main__":
# Ensure a graceful close:
producer.close()
def consumer_test():
# Create and connect the producer:
consumer = ConsumerKafka(
topic = "tickers",
config = ConsumerKafka.create_config(
bootstrap_servers = "del.ditscentre.in:9092",
group_id = "test-group",
security_protocol = "SSL",
ca_file = r"../../creds/kafka/cert_authority.pem",
cert_file = r"../../creds/kafka/fullchain.pem",
key_file = r"../../creds/kafka/privkey.pem",
)
)
consumer.connect()
print("CONSUMER READY!")
for _ in range(5):
messages = consumer.consume(count = 3, timeout = 2.5)
print(json.to_string(messages, default = str))
consumer.close()
# Run the test code:
producer_test(count = 5)
# producer_test(count = 5)
consumer_test()