From 215e05e784434a839226181e6364e20cc79f0d6a Mon Sep 17 00:00:00 2001 From: khushal Date: Tue, 12 Nov 2024 11:09:40 +0530 Subject: [PATCH] Squashed 'utils_v2/' content from commit a9c9cb7 git-subtree-dir: utils_v2 git-subtree-split: a9c9cb7c91a19090b657df809e381fad2959143c --- .gitignore | 8 + __init__.py | 0 ai/__init__.py | 0 ai/image_classification/__init__.py | 0 ai/image_classification/async_blur.py | 202 ++ ai/image_classification/async_nsfw.py | 188 ++ ai/image_segmentation/__init__.py | 0 .../async_dichotomous_segmentation.py | 223 +++ ai/object_detection/__init__.py | 0 ai/object_detection/yolo.py | 176 ++ ai/tts/__init__.py | 0 ai/tts/easy_tts.py | 175 ++ ai/tts/test.wav | Bin 0 -> 162150 bytes api/__init__.py | 0 api/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 166 bytes api/__pycache__/async_quart.cpython-310.pyc | Bin 0 -> 28257 bytes api/__pycache__/codes.cpython-310.pyc | Bin 0 -> 3712 bytes api/__pycache__/log.cpython-310.pyc | Bin 0 -> 1746 bytes .../metrics_prometheus.cpython-310.pyc | Bin 0 -> 5163 bytes api/__pycache__/response.cpython-310.pyc | Bin 0 -> 1877 bytes api/async_quart.py | 1161 +++++++++++ api/codes.py | 164 ++ api/log.py | 138 ++ api/response.py | 133 ++ cache/__init__.py | 0 cache/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 168 bytes .../async_redis_cache.cpython-310.pyc | Bin 0 -> 13364 bytes cache/async_redis_cache.py | 497 +++++ cache/async_redis_cache_v2.py | 458 +++++ database/__init__.py | 0 database/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 171 bytes .../async_mongo_v2.cpython-310.pyc | Bin 0 -> 40273 bytes .../async_mysql_v2.cpython-310.pyc | Bin 0 -> 9102 bytes database/async_firebase.py | 504 +++++ database/async_mongo.py | 843 ++++++++ database/async_mongo_storage.py | 358 ++++ database/async_mongo_v2.py | 1736 +++++++++++++++++ database/async_mysql_v2.py | 424 ++++ dataframe/__init__.py | 0 dataframe/splitter.py | 118 ++ date_time/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 172 bytes .../__pycache__/date_time.cpython-310.pyc | Bin 0 -> 5866 bytes date_time/date_time.py | 271 +++ image/__init__.py | 0 image/scanner/__init__.py | 0 image/scanner/scanner.py | 410 ++++ image/scanner/scanner_v2.py | 379 ++++ logging/__init__.py | 0 logging/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 170 bytes logging/__pycache__/model.cpython-310.pyc | Bin 0 -> 1713 bytes logging/context.py | 469 +++++ logging/model.py | 131 ++ mail/__init__.py | 0 mail/async_mail.py | 440 +++++ mail/mail_message.py | 268 +++ network/__init__.py | 0 network/ip.py | 173 ++ network/ping.py | 218 +++ pdf/__init__.py | 0 pdf/pdf_maker.py | 1642 ++++++++++++++++ queue/__init__.py | 0 queue/async_kafka.py | 554 ++++++ rate_limit/__init__.py | 0 rate_limit/async_token_bucket.py | 212 ++ security/__init__.py | 0 security/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 171 bytes .../__pycache__/sanitizers.cpython-310.pyc | Bin 0 -> 1933 bytes security/hash.py | 245 +++ security/otp.py | 156 ++ security/sanitizers.py | 135 ++ serialization/__init__.py | 0 serialization/json_serializer.py | 129 ++ serialization/pickle_serializer.py | 122 ++ serialization/universal_serializer.py | 265 +++ sms/__init__.py | 0 sms/async_nimbus.py | 252 +++ string/__init__.py | 0 string/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 169 bytes string/__pycache__/json.cpython-310.pyc | Bin 0 -> 3546 bytes string/__pycache__/regex.cpython-310.pyc | Bin 0 -> 7489 bytes string/fuzzy.py | 168 ++ string/help.py | 177 ++ string/json.py | 202 ++ string/regex.py | 288 +++ system/__init__.py | 0 system/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 169 bytes system/__pycache__/files.cpython-310.pyc | Bin 0 -> 5497 bytes system/files.py | 308 +++ 89 files changed, 15120 insertions(+) create mode 100644 .gitignore create mode 100644 __init__.py create mode 100644 ai/__init__.py create mode 100644 ai/image_classification/__init__.py create mode 100644 ai/image_classification/async_blur.py create mode 100644 ai/image_classification/async_nsfw.py create mode 100644 ai/image_segmentation/__init__.py create mode 100644 ai/image_segmentation/async_dichotomous_segmentation.py create mode 100644 ai/object_detection/__init__.py create mode 100644 ai/object_detection/yolo.py create mode 100644 ai/tts/__init__.py create mode 100644 ai/tts/easy_tts.py create mode 100644 ai/tts/test.wav create mode 100644 api/__init__.py create mode 100644 api/__pycache__/__init__.cpython-310.pyc create mode 100644 api/__pycache__/async_quart.cpython-310.pyc create mode 100644 api/__pycache__/codes.cpython-310.pyc create mode 100644 api/__pycache__/log.cpython-310.pyc create mode 100644 api/__pycache__/metrics_prometheus.cpython-310.pyc create mode 100644 api/__pycache__/response.cpython-310.pyc create mode 100644 api/async_quart.py create mode 100644 api/codes.py create mode 100644 api/log.py create mode 100644 api/response.py create mode 100644 cache/__init__.py create mode 100644 cache/__pycache__/__init__.cpython-310.pyc create mode 100644 cache/__pycache__/async_redis_cache.cpython-310.pyc create mode 100644 cache/async_redis_cache.py create mode 100644 cache/async_redis_cache_v2.py create mode 100644 database/__init__.py create mode 100644 database/__pycache__/__init__.cpython-310.pyc create mode 100644 database/__pycache__/async_mongo_v2.cpython-310.pyc create mode 100644 database/__pycache__/async_mysql_v2.cpython-310.pyc create mode 100644 database/async_firebase.py create mode 100644 database/async_mongo.py create mode 100644 database/async_mongo_storage.py create mode 100644 database/async_mongo_v2.py create mode 100644 database/async_mysql_v2.py create mode 100644 dataframe/__init__.py create mode 100644 dataframe/splitter.py create mode 100644 date_time/__init__.py create mode 100644 date_time/__pycache__/__init__.cpython-310.pyc create mode 100644 date_time/__pycache__/date_time.cpython-310.pyc create mode 100644 date_time/date_time.py create mode 100644 image/__init__.py create mode 100644 image/scanner/__init__.py create mode 100644 image/scanner/scanner.py create mode 100644 image/scanner/scanner_v2.py create mode 100644 logging/__init__.py create mode 100644 logging/__pycache__/__init__.cpython-310.pyc create mode 100644 logging/__pycache__/model.cpython-310.pyc create mode 100644 logging/context.py create mode 100644 logging/model.py create mode 100644 mail/__init__.py create mode 100644 mail/async_mail.py create mode 100644 mail/mail_message.py create mode 100644 network/__init__.py create mode 100644 network/ip.py create mode 100644 network/ping.py create mode 100644 pdf/__init__.py create mode 100644 pdf/pdf_maker.py create mode 100644 queue/__init__.py create mode 100644 queue/async_kafka.py create mode 100644 rate_limit/__init__.py create mode 100644 rate_limit/async_token_bucket.py create mode 100644 security/__init__.py create mode 100644 security/__pycache__/__init__.cpython-310.pyc create mode 100644 security/__pycache__/sanitizers.cpython-310.pyc create mode 100644 security/hash.py create mode 100644 security/otp.py create mode 100644 security/sanitizers.py create mode 100644 serialization/__init__.py create mode 100644 serialization/json_serializer.py create mode 100644 serialization/pickle_serializer.py create mode 100644 serialization/universal_serializer.py create mode 100644 sms/__init__.py create mode 100644 sms/async_nimbus.py create mode 100644 string/__init__.py create mode 100644 string/__pycache__/__init__.cpython-310.pyc create mode 100644 string/__pycache__/json.cpython-310.pyc create mode 100644 string/__pycache__/regex.cpython-310.pyc create mode 100644 string/fuzzy.py create mode 100644 string/help.py create mode 100644 string/json.py create mode 100644 string/regex.py create mode 100644 system/__init__.py create mode 100644 system/__pycache__/__init__.cpython-310.pyc create mode 100644 system/__pycache__/files.cpython-310.pyc create mode 100644 system/files.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebb370b --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/.venv/ +/.idea/ +**/__pycache__/ +__pycache__/ + +*.pem +*.pyc +*.pyd \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/__init__.py b/ai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/image_classification/__init__.py b/ai/image_classification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/image_classification/async_blur.py b/ai/image_classification/async_blur.py new file mode 100644 index 0000000..f69af42 --- /dev/null +++ b/ai/image_classification/async_blur.py @@ -0,0 +1,202 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 2nd Sept., 2024 + + OBJECTIVE: + + To provide an easy way to assess images for blurriness. + Tried and implemented using HuggingFace behaviour_models. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# For using the AI model: +import torch +from transformers import pipeline + +# To download images from URLs: +import requests + +# To read images: +from PIL import Image + +# Common: +from shared import variables + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AssessImageBlur: + + def __init__( + self, + model, + blur_label = "Blur", + clarity_label = "Normal", + blur_threshold = 0.25, + clarity_threshold = 0.65 + ): + + """ + This class assesses the input image and tells if it is blurry or clear. + :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is + stored. + :param blur_threshold: The max allowed blurriness (0 to 1 range). + :param clarity_threshold: The minimum needed clarity (0 to 1 range). + """ + + # Note down the config: + self.__model = model + self.__device = "gpu" if torch.cuda.is_available() else "cpu" + self.__blur_label = blur_label + self.__clarity_label = clarity_label + self.__blur_threshold = blur_threshold + self.__clarity_threshold = clarity_threshold + + # Initialize the classifier: + self.__classifier = pipeline( + task = "image-classification", + model = self.__model, + device = self.__device + ) + + @staticmethod + async def read_image(source): + + """ + Reads an image in whichever format it is provided and returns it as a PIL object. + :param source: The image as either a path or a URL or a io.BytesIO object. + :return: The image opened as a PIL object. + """ + + # If the input image is already a PIL image: + if isinstance(source, Image.Image): return source + + # If a buffer is provided: + elif isinstance(source, io.BytesIO): + source.seek(0) + return Image.open(source) + + # If a string is provided (local path or URL): + elif isinstance(source, str): + if source.startswith("http://") or source.startswith("https://"): + response = await variables.http_client.get(source) + return Image.open(io.BytesIO(response.content)) + else: return Image.open(source) + + async def classify(self, image): + + """ + To get the prediction of the model from the given input image. + :param image: The image (as a PIL object or file path or a URL). + :return: The dictionary of classes with their respective predictions. + """ + + image = await self.read_image(image) + predictions = self.__classifier(image) + classes = {p["label"]: p["score"] for p in predictions} + return classes + + async def is_ok( + self, + image, + blur_threshold = None, + clarity_threshold = None + ): + + """ + Checks if the image given to it can be used, or should be rejected. + :param image: The image (as a PIL object or file path or a URL). + :param blur_threshold: A custom threshold to test against. If not provided, the default will be taken that was + provided when the instance was created. + :param clarity_threshold: A custom threshold to test against. If not provided, the default will be taken that + was provided when the instance was created. + :return: True if the image is okay, else False. + """ + + image = await self.read_image(image) + blur_threshold = blur_threshold or self.__blur_threshold + clarity_threshold = clarity_threshold or self.__clarity_threshold + classes = await self.classify(image) + if ( + classes[self.__blur_label] <= blur_threshold and + classes[self.__clarity_label] >= clarity_threshold + ): return True + return False + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + + async def main(): + image = r"/home/developer/Downloads/low-res-check.png" + assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh") + usable = await assessor.is_ok(image) + classes = await assessor.classify(image) + print("IS OKAY:", usable) + print("CLASSES:", classes) + + asyncio.run(main()) diff --git a/ai/image_classification/async_nsfw.py b/ai/image_classification/async_nsfw.py new file mode 100644 index 0000000..446c34a --- /dev/null +++ b/ai/image_classification/async_nsfw.py @@ -0,0 +1,188 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 2nd Sept., 2024 + + OBJECTIVE: + + To provide an easy way to assess images for adult content. + Tried and implemented using HuggingFace behaviour_models. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# For using the AI model: +import torch +from transformers import pipeline + +# To download images from URLs: +import requests + +# To read images: +from PIL import Image + +# Common: +from shared import variables + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AssessImageNSFW: + + def __init__( + self, + model, + nsfw_label = "nsfw", + nsfw_threshold = 0.25, + ): + + """ + This class assesses the input image and tells if it is blurry or clear. + :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is + stored. + :param nsfw_threshold: The max allowed blurriness (0 to 1 range). + """ + + # Note down the config: + self.__model = model + self.__device = "gpu" if torch.cuda.is_available() else "cpu" + self.__nsfw_label = nsfw_label + self.__nsfw_threshold = nsfw_threshold + + # Initialize the classifier: + self.__classifier = pipeline( + task = "image-classification", + model = self.__model, + device = self.__device + ) + + @staticmethod + async def read_image(source): + + """ + Reads an image in whichever format it is provided and returns it as a PIL object. + :param source: The image as either a path or a URL or a io.BytesIO object. + :return: The image opened as a PIL object. + """ + + # If the input image is already a PIL image: + if isinstance(source, Image.Image): return source + + # If a buffer is provided: + elif isinstance(source, io.BytesIO): + source.seek(0) + return Image.open(source) + + # If a string is provided (local path or URL): + elif isinstance(source, str): + if source.startswith("http://") or source.startswith("https://"): + response = await variables.http_client.get(source) + return Image.open(io.BytesIO(response.content)) + else: return Image.open(source) + + async def classify(self, image): + + """ + To get the prediction of the model from the given input image. + :param image: The image (as a PIL object or file path or a URL). + :return: The dictionary of classes with their respective predictions. + """ + + image = await self.read_image(image) + predictions = self.__classifier(image) + classes = {p["label"]: p["score"] for p in predictions} + return classes + + async def is_ok( + self, + image, + nsfw_threshold = None + ): + + """ + Checks if the image given to it can be used, or should be rejected. + :param image: The image (as a PIL object or file path or a URL). + :param nsfw_threshold: A custom threshold to test against. If not provided, the default will be taken that was + provided when the instance was created. + :return: True if the image is okay, else False. + """ + + image = await self.read_image(image) + nsfw_threshold = nsfw_threshold or self.__nsfw_threshold + classes = await self.classify(image) + return True if classes[self.__nsfw_label] <= nsfw_threshold else False + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + + async def main(): + + assessor = AssessImageNSFW(model = r"/path/to/model") + usable = await assessor.is_ok(r"/path/to/image/img.jpg") + classes = await assessor.classify(r"https://...") + + + asyncio.run(main()) diff --git a/ai/image_segmentation/__init__.py b/ai/image_segmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/image_segmentation/async_dichotomous_segmentation.py b/ai/image_segmentation/async_dichotomous_segmentation.py new file mode 100644 index 0000000..fb6d6c7 --- /dev/null +++ b/ai/image_segmentation/async_dichotomous_segmentation.py @@ -0,0 +1,223 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 14th Sept., 2024 + + OBJECTIVE: + + To provide an easy way to get masks from dichotomous image segmentation. + + This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait". + You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This + means that the model should have only two classes like "foreground", and "background". The specified model was + trained for implementing portrait mode style blurring of backgrounds. + + The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not + support drop-in replacement for other behaviour_models, please be aware about this. + + REFERENCES: + + 01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait + 02. https://github.com/ZhengPeng7/BiRefNet + + DOWNLOADS: + + 01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait/tree/main + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For system-level activities: +import io + +# To work with PIL images: +from PIL import Image + +# To make asynchronous HTTP calls: +from shared.variables import http_client + +# For using the AI model: +import torch +from torchvision import transforms +from transformers import AutoModelForImageSegmentation + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DichotomousSegmenter: + + def __init__( + self, + model = r"zhengpeng7/BiRefNet-portrait" + ): + + """ + This class uses Dichotomous Image Segmentation to produce a mask of what is in the foreground (or what is of + interest in a given image). + :param model: The path of the model. + """ + + # Check if GPU is available for faster predictions: + self.__cuda_is_available = True if torch.cuda.is_available() else False + + # Initialize the model: + self.__model = AutoModelForImageSegmentation.from_pretrained( + pretrained_model_name_or_path = model, + trust_remote_code = True + ) + torch.set_float32_matmul_precision(["high", "highest"][0]) + self.__model.to("cuda" if self.__cuda_is_available else "cpu") + + @staticmethod + async def read_image(source): + + """ + Reads an image in whichever format it is provided and returns it as a PIL object. + :param source: The image as either a path or a URL or a io.BytesIO object. + :return: The image opened as a PIL object. + """ + + # If the input image is already a PIL image: + if isinstance(source, Image.Image): return source + + # If a buffer is provided: + elif isinstance(source, io.BytesIO): + source.seek(0) + return Image.open(source) + + # If a string is provided (local path or URL): + elif isinstance(source, str): + if source.startswith("http://") or source.startswith("https://"): + response = await http_client.get(source) + return Image.open(io.BytesIO(response.content)) + else: return Image.open(source) + + async def get_mask(self, image): + + """ + Process the image to figure out the mask. In some cases (like the image of a sunset) you will have no white + pixels in the mask. White pixels represent the areas that the AI considered to be the foreground (area of + interest) and the black parts are the background. + :param image: The image data as either a PIL object, or a path to a file on the local disk, or a URL. + :return: The mask as a PIL object. + """ + + # Prepare the image transformer: + image_size = (1024, 1024) + transform_image = transforms.Compose([ + transforms.Resize(image_size), + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ]) + + # Read and transform the image: + image = await self.read_image(image) + input_image = transform_image(image).unsqueeze(0).to("cuda" if self.__cuda_is_available else "cpu") + + # Make the prediction: + with torch.no_grad(): + if self.__cuda_is_available: mask = self.__model(input_image)[-1].sigmoid().cuda() + else: mask = self.__model(input_image)[-1].sigmoid().cpu() + + # Create a PIL object of the mask data: + mask = mask[0].squeeze() + mask = transforms.ToPILImage()(mask) + mask = mask.resize(image.size) + + # Done here: + return mask + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + import numpy as np + + async def main(): + + model = DichotomousSegmenter() + + images = [ + # r"/home/developer/Downloads/kate.jpg", + # r"/home/developer/Downloads/IMG-20240730-WA0001.jpg", + # r"/home/developer/Downloads/pexels-pixabay-57416.jpg", + # r"/home/developer/Downloads/summer_clothes.jpg", + # r"/home/developer/Downloads/low_res_cat - upscaled.png", + # r"/home/developer/Downloads/blurry_traffic.jpg", + r"/home/developer/Downloads/card_0.jpg", + r"/home/developer/Downloads/card_1.jpg", + r"/home/developer/Downloads/card_2.jpg", + r"/home/developer/Downloads/card_3.jpg", + r"/home/developer/Downloads/sushmita_card.jpg", + r"/home/developer/Downloads/niranjan_card.jpg", + r"/home/developer/Downloads/niranjan_card_2.jpg", + ] + + for image in images: + print(image.split("/")[-1]) + image = await DichotomousSegmenter.read_image(image) + mask = await model.get_mask(image) + image.show() + mask.show() + print("MASK!") + array = np.array(mask) + print("Dimensions of the array:", array.shape) + masked_count = np.sum(array > 125) + print(masked_count, type(masked_count), np.size, type(np.size)) + percentage = masked_count / np.prod(array.shape) + print("Area:", round(percentage * 100, 4)) + # image.putalpha(mask) + # image.show() + # mask.show() + + + asyncio.run(main()) diff --git a/ai/object_detection/__init__.py b/ai/object_detection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/object_detection/yolo.py b/ai/object_detection/yolo.py new file mode 100644 index 0000000..a435ca6 --- /dev/null +++ b/ai/object_detection/yolo.py @@ -0,0 +1,176 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 4th Jul, 2024 + + OBJECTIVE: + + To provide a class to detect objects in images using YOLO behaviour_models. + + REFERENCES: + + 1) Code Examples: https://docs.ultralytics.com/usage/python/ + 2) Models: https://docs.ultralytics.com/models/yolov8/#supported-tasks-and-modes + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import os + +# To use YOLO architecture: +from ultralytics import YOLO + +# To work with images: +from PIL import Image + +# utils: +from utils_v2.string import json + +# Common: +from shared import constants + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class YoloDetect: + + def __init__( + self, + model_file, + debug = True, + debug_prefix = "YOLO | " + ): + + # Load the model: + self._model = YOLO(model = model_file) + + # Initialize the dbugging tool: + self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self._printer.disable() + + def disable_debug(self): + self._printer.disable() + + def enable_debug(self): + self._printer.enable() + + @staticmethod + def _process_result(raw_result): + + # Extract the various types of results available from the inference: + speed = {k: (v / 100.0) for k, v in raw_result.speed.items()} + class_mapping = raw_result.names + probabilities = raw_result.probs + boxes = raw_result.boxes + masks = raw_result.masks + + # Construct a default response: + response_json = { + "speed": speed, + "classes": class_mapping, + "boxes": None + } + + # Object-Detection results: + formatted_boxes = [] + for box in boxes: + detected_class = int(box.cls.numpy()[0]) + x1, y1, x2, y2 = box.xyxy[0] + formatted_boxes.append( + { + "class": detected_class, + "className": class_mapping[detected_class], + "confidence": float(box.conf.numpy()[0]), + "x1": int(x1.numpy()), + "y1": int(y1.numpy()), + "x2": int(x2.numpy()), + "y2": int(y2.numpy()) + } + ) + response_json["boxes"] = formatted_boxes + + # Done here: + return response_json + + def predict(self, image, show = False, verbose = False): + + # Process the input image with the given task: + results = self._model( + source = image, + show = show, + verbose = verbose + ) + + # Based on the task, interpret the results: + results_json = self._process_result(results[0]) + + # Done here: + return results_json + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt") + model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt" + # sample_image_path = r"/home/developer/Downloads/2_cats.jpg" + sample_image_path = r"/home/developer/Downloads/flattened_image.jpg" + my_yolo = YoloDetect(model_file = model_file_path) + results = my_yolo.predict(image = Image.open(sample_image_path)) + print("FINAL RESULTS:", json.to_string(results)) + diff --git a/ai/tts/__init__.py b/ai/tts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/tts/easy_tts.py b/ai/tts/easy_tts.py new file mode 100644 index 0000000..fc4bb9b --- /dev/null +++ b/ai/tts/easy_tts.py @@ -0,0 +1,175 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 26th Sept., 2024 + + OBJECTIVE: + + To provide a fast way to make audio from TTS engines. + + REFERENCES: + + 01. Usage: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#python-api + 02. Installation: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#linux-and-macos-install + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To use the AI: +from melo.api import TTS +import numpy as np +import soundfile + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class EasyTTS: + + __sampling_rate = 44_100 + + def __init__( + self, + language = "EN", + speed = 1.0 + ): + + self.__audio = np.zeros(1) + self.__language = language + self.__speed = speed + self.__model = TTS(language = language, device = "auto") + self.__speakers = self.__model.hps.data.spk2id + + def list_speakers(self): + return list(self.__speakers.keys()) + + def speak( + self, + text, + speaker + ): + + this_audio = self.__model.tts_to_file( + text, + self.__speakers[speaker], + speed = self.__speed, + quiet = True + ) + + self.__audio = np.concatenate((self.__audio, this_audio)) + + def pause(self, seconds): + + self.__audio = np.concatenate(( + self.__audio, + np.zeros(int(self.__sampling_rate * seconds)) + )) + + def to_wav(self, path = None): + + if path is None: + audio_buffer = io.BytesIO() + soundfile.write(audio_buffer, self.__audio, self.__sampling_rate) + return audio_buffer + + else: soundfile.write(path, self.__audio, self.__sampling_rate) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + speaker = "EN-BR" + tts_maker = EasyTTS(language = "EN", speed = 0.9) + + tts_maker.speak( + text = """ + Imagine delighting your doctors with a personalized calendar, + crafted from their own cherished memories. + """, + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = """ + Every day, as they turn the page, + they’ll not only relive those special moments but also remember you, + the one who made it happen. + """, + speaker = speaker + ) + tts_maker.pause(seconds = 0.75) + tts_maker.speak( + text = "STEP 1:", + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = "Start by identifying the doctors you’d like to engage with, and add them to our app.", + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = "No rush, you can add their photographs later as well.", + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = "With this, your engagement funnel is created.", + speaker = speaker + ) + + tts_maker.to_wav(r"/home/developer/Downloads/voiceover.wav") diff --git a/ai/tts/test.wav b/ai/tts/test.wav new file mode 100644 index 0000000000000000000000000000000000000000..d38cfc00b1dfaeac3bbc500cb2dea7b17deeca71 GIT binary patch literal 162150 zcmeD^2bdH^vy*2w-CiIoIfEbwsGy)^kqm;M1j$JOMS>DllA=UO0-}fr5*1XCq=14F z6cqzm^1-Feljl`kJ8$<+E(!Yo?|t5W>~yHy)z#HKz1pc&ix$0JV5r9%cWplKndi&I zQxrwRr`ZOIn%Rw}7%HCX-{<*04FJZ1KPvN^lmPUPKdk5VmlN#GDcaEFnLpZjLH+;e zkJ{^I`k=jHN6ChX+L1um?YI7nQbt-?Yj!<%nQvW&SPDdcB1iN)7NX!~q*HLLf?z}? zC8a3t-_P6Tm0bqAZuj5(nLntTH$Up_P_UO+cft1_j3IIP2IL*{E}RDx~yxN}Qc`?i_EwWVdFi?9%5|kCi_!*!=X{^zs9XFu7azW>o%J z`}|pg-1ytt=LVCVBI0Ck%UX~2-;93V_*tplYAu%E&EzR4{U{Am`YZT3<7B69*K@}B zJA3{L_rGS9+ouKc&yAX`Q|vQ$_}qF6+84_;Cpi%&e{^kX`GdYWR90>_Me@8k#;vtx z<#%he^2RQx9=85@Qzj7E>D%~Z1he<%m+D`Kv-#&l!S1)8Il<+({qN8)*88*5&tA{3 zoH@bWE;~f&BgLF*(e|86S@Cm*%U=)CFK1ZNA1iUru=(k+>1BQ92lMZOK}n*tiCoe8 zt(LM<&kwcSF^GKGzw<*kzc8`ZxEW4%-lTpr$eXuig~!^>;r_nX+&L2YvVZ3+P43ik z#=Ch>l<&>^3%bJ~^m3w=pLV1Wg|p?0w*2>%V&%w z6l{xSecpaP`K4&fkze?Nu3!15d;1{K*KNJX^Y+osi&B(tPLKtY2%&SLl)v^^ex!Ws zH1o$9p<=n`AFCjc1+$NBfr4=OZ-TYux3$hbmf8FNEPHm0KSTW8Ew=3W<5Mu`{Lzd; z5qm}Jd6)h#*)Qr1f3FpDvsyvzl{b9e5IM^ewMfn|x%b!)t55v3YiiiOV&I~MM*mt#2)DGEZ%mb)N0 z|0Jv}U+gnC{s@;_f5G~qTyw(98zPEn!$w;QUP2l+mQ9a5?bh4-%uYMX{r2)9)R0?2 z=o0A)hEovwe>bcxzpVkO+gzi)cE8;deP+dow%@!|FzB1p$gLIW=hpvs_YwUHBE#Qp zk-Ri67=K&8f@qkR1O@BA{hU!JB16G!fjG833Z}z<6LeOcv+85lvclY&^tP>UjY<^T zmNDA$_m+@fEQjp1?ESH@|JTd5t^Y6SSTI=&2L1nU{%7~l*by(NZL-t)&(w(qw~M!J zkDTen^2-~Vcu>yN{=0f?eejuI$?}4YrD>P*OZ)GD%dS~o^8Xz&6ojU2v#ig8P`G`t zQ5o`2li=hZ{-1`7>J%;gb28-3(}u`fCL1!lhv$tM3xQOk_1i-;N-wVx;g=WGt@Oug zmp2_-|GXjpJ0bFx--aM|!P>29a;x}xanDXadp$4w-1}{7y>)v(F1a3Uxl;fCtBMzo0?tE zO=<#TW90QL7<69L{z`wr#VX|@BflTZ0+!we+_RhWI=i74UZ79{-1w&2`2f@ zKUQ`~n>)70mb1f=dfpH>$Gdqekv*1X?8ruYV@o%ul;74^Uirnj6*waKtx$-?we1kw zmbDzKU*6E!M?4lgE8U!GvE!8!O#a(*=Wpvnp82Pm8)R0Rb}cuItbMmr7Nm^yVmV~5 z$M)s7Ok}jb^NW=~IGbwpnLoV$7O1G)cIls$jBw4Jr!ATN%pLl6_Szird24i|lx`QD zD8*Zq?bal^9oVtDU^#D^1;@ypCgGYN&V)|>A#cu?;Aht&TDP(O++z1cpEgXi<@S|q z+Ii`lSATYiV&SnITh9)Q_1JzQQ!G?e%A1#BVY8P13{Tdeto)+{x)xXM(IcUV{xMWS!KvX%=VQV zy;#sFeH`ff~wf%>1cKNS8P0IvCq=HZUCeO^uP3m?#!HL%LQ_4!2VBDMe*QQCJNsYAHHi@NTSVlGr+0sTeiIp$PpOolK22ox%cJ9wusff(B z+_ADnX&^juvZ10S!k^G1tx<^xv@IL`zn+C~qz~Jo9%ECnpElH=Wg@mgI?)m-6WTVW zU9%}>eQY%5@Ybd(4zw&5Rgwz-jdtSp%znNU$UyUl(Qc%(sEY-;3b z+r)0QxgjjJ+19Y32@TuUgr;pf{Qn@>e_44U6@2_}jFj(nK||xnPS;$8L+I5j%2(p50Oj zJ2#t0Rwx^mXiE6oQp9Q+^>r46w8Jp^vO52*_7;d(iU5@F|$7jCTT@@BAs2wC!u2Vun*zbG<+3^1bZp8|yzG#qhrpw>q~C_JxyM3bAA!q0 z{}IZ@Bwm=cCAOE8?S6a`?pPv!u{zjou{}f%yT|5hKeIzdsYUT@J#3uVdRDqN4uQZv zfv4c5jF}D zTShBJdr3KKe-sZ(x%Fj-%r9(K{#mJHuSH?A_7KUlwqY$N_&-Db*%Aci<`iOei!I-b zZfqNo^X3?_tu|KzZ&S!xvs-P+WAUSXSxY%l$O`pu?OzN0Yk_|)@UI2_wZOj?_}2n| zzXfv2U9dfSU-o)->9XVGRlga&-Ig6Uw%snrK5h8_{wK*W?R+r$B=){_`Lol{PS@_w z-cGC;tR1*@vv;S z$3C%a*R#WCuM=35pkVg1>D$ljbhFo^e4@Ov!^GBYy0Ny2?a!%f<7Mx+>11!ush$&^ zoZxe6Cwk{R9z-ttJ0~iLf4d~gnMZCte>ZoVzHJG+elwW}46#XWaEO~%|IO(C9c^U% z?C(Ewi0U0J{Tb@7wGfJb4fl3ok#Vqfi&DKE<6~3BXO!~oDaFzz<=aE|uTm+PhOyHB zRgSlUXY%09F^WtAH|H_M6*8YNPQ*ex|9RJfb zTYh}rOqSUGoM3K-mvh_A`Q!}$-_~<`tPs_TSU6h$Z!Tr^hT9`)R+|4=YuK{f9?fnC zmH)Iv`Qsm*(+Gb%^!}%47pw(^1U^#u^Rk!O0iVz@?WPG#Y9@sTJ{p* zgsnEu+~K46QTS-v%}P-?yL2-syUm8l`m}qqKBF+C9rgUPu2CqvbTca0X7@**c3W2Z z*?HttkJ8Crvg!W0JytebpPcANWsa54rWvJ_6TS^ep1H$EWkKjzeo>o8VPgAjE(9(X zCbn#Ix1qB>V{x&^wqNWs7ACeFOUZ_}Y1nW!jNNay->e?nXZPFqu}>Q=wk@`tS3kn$ zMn9HTLCZGpSo*Q$?DTB-?Ct;7|Fyus7Wmf!|61U7Tfklq3;IMkqBvP=(e}JbM4P;z z3fdnl8RF(&x~z~yx~y%v$&wX5yA*k0+WoolvpFEFUCs_4{pEpuOP)~94rlYr>zS1b zLKDvR)0TtKA-{Zx{{F#+CB5X=E#zi#c6d^^zY#Bb1KGwVzpSwDz{hg7%f#wLW?ORX zvoUQ-*kX^6ZLwJ1QF^3g%WP);Dlw{`jf2l94yi@^u*a_1C6ta}5ZOpQ_V*~U?NKb0 zfc!F&STkxT!j-g=-?ShFgeJPiN`bJ%J_MiCi6w|sgpRE@_7fS%H=#=;v9Y4Rq(Ru| zuWHEe84yFy{AB%Ir)^7HZbHT8Yxg35{3Cqz|Fr;Ov48v$2m277e4`9Vh2Y?Ca2SgF z7p#eBgv?cKB99Nv{nHU;i7gW<3_B7 zRwgpYU>lSjsUjt$inPe+AvS?Ue?sfv2qI)uny6$bC7CNYo;+~3M;T!VbsR|*eLiGaw0i1dajP zlt>V^{O}0^hCSEFNFxsce8LGIq-^s;|3U3kkQ+x1Kd3Kiike|x)I!)!#vXaveuKIS znQ~B6$V#=YTWO#j(jjvLJzRkr3;g3KFZ_!GDR`i*WR8*nmbbl+q7*A&`9OyV*b#L^ zkFh|ufRzsIGT0Wq+XEkWrjMiECO}G1cLA&cm;oyTK0dGmnIXgnP);e+298V{phpl6 z45h(nAm0q&EQ73Mq|v@S*p@^zLIw2~flsW>5Z(d&otfkDUq2l4=1ZWvrk%k2}##zjQ#Gy?6 zwMapydN^%77;Y98q5!V~ii@ zxh&wi{;<2Asc||$`#}m0wD3Y71Gd2#5eA8fkKtTHInk0~po0=PGHphD8~u$$Tht7g z04B~&88Fa80w9Z?87u6VN#KdZ_89pytxs%CA{NdOoPi!H5k7HHMo$fcBqr#AG%VoZ zvu?l#=Z}>sFAcE*Es7k`liXkd^bsfE<2-O?+78Dm2wI}{flNOjQHjJe8em<3?6W!G)Bc4jnKdLrqUG%MmqcNt zrh_(tOx_p|$UH{hWHQGDZGp0)RwPD605&7j<|Hn$(1W9e@e5}?Ss_UzB=*Jl#)2(% z&=1Ee2o}Z&6oANd1O5d;DjsZ+L=~k9XU-qg)wU*jKEnE81kn4V<73asFyN!#5ud^J z24g428H@p*Ogobpi@M_6Ml0h;5Q#7{;TSQXWjsic2>P?f z(1URvBR$SrGSASmXlDb)5$%g*ve#T5hnLjqWgGVV-$ zbbv{*lHp$vq{3bjDbZ$Ry+p4>k0MzHN`^jxGa9uZaTKFIiEI}5j|XN*B7AcY!*JC| z2OAQNaTG|5!cif41d$E>oMddaKjRZe7jrg@Hl!DQpU6To0$i1`1?RFKxP>yUi+$Km zGOFmhgCmG*8L>IeXtH)A7xWc7vJe00 zDO!nmNHp6d^AT5fTyN2G9vC@$w9(!;3&Y@_sql~33?mNl0h9@65?N_UEV1VpM(6;H z09naVI$RfUy+OO%aU0{g3~;#027nf>ICkVkuO=gm@d#%xt_B#PP=1nW;4H_r4d;Np z&PC@~G-t+K1o@*CFn2-^#7K@&0P~}?%=l_rB4`#5|8S*2{tQreg1#PTLErI1yhw(B z=^!nxe5f6X`M74G*W{(iy1YJjRP!AA{_cKt_7HdAYb%@bQmifEsU5Xl42~y z*nyFRDZGVTkw%z|qc2LYW?la}u**%v*70;MkM(4Q+v1 z;(Wv07}p39B=o{RA~*Ue`WuP0wuhjWm`f4LIEU<1d4V zUnLQ>O3N$ggCja4m0{Bjm%q&C%+mICuqY%l#D4?DWKIH?M$!HKi#<3!~BW4}M zQY332(NG4SILpXPA(1I_YGuwY95JMiG|0+kujaTXwRcP?1?HNVZ^w`Y5ix zB%82zi!RWjBzy`3Wz57;?iA3J?51!I;5;W;d=ltf4E`kmZL*I-&X_IWZU^&h+&y9x zLk%$M61x@x>+&E$22e(SCnJexA{tpW05EjxDd znOT<(^(4@k=q$m^xCU^)L#)DF6e9}mhROJ24un}3O5yi7 zF29GS9XvX=WfI%Tt^y?> zYbwcKB&g#$f$-QvMw!eJ%$adjLQCU3!(D0s?1?^yql3FdjEW?$A-jeUXmJJfzYO2W z0Iy|c*zo{|XE0vyn4&Ne#lTt)7>SHbO|Ak?I+W25@fijViPW7`S*ko$2H?faxS0Yy znDL^`d_eoWbq07SmIvsSf=>y+VKYyZZe-FxPr#TJ%=AqEA95!_)Tu~S0{A4bl?jlS z{-EOoD|#}uC1+!p@#9L0Sp+#J48h1=$dBm|WECd+HXp!UwJv~GxR!*$3yT0`HL3&zAC zy#ZX^V3Cp_e<2vHbchS5;a?hXz?GEvO9r(3XdQ)rXMw6}xd0w1Rmk)yk^vDv<{_&4 zf$}-9!5P3ppQ1r_oUeu9A6ajSFJYv>D4_xdS{V1mWDb(Ni^Sa`RB@0m4)jY0AGidN zB-0nc-U(nw^z#T<2xCkVRhha6^eY4&h;b9+e+2vx&;1-A8G5=O>KHrG#}dJEB{HSL zejV&|8lvbWkc$HdTp3+J=_+V*0Wfv&jFK?Y6`;og9Z$gL9LQ~1@z8!JP?SMBT(>E} zE&-lao+=GABv>=>Tm@HMJRc$XHm(W2OiP^xt6l~=6iAr}9)hb5$&XDK8`K`#$Qc6q zgJBh;?gl+8!koSnxD*DjLp^Z^5P|+1Ko?iQvoKz#pybVz`W~tw^&ogl5=fa0vjkU0 zT&X>vjT6QlrN{uiOHg&FHdGI)JM}PF=8E;TwcYyE+GBkO@dsA{j4r9*RVS?Du-;q& z`k1X@o*abwr`Aeqg|!Zz2Y{k!-APrYs(_3|zzQLNz65-6$8LfpE&}{l)+g2%FoI>M zCR7KiC-n^VJk_6S2HsT_yt_WgTNFlC1&(Jxn_ploSg?Ofsuw`@rn*oq0MY@JH7fx| zrWW8lNVTRqQH=pF9XN!nIDoHDb)@=HPf<;wHO2bDI&2-W4uVc!T0a5B6!6pUtzFh; z>oee?SS6_YsCHCmpru$ltQpo2Yk)P;T4n7AYbJvy?6jsq%TQ~IwZQrSG(gGLTH~z8 ztol|ttB6(6YHdxjwphP`rEumJgE7Q)!VUAUHfU5jbKDAn4X%RC&t=AS5A^``EcGFE zhGOXiny0T(`>7?=FseC>bA75SHIAA`t)_NRhpBI<&D3maBJ~n{ub@7n4pKi*$EeQ% zdIZ%9#;zLp|NT^37{AU?Zv}QO2lh(?kHSb<5+rB{5FB*|benIDu=-eoLG~He8`f)} z(M0QIYdU=9S(`FPe!A7xs$h96$_ko(lL7r6wwgeTWu}=vGh(W+G{%9(_rs^Cb;ev} zPBMp>kDHy$XU!dEA)x+-^_H~+eBu+BSv>H5jrxT$sO!{L>O~mwd%%N};9or$vw@)R z5b%J>)N*PsbqOp{ioTz2L${|J(|6OA{`l0TyVA4i-{>05L}oX0l?gIunGcw!n6iwI z{(=62{*?ZR-b#N)zef)ROYWs!0R7in&8)IkL#vC`)GBPHnTO1|W@D2!DYLWro7vQQ z!n((jEX~wR+2jFkIn4EY0PiPjy!EKn$f|7l%suAk=J$a4x>?`Uj5Ed$#t~z$vBlVC zoHI(AkC^Sv4(8M55_7A$-dtq9X>KupHdDus7#3u7yXj@`#H#5w;tnSwH)|=Kb zt28x|`k0cbKJ<0EAG4g9&kSK|F+b3)=r3W!U!e|A@pL=-d3qu}f_{vyPv1p1qvz8O zW+r1YBiIvc5v~*$W_Po**x~F@c0BtQluxqf*llbtR%hO4?qjCWGpONKZ?ltem(FQr z)z6h3O0x2>nyeaHQ{zdq0?fhf)K+>Dlfsl?+X9_->>-9`n$vq=c3lSlxD&=+Gf$YK z%&;-rXl)cSj_8y0R(gHCwq8U}*Uo5ut%g2B=Z*J_wr09H1){_~;LR2PSjBF@{GMc0 zvwk$Efz3tpigDd=nRUQggUpxB55PWdm^TApCON24)FH}CkEC^aDN}+S!M@FIU|(Zv zu^%yYnUnPU^ec2$GMTbkH1&ATafwB_+h-ky~N$e zHQ+{a?{OQs_FQeYB;DMsq4k%a3Xcly2t~pl%FWe5T875xLG2#hZMVEZeb)EX2 zdRm>Vy{vy^oG`z&##6shzrq+=)*P!pM2v4B8eF$NvbtKQ%@*bsqpi`@c-;sXEzODM z+u$wln=hD6%o1jTS;g!DJ`@l0u?V%2N(2c$pns%OX@Pl|*~m0xe`LFItGPKa@)wxt z^!?Q1R#j`8wU-LgJdFHlb}`$OZOnemzQQf$zZDLO)uhAXIpHAxBUg`0ZS|JdjThd&D#lMQg?`42djTg%R4Ut)W* z^;m{|hcW58^k^#CT5Y{*^|eH6lKHFQHOlMtwO7^Zsz)uY_E7h#4YZ?RkH?KBW<6L5 zexcgZv*?Xr%ds>^PlI{C8Y0OW@Pzwd{`7zuRMI>HUNp?O+YpVSMjvCB5iulFF)kRV z48Kvsd=+BNPHQNYOjV~l!q}{$kI`Too7~3UmLHf<0Dl<#{#_r zTLbHY-tfvuyz+qBNE@u*Fg94d=)+7ZyNfH$d-zSkxsbLQI}&^|l<=8!&5LGfEl9^p*On`fPoNP8p4j{>EhEQ$sYnfcKc@ zM989gQ0u7xRSu#=NjjeP(1l@)CIjV}FtgXPaa>#Od2TY-mGiSNu*KN#nHkK3OnGJ; z)0!O%b~!365sOQmqz9$b;&Wn4v7b0eEGtSvDQ*aD8O78J;RF8V8Cv?TjA8!y!DHbg zaz}Nb)>~h0w72@xOWD`>Gr~Zzj`*z5jjzQ$%6`XeW_ChscCr;%j$O?ZWrotTsR>pG z^KOVmt+dxvzcOAas_c~~%g@PsISQq;)a~HD{*0mN?Ez30C)g9U( z#Us~@91EWfH;bHzjFIapcdJipH}tjUH0ohy0o#vT$34Zp$9}>bhpp~?)Em}au59I=c*lIl>Q9}es=$iU9inMb z`U*suPLQjXq8fpPO4Fz4sf+@a8N+U3Iqnrs;5)&`#SiBCvE7(!^l)Y`JB1%0dL8l3 z`<#0mw4=WCwm3@MCT?IH=<|%isXcEb-{_XM!zYB^i%eBcslRET=@*PI ztTv2~3kWyF>tZXhyU>9j%BgHU?gqDvKh1CCoAMuV*Vr@62KpJQoVDF3s}EBT%hMvC zhr5Scg=d8;N8;sL%ByM_{fsfoQYjDfF>{SM469;8Mx_tYuS4E2nmTCpvbtFxT1_d2 zewW@y*P_E#J9CNNQu{#tSgoY()1J^j(?8M&>UH(udRe2Q`H0nxsz$E?&sYQN>~UCu z8^XRtwMs$UFG6plyD}G;8f*==2zv*_nMttzw&iwn7rEt}hkKQMhPBuZd>yg1qm-+e zd%9b8U39K+%#ecO6|hS;p*?%l9H1-<6ivI9e8;u4Ya4I0Pv-)S!yn5%)va0s;{oe4 z`Z-P(c1i;sP6sP}B20j|+kxu>Bk%&q=<$1{`Yb1=3;spqJ5kZ;tY@1u*+m!aI9 zx&Zmy?97?;n7&49sKsk(>S6V`T1tCIYpxeI3Ym98hW|FDL9BAnVOVpIP^+jnA>&l2 z`(Q;j=(&u@wt{&yf}PARV3)IV*!k?QYpJAqEmBSaF_7xavyQ^ za$3@(QVHovF`h3=7t@~(Ka=tO^`loNUM_NV@AXpYH3Dgl+s08Vf5pO8RD^nz z`i`1LH)i%SzcSmH)ey(7F>baG>*jvq+VKzbKZ9j9bJO`W;eeFxEa&+o&h4!h_pY0E zO>oqZT8M88gZM4XJ?0SE=|kpo)T%K^wD-05>=F0&o1Onz^Em1quH~}AVy@?Kzz6aS?C#v3d87ikXt`Pe@ZQ| zN}7xG7FsX$cV&yRR@tO%hR-3Tr24&@qCH`pGRsgry_Am7#h7Gz0!`Da!S4pbo@Fmp zm;RRS%)HBd%6!X6kWIV^IqPC}I~!)paS7Zx_IGw5H=4gDY>;+3e{c_o)8b~wz2GVB z-sK$WXf2fyXYmFjSZ^sU0`Anbs~=oCc=7Ga=aYM;F9@!XCu?hs-qtB<60?cxC0=oi zbFFs0<-8>EVrRY(H;w&_9l^EWukxjZ(|l#V0(YJ1PfxYR8w<5mrJtM{-X8J?=Lg3I z*9D&leH>mR8|n~ajr9_Jlo`h^VF$CNVSS&)bY{vy>}^GV20O~W)EcT7{UKeI=>$3R z{j_ckFt_Wiwc2WXWfQDxN97($NO?;gptaHOHHw-6^O*HM)gL14@AS*G4m+Kq)bDUQ zxs)11Kf`olWmtzAa&@@cTp{i>dxTBkp5aDw23wOI%s3&>d6Rvge^#31x;t*I_q4a3 zw{YAFcOh5M@vx($^aAgt`)FKfc-n|-oi5#dvH0bR$$Z*p{@vk#(nBw4Zn8Gfy||X* zn~uHC{mv7PdQu}{G}i|93D2>8x$e9m^bm##6@)%~UG6*PQF^8|)_6dRSIS4Gg&GHk z2dW0%gna1!aH3pGovt4?S5UvvE14v=4yvMQx3=0COSI1FOUj*!SE;TvQ0geHm4nI#^`zF=IBAZ8JxC8`6jO?MnLYrgAd50{ z<{-qKk>D@2*#J9_E6R`MC-4dU6z*=WEXQzLU`(E2x3Cg73+7Tev5w;jR~b*0xPEco zxWS&W?uT3#98;u2LK63@RYqM9D3!YYs(JCl3+_wIwGUGl1g6Ly{U!53>P319Q-<4Ulb_#Qht_s=RJnAU*G~JWw#g^xu z=Wp)G zar3>E`ja>8t?Wdc#b%WIciD$#c})udYD!?xF{vIHGJ;prCza!0d_Hp)q(!6v$>=6X|9!&+luSFlEzwP2^*+IpVaMW1BW zun%y@xXFAwp`v(H{7U>#n98lD8=3Rfj*&j0_|TEicj0}JMe^OsB;`}3hU(Pn>n|El zo4d_%CTo6ftTzrBRpGR!HFYP`h-)ic7Sp8BjxLT)(gvX>Ka6V$KJqGefqRnQ%b(%b z^CS3Ud}r~Jzlt*Nq*<7e*_y zhqakk;jgHd(J~9-_{$-wK5tgPfm(hp%w7kh%-=xKnI(ZZY>R_aW!u zC-KYpw|GXV262~jr^ltlPe=&HukrTsRCl_?yZC-wW41GOw>CKFNNsWDnct6|>wIC` zH8o?X+{74aEw+Z38}v`K_w~oD@yrqKT|S+w%GRTukUtr8G2Sl?ch8CwymjKnxC=Q) zh<9<> zJZqh$4$-?AA3Ke&Cbp9{N?oO9Vt-y>Pf_))RAZOEO}n5rP;V$5)y`ToeW^Z3zpQcE zH_9&g1({PWDO=Tc+GkoNJ>J-9{!R(3!Tll(mKr!U&4pNKZYwt(j#q@ zirPa)8>=XsJDy`RxD)()!eCJs%Sh|QcZC`-BPY;{t-@w~BdCXUP1p6H{*Jyw8>}{h zJnuvKj9gTCOnFs#4?dql9$Z8_pu4T<^lG*+yb%6Ks03b8OX$l_<<|;9@vyYSLAzGD zTY5&i9j+eEADz0Zr{`E)`S^_Z-SJ(#f@i(6renEuL@dFtwQfXerJujj`+S2lU!G05 zv@_Ki?xxX3ves8w99|ae7kn^0M%kmMn_s}Ym5~&~Jjb;Xn>uTIR(YKX9THB)i}9!8 zs(Vg5Q^a{(XZoO7-e|8^fm0+GoUt|4eQF!!naGMz*WmuZ%E0@9^MU@sBBA6E1!oWC z)w9}SqrBCCIt3@x66~@cgq&s;*Np#?H+WX~nAbRtJI=JHcf$(&G@R)UFskTj>KY|U zem~qaTqklu9;m*qRfUt{E80$&lf(4>hHfTPGa-}h$34sk`MZRoLI7^@AZC{-@iEZ z@>AEJ&6p9YBX^g#g{ud@@@>u7=UX1U7U`)j(rCS#{-7aPJ0YTMl+s*%;?8>CiTB0- z5YNW zL0^cHGnC$11-+-<4?LiYvA}FWJ-`&<2Js3X;y>n}=BxAPxJhs}TZ1ji9Hw5kW|_N; zYr3IT(B`Vm)q1dh466&ZU$u$a7jQ~St8>*oS{qm=U@y*m3p=oF++MCN-$JM{m%yRX1PItT|u4bnhYN$clhO1{T z-g}8jekLP6Qc-QCxWaY)m(q-suBl?ixj>o7XYy%zxBR|xj}A9?xC!EY&dTnpo(`V7 zJ$*dCd)#rYJTqPQ!j+Df_%K%jX7FnMDZV{-A2Yz3qEl)h(lRWC8V0`#Ob(0+d=h91 z*PsrEW`@f})<*7?56N9&e_UHzuY1jMR#~bmy_pHI75Mvvv%)UnYyN(&H?x&G0`Gwu z!W*jXaB95(R^rS0xB4WbGMw^C#wl%|8dSbhrm1JN1;$I(L-f7O^UQb5CiW>f!*9wB z=I4vk9s8Zf;UuA}=cGH)z0XzJUBjcsB_vEsyq>rwkx%?SUi4P-ly&*U3}%=9M7UGN zpc^%kS6|#!ag z87dYk85$F+6}~6J%6G~8A`eH3MkYjFlk2OU^m*oPYBqC-9nYmf-d>8I$t`9lFwN+a zRC_Dg+;5&VTfw`?Ch(?m44l>cXl*ga7*FW6w2JB#Wsh22Ut?6VKBD58O0179%gtru z*^O)hpCFtUG^wL|Y}~IN%T?3W&ov9I(lqYDgo;ToClyLMp3u)*%Co~=*b{N~;b)o; z$`$>KQtDj)Jtf00%d&o#c~%=5uAlMsjS0zzt|`e6q+Wt6R>^_aLgVCZ+II6BdMND4 zd44*anmor37aB|Rok7=j*KTLTF~}h}+BxbtS2}k&&WZE*txOy1N&SFYUzL@BGDN+g zu2T2G`dUY>6!|6edvIhhA-FuSBrqmOg^xydD_gZL#sE0gUtv{+bJ-y@!=%tZQyr{F z;a$M1=F3)nx+Oc8>%+BWTQQ6240uEAhPMW@OGCYiR}#lNs=1c9db&EfTDkT*8#@m>JGtZHj(fjIsGZb4X<=eY!u%kwaR4bE(-*{~(g$3#a^=oPKTQ_0nmZeH{Z&1QrCo4UP#vC7Y_# z>`XO>sIh?lo}R~aOzLfTAAHKv&26wQ4yP8=#~2QFQxQ1* zYtP=ztcJ6ggUo)mC$EV|oPFF+x>ve%=Thf8&Q7lG?vkEwJ-^1S_Hy2Zp4pyp-Vfqi z#eE@7WDe`CBW?Uk(syS(5<0Jbh;J_S-tx8JQeVR~^TwSi7gBqsUr1k)@s9udP&Z|% z&cZ#2qgEK+;uAGQWZx%#|~KglN3rB}NM4cQtFY7QUU2@g z7v4WtgY*3IaNagdpQZ-nUXe|qXM!n#9|LCs-Gk#pC&E8QI>LF%9Qh;JkUJ~uls0NB z?V|RKwoJXEF4t$lxyrXxfZoKkWE#=CVUL+ck7MS*+l|)r5V-fzlzNcf$H2QlJ`LUt zzRGQs8x&n7Mu|*6&|5HVobFn=JWa^%BIk#zDa3qQZJ{eup^$8DJqZUN5G0mn%d%h9`y21bxAYq1~bNq14dG@bGX-h!3}jq{yY! zQ)+RoqxzTp`M(R$pfKuxDdFtdU7NIaQjFK}1*fx-+p z_xTxeR+cLQr;?TUm-tq~MzNPeaTam+a7*qVTsK@BUEe#KJJX$ayLZAKZkcZ3}-$(TbmK-7p((EBfYFPL`_vjLGC*_QUmq`VyJy6BXk2U?!PEsifjn)4}B8q z9ey&>Q2sz(AioOd5J%MG`e}0^^&FiJF?W~I!=9zh}YAor$l z2(rIl#D~SVgf_xap_=rqBf~ksHPQ8k>vh*-t|V7SS5sJnwt8B5lHC~|&_pP3`6(#%D8L--xMO*s$ePnX%* zd?7Ia?~*0ysJLA;#j(Z+F`E_tkaz<3$rbQNle-a_sjR5Cni<5D6iU#2~pQPtPjUpx3x zxWD|gGF+J;7m1vPHPs4M3De3oeI`|rP3KAo6ND
  • =4x^Y`%c_@1yXzb&2;4e_k_ zqF6HXE!{vNoqwEf#68XQqBfbMjWPP$8m-M&o2b=bpZA2SgM|vo!@~CkoBFr-PWr`A z{m5ati!w*v7iL1EgA+m@M%t*W^)$HYaULS^Y<-{6*7}M%NTI)^)Jx`wzrc|P(y5;w?OGJck~W}MqI%w66+$-Tkj^rmHB?*nI`=bQM0kzg24}{}+*sbv9}s>4>F$}~a z*cf~~yf<CQj_RwbOYuIrZ=;J8NjyW?tnGmv~Wf| zA+2*XhZXQ^_{TVTN0Rh7yt|I$=dvr{w#iqti|zwA%Z^yht>)%My`%Q2!pKSCCczhA zACMjz7I{D(A#aVm7Fyz8kg+VIm48*Rc4W0uLo1_2l%VXFuPSY{SM{dG8^%$iDcmfo zLDgb5a&N%tn=H`cNHJYpAnkC}bTx1E@4RgIqxg+ z%MyN1=$s%VR7~g=|E6c6V*vjR{jS+VKcQ7ICRmH<4$Lpqy~ZrLN^p5bICVnG_>|{U z2c)H@%}8I6@tyBC|FXc9z@ve^K6ggd^a&7ymjv9ACF(HaVaUPfP!r(RSVwvg(}gX{ zIpM^#xVTnqD%En_3oBrES2tIRv$nIhV~o^AtSGdDw^rTY76`-6V6M^@-0r!{x?$AN zN2p$9U!-q%ZsiXVaY-Z~2&@5J^_rYp<)L)`7YObIivJ~Y#tz>TiH zaJ%gtYYf!}yk{KqD%+Z`C4M07b$sa@?dtDZ;oRns94|^4;#_gGI7F;3287Rru|f*$ znijGqJ&H=Q-ZxZjyy{cNz?;-^+V3i*oq-jva_Dj2iu9@(hkYjkW5d0bdfE@_6nS&# zK%hh5&fwb6{KyJrxb~jj$oSUSX6~`3!5irWrZjs2-itNjw+QW|mmDRWJ)B3KS6y>G zmAsF~uTOX>Nh@@w&=X1N31bsRCyY!u19|_JLZb^GEc9q%(6dV#$g!|T|B>F!t`O!) zABlyyZq|PF)9_1yg}zf6`!cR&RQ5IUmG$-Xb@Vq2aDgX$ccw>D5^kJGE_JB%A)XZfw}&6fu*4V z@?kYWpQF`R+~HAy2mJl~8vILfwOH0*eCsf^ng(f%(4q>GM;Y z-e{M6_S(4Xi&K+)M+0r)-TPHoN#_So1u6tPhgOHrMqX2nYBZc(w}l&7o#9qVBknrO zaHF{U_+~;cv9Ht__6hqOA!&s4o^(>`AoUZ^!zocQ!Qv#y)+n|;vz_h(H+bHJJ5tYB zRm|f0m (1L3{lEfGO+sL!bb)PqVBxmNg#;Ev!cAtvmC-Dk;28CjK2DxCHxoDi=z zCK(NlxyBXq26Z0B=o0T0pA{>K-9^!{(zVlb8qO8E#xIV4DE`6tsR=(NWfVSFWJr;( z3$;re>0RY+=TsqMe-gYbZ)W{AFa9JYU(aY?s%B z(*kMm!=hC;+Fx&e{f!%!Q(uQ&!HK~0!R0|FxZXd&7w`Mrch-MjFeT)W+o+H0*Wny! zFT6cBDG$8`PWYS9^BIF3$fpbarB;ryj(Z(nOLs{VMMc;ttPrL{j{3Inp)glyDy-o@ z;kL7Gb~^Ju-H7_mJZcC=l3rX}qXyx`=dAW3+_##itdQ47-V5&xbql=}suW%dKkQ!> zSt!4!ELWe`yn1uJjDA&1*7`uM^Pts&F2`=>JiOkgw%yTO22T=lI5RBffP)g7GD3Usk)p*-m#=me+=h2PgSWLJJez31MVmOZJ|2c&9a0&!e(KK&{Mcy zXbkyjXJNL$iX+4?#YCx?R2t4EK4N~Ro`m>Z60(=?tc~=e%qqGrRma?;J+6EJZ`5i= zLXp$*v&x&wbmb-GMWsFDohkBh`8&BR+z++XYB0a%ns30)DMGUe;f%HH|#8)A(BG?(;J-@1qg?-LxbtUA=_0+ZMdsOf*kK=xW(}el(p1B;nL6qn{bZxjjve{~7 zjM8SP_o)x5)6{R^PG=kS3pm%Uqr9TjRnMx$v}+E-w!&+y~wAN5*a3kP^c_*BH6f(;h#kHPt>riF?%=E5l#nMOl`Ug*h z8F^!*1kATzeC;z{O@BW9YWnvXHT|aovqDLcner)RtvXx%OxZ10ksHZ(C?CS^csbl` ztiLLU)cSA$`0as!~4FkghFuJufMoT7zgXu3%pyH${%1?(HE^# zkk_5D(&$GadTiwDv(2q%w1^xpKP=Z&7O0oCclG1?Z~A$-Z}+UeUz?&Hl>0^=jJzUm zSNj-6sY=YxOk<`6y@+a0O^5q1_p>aV9xV_*l74Yib{%mKj;rg9_kI)iecbT4RCmbv zz2igpEOfMWj&wKjHcObAaM(N6-5O4OS5SAG{o&o>ThzU9(|}>-(nV0jvk;ojmgxRW-R+sajkeHJ4y!eM?b|0%zj zZ^BRFK8G8WLzrc7g*=(+Nz=?&W;E=TW>H~tl+jS{qxn?{?gr~{)3XY^X}hXz)rM%i zHVNLStWtu?IW-Q>rMs9d;5QN){Py6m^%ymgKFi!-CvqltlJ^Pc#BEZtW47yt`*Dxc zbH-f{_Lq%3va6Z%XGxGwK%9Em*~#;4{LX~(@ejD47LPNNTyJwp2EQt9@^#mrAEHcKMD4eha+~5uumT z67FZH@)5b2yf!iivfOEqqm(sRYckajZtjkRdvQ(R7YWCx9S{}9!m8Gho5}5fQ-&qn zShxvylzW>i&h2EsWp(x}+m&6!{0295y>uL1lYRoQYt#MUEbM3Fcl~Mo8GXHeRzIWf zhI_4Z^;!Btc#hUh?FsEL+_yRe=i-kT=S_iH0OwmYH4N^Em!WFHz5fPmAFdYP6W+;I z6Bdf)9iyGkyHZ`>yQaBDyPkq`iW`n{j*e0_@loNr(ATlmy~5idVPkw}&wlX?{haZ* zc0hZ{9La3qS#h+mj5|YLGuP|gG*LAoUxfw+>jjnIs&Je#OnY1R>OU*5gv$kd8KW{v z`yTgugH^-7N8XixfVb^)m1mV>a(VfkNTW#i$S}EsI#lmq9*3JRL#bb2SM`wPgxIB?0B90#%!<|Q6@4L6VsySK--MDYr zncV9_#L+XZYGSD*!~3J7ExX$6t^2hkx?s-M9*;~7oX+?<{rQY#{x;xGot1ahXS7yY9rcL3CbBDhDKr9dmGsC_ zbsgNSngzdJ84T|?%bB0+o%CwPTc%}op${=7xO1Gwt>sE`li@^WG0d$cjK$1hOL6aT zZoWLfmivjV!X9ASGVj9O*=lrexaa&9+!5FUzcJYjw}T5=AHuEZ{l*viUTq}YK(DR0 z(7V9?eWN~A?+W)|7wd=NbTtIt)>lu~OPNjJhG#>##pq=_!tJBu^zY0TwhK3(e?S~4 zosk+jPCCXoQ=Nh<%~3)8j=e?~VrH{<3XeONc^f93PuvpsfwYL(VU*PdYVl?|{S7}t zN|A<$_w#+&Iq(DEYjB4}2`>!I2u+EU(k{Vib&zgGy`~?IEDeiuhY=*W@bLb_EK5)W( z7=G8(i8@EUL05ranG}Jw!L?Nj*_GV!|~zN@GPa3!O>5% zqu7&FKYeJVdEiRM*BRUWUxzlxAHiM24Y2mK)m_@N%13ZY)G>G{a6Y&wa-X&mZb$Zq zyEJ925=Kp}zIs6IsVA6&t!32D^sh`G_6_*OObhrG!+u(V`-~=)MmK~vOd3~+Kg8X^ zJqWke=fK;}ci~q>6_{V)p7c>_7gZMSkoSdKqK{B-THl$~;CBBhBhkFa^ci)HHu`q$ zLl~Ey`c-|bu>o#QJ*0Qh($!kfmZ6_EW|(KqE3o5P4fof5aASEDtS}ehwon)52(y|E zbK`_}#PX2umXT^o6{YvZ>4FPZ;QQg%Jg@LCNQv&E-s17?;wm`>eg^fM@v`x_^)hpm zFD3b;UeeD(Iv3A2rR!Q7bxFNTPL7OHI_Yby{>*FaK*nXgs5S|&4-^j+3*LbLz{@42 zzV;p5@%S1>vyAqc@^a*%(3L>Z;BTS-A?hsPtgN2L{pQVGI;6X#yOeGvMUW6d0TBVE zRU`x?RTPv)KqN(^K^jQ`0qO3JWtZK(_rCpq&fcH@FZZ*s?A|yrXU?2CGtcadWwI@I zExA{+_oscrJZ}*1N(@fasX+$eA?CUXBQim;S#8~*eJJzniPL$DGy9TxXH9-MQ@C zC)=a5bI$H(zixN2H`-O5gH9`VwR?ixn9sTI|HUmvCSnWeywroUY;_{sva7SzLg>%| zdP(0kvKQI~_6Ek{MenSaP-l@flTJeN{ov-r%0zeFNf_$iB7f~%v-a6*yw&Qb{;k1( zf)3Aprj(F>VMqGIT4xN8eHMFQth9?s#nd|5H_C@@bz@Ijlb0o4s%g2S@0jcCq3m4e zEX5jaCSftZjBX`r^vm=Kk(5|NyO&o*E-ruOnf7$^VQdj6u@d%Fw*eU$pUYF&``XEk zh$(y_^^y*F6WI65%3HZPPSW!0OSA>*$I5(SW$(*9$z558{IyVC%BQ#w>#O{NxBd=U z9izD6=}taYIr7zdc=^2DF8i1}kj$AnZf5r!`44_)o;||eZdV|)=l~fr`N$`F?Cf{W z@!hBHN%uAMV_WGV@retG<-H^4R4OPRC?(Xf+Fe%eQh${|l|Uu`2l`<3ANiORk&Y;q zzBc$Y;YOxKnKp+X`0mLy-Q~nGwU!EMZ}=Ys6h<(Y@2vVj9xVOsPP1E@no-v<%}nlE zxg8pOj55Q$VJt|`$qjnmbUV7voMpe^MC|7F39GSH##|Ds6D=9(klrBuNaT*u(|O|E zA}+9`JIs2`Xp8rHxOu~F>0OcDCLX`KTm#8#!fH&EyE1|oiHbQXH^EnbP~W0&(mJYz zl=sNJIU#?mv?v2Is(Am4pxOF8zLU9_Q z=Z-itIdM(puKr8wZR@;s-2RQMfVbWDZovJPJc`QBXvc77lgqTpD=nRqHpqR*aJiy9 zRj!fivP$u(f0JSKfzR-Jfx^KPf$@P9e{Y|u-XI%ij_gz0=u-oQ6IN$ho#~hG7d};v z+U2d~mgH`bm+74X+XD&yueH~eFQg>=@Z+qj#-&(V>?1Sa)Rq*rhGr=rc|V$oks7I2 zUfxN2BevcOxT<&1o#*)MO=iUS&IlN9#D0x_6>SnLWj;d6`^ih>PoxI!R%^SF%TR%B zw0C<+d*s^W;#?&oVu4&mUQCAbWO8c8dsC#s%0;!3zEiKM-z3hpDp^dQkX<@i?ueGU zK~JYDomkb^@Jx_rL}uWt$Y67C2|2M($;j!*Xts23kQ2GfnNH@(HT!S-ft}y^kodG5 z`1wk@3!JNVH@gQp7daW}X0Gk_AosL8IZhvVACv928A~|g^_42hd6Xy0Wc9A```M=akdV9YWsfE1WWpT1$|h zpnbufi^uYS^D`M??_eMP=sYIYyoI}w+vd;6a;_znG!LV;V6ceXYUHXWl_Fa|%h7-JRBQV|`3Dx|#0~k6j-d|1CL%Jnr|%e0xHl-gEwP zcDp&G59LuxCG6S1lx^IMOUX8T1D*K;8M(*sNSq@&wuh^`8=a9(C$h)7up3F_XSOHHw=lqqBhF9G zZ0EF7l>FD-uEU7FNmfxiv`A-W=`>or0v?2e(rxLPl#>(A_h_2U}`>VG3tJ5WX0 zWc-mbKRqWp%xuacfy znd)A(o|}JI{hU_bKhj2`0|qPA)b_*>y`_)Tjw!n&(|ygo=N^(8tCP9)Tj1-e?^b&# zpVRv`WW*I!@?yvC!B6o6E9hIbuey$0jp5SwUPUx-K9~Z<+I6{VME%zk5!VqVmGuWBoTycohX6==L4sZ>k!fHk=w|9?h`L?X6rawv+$-le$#>j$ZtveyP5pR#UsF8;D>3k~`1FdM~}4 z{yW!DdDU9vbbUq!Qfqa+c3S_)*V#8#Kc~(ka-fD=jJwLl($Aa)R+67_Q0W1_BAN9* z*m-8Jwl~?bGu3H9>(0TC=-_2`zp;y3`OFPQ4kB>AvS(sVm3D7CeVwy*EBmpvn@HF{ ztzFhV>jS$czxx|4(a1C0F76-ByUy2+?aXmsr=RVm$6N7w5$+xzxo3%AwurL|ldB!z#Pd4_qq2W_ny^=QE!fXEP-#44kx4tOpf9ZByy3t@dW$pCS;eY(jcf6 zr|}2QXdEhGr22IlKkX>J;0vWM5|Zv+ojw;ehOpY9rWi` zatUjZjoMUJ<=@c|Mc4;J^r?$9G9%OQy7Yp~=BJF>Sble!7R`s>Q5m|&yIx<$r8hi@_WZOb`$0`I6ta0T zdawjPnT|wnVW?E$+sc zbZ1n*gA8$4+D#_nIHKN~l2h0ZdtxS|`yJ2g$Jlh?c^&Y^Rf5x!%!&RE>=MCN6x7ih z-e0s}AGX8+{#^xwU|TSm@`U*n zGry1dTl(W z=T%~~%JTIp5>b%x4wH2pXF1;Iy;XWDNWp#xMNjF+zx3!-27Bl!1fMu*S`c;O+%G{# zk{Frnyi;k`XI(7hTD&eX1!+kD85L|*nePPG>KU&-!z*7hibWWaGOSXYcRs3MeMlR$sSya1;g(@p0r>;3I^AI1Vq82N<^XrAuTg)Ol4*s^ANn_ z|A^#*$0o)$&Zv5U{D{^#^j*;Z9%hW~V?IexF2%D1mFxi;gq#{lX7Q{f7yYL}#u-UoPQ1wv`^8Wt|I|;ii{Wc$hVMnRykQzc_1FP+A4sQ_%ae z(u<DZ}||}c|gw%)=)9lM;%&T7(IEGPhAOz zW+Qqi!fK7rL1tvy;8PzUZ^voX5p=;bMzaL#Sm?tN%v+pNEx4N58OwU;=#D&E^SOdM zCRSN#)@mId_4rxK40)_d%L*{l*=eibunMZ&vkZ+S^veszORT+nv}G?l?F!npmKpmS znZNdb>*W^u=n}0F+Us9*j37kDS>p=Q#RViS7t$!Wnu5eCIFR`nox*%-d|wl!OFcus z7ey*_@tOyWg3!RXk!**Rl?9m;M9{c4JjyKm&1VTcDp-p`|2<$#exrR`n8}x3HZ)Ov zB%~tmDpt-tBrn4I6l08=A~zkFiRu}0C@7+ZGiIzTqm!Re(3!(3Jn|#;e)w}2k&1`V zei~%>GhQMgQ?NYxVP)*#J;nOXMc-^@Q?y_%`|AiE)0o4v><;h10Bb<|&LZuru(`%F ziZ&TY?HT`?d~QB^_K458mLcoec(>Qst@`kIo3>`cE|!sL-T>n<)0wtBju5!)~JSLiD60eJg@=Mv&dVS(7)ISs5)*ig^&s(LB65 zAMYWwej1W3cz>_(3bBjlp@nM3{|h2z6&TrK^f-)U+h`KOHx=w^QENl6D+}^YS?I5@ zNRJ~kN9fyq+A2t*5~KMtW5>Hp8xA1nn^}dId9Kh%aR%WPo-Ztu+sxV%dJ?6bf=6o6 z4?%gBGvr59q7Vd0mG=}xVIjG>GWrl-e+BrBAm9ovwctA@Ffu~ogjFDRhQ~cHQLFf{GdBb%OMmjj_nfGklExW5)0fpCDMMd6_k_iW2F! zpdt%$VVn>d-zjqOn|KWfLC%g_ce&{CE4)U~J5|P95P$_;TJZG*)h<7u7~(gA{VQw~ z(T@}~jTi+%m=)ypBw8sXTToF`GhQnw(Sj0q5qZ1F7z({EG?U;k3qq;b%Y<$5knhr2 zIXRfG;ym*4EQeQ$UE&4vDn_#~(ou>r6_nka^j-8JP74=%L(Ys>mOw^?btmQ}zT?DM zk7BnKWK>a6MeGn_KNDO_q0t4uQdAd+bL+)VpYkQF2~n*?@NETCSnSD3^jJ{1MTHtc zQWZO-%kM-b1o19D<|HR=FTe~4*^Tq?g;w_1orF{up>IX`*N>g>fN^<{F{6T?nT1yf z3pB>(Jfhck8JT;0AFpyF_EkZrewi`P=aIDK|>bxT?Ers?1bV(ai1O} z)35)CIrf?+K7&EDiJ!-7#fUS2AkD@r3H-0^LUP1@7$>_5-f_I5L%d>$VAPAA z3tbwoMk0vdf+sGB&GCN)g*aYUK-gl!J{4<4yiU~j5!8NR4aVzU30}XLM=@3^e~Uly zQ4?!V5L-nRhj<+ju^z?P$16SvnH8E-tXLuMqE-LZ;}A4`L7$g-x41nmo+UJ1yyAeU zNg$pr=0LF1#f*tLwOLPMRs~^Se5T-t+q7K#G){?*S5SycjUYLTswHC9P@Fm*tUmY1Sf^qt{u(#uN0EDdk@0=jG%)<5(GaybD@wJ-5&*C;?F+~U=u zO+v@SM?G#IiK+pV!V(fF{<&34`QwbDP8Q_i5cS=*S7I7j#rTqcASu| z_}Ikh)uO@!*9jR9ArWGJ;{1M@PZhjs@!9d#$45@|H(ql=Xp=bQTD+t9typ8?dpBc@ z-ILeXn6N@9Pym!aX4d6sx)G9x4`Bw-`P zDcNF-Ma_o@vliFkLc)ZWk5~2)egfg05IR-pJF%O`HL;K@L17oVEZ+Bcr5Lf7iBS}? zBHmr>J%V5V->MOP5%m%TeLg;ZVm-twH2qgGNz9QLk$4p{v0lXpi}R8=J&Ln^oNq6P z$D$SgX<N#U7YTO&KKSyA$c4NM%^0r|B6Pl(P2uGf@0`f! zh*d0RQ84b~>s4^^#d-|z9%B89QU9;Ho~W%5XJ!lCEi{1`AJI=S5<&(<&xNduwG^*_ zD4rMBw?fMZX%rGGJQr~-E%a>MY7*Kw?zxCpFA|?BR<7`L#CVVR8i-4!Xyt#ELvm!; zd4i5FG)-Jq#P@M8T)g^#Sbg!`KDC|tC)YW8;E^L=&bnKh>tc?w`#N{VvhExi< z5^GUZ&&f(_@?j+j@1&TI_(|n|6*)wo;;T>eQOsSu&dDP*#$B{ST+<3KkMLt%qhu=L_I9gX0dLC+zMMKUZF*JJ>$Ex z@QjF8i`I*ML)axkQ$EVjYe^Z}NW4~friAtq7PqMIB)d+ zF-X)F64f!pS`p(f_E2%M6V(bz;&Ca*d0L!D#G22|IEs@?UVc)76Kf%udMHF(zqk^V3IlB>8m8pa*d2e9V{k|EWfzdzx|=>u5H1BvMv!B4tz z>(mtLdKn_Z@?;R$oAMJunqG|;;UPYq2HeJdijTh&UcSoot|I?_3^n@EiXGnciZpEI#kPE>c?^u?IpW{-!#8gm6+QFauEoTo4xGrJ!=uUgc~ALBd*62jUqXCj8Fd74v@7r~iRk$ld7hhzXzEXHZg+V&bj8oO`M!h3+u@dSmRKK{ z6AaaO5^H3*M$8P_C5WIs>~$qZu&*|StgI=%7QXNFi4bnzRtCyzkmrr~q`t?i_{ggd znW7bU(3fG#mzBSgmV1lcABk)zXz#bavRYZCc(jE(CEKg)m5u?+@RnR$tq9Myj-H?o z(8j65m5=0)q$%DlxbySeTW)>)URROY=e+-W+$m;d9zVyodzi0JJq=dGP2y3yI@6qF zX9Tpl_2l7fCt5~C#t1sV5^l%qK)UEE?IdcVpBp1iAsR(s{E19?@i4WZ5Rpj=t z9+7@mWlhNf9e*`rqr*}@#l7NoX{EG?duVYVJqminWl19z_BQO(ZmgNbM7b2@MzbNL zdO^|cg4F$pq?ec0N%iF|a*F&?J^+*6;5IOYJH`IobB@Mqy@c6l0bhSUu^3a_Q*H%b zmEzUpp0E|GCm%LMdAx!x@aA>lb6Vpy%$ngFx`xkgKkH&WzVA_dYK(Y~iA1YRmUzM^ zaRXPMm06wD-H9Ihi3sWltF;u-=s-0bj{Bcs*53vPe5_4!UM=U-y!pA+?d z!R%m~#&e^dnQR(XE%**+y_ob|UPI2&BCXS z?>M!cmPFS)aGEgRbKEb9X1z_UNH^yr_~sR;y3mN5l5E^n4B|%Pd!i=lQr{srQSP&d zX&+BS;tIJu@m=H5asQwnJ@0+CbqF#iJR3|xObQ!T_XaosGNgnh%>SO zV_lXYBD4r!Z=(^edc}xyxg&KUR=b7rC*15cYF@JPCP1XVsg;Jz&{OZNPu0)r`F*eZ zn)rJA_LAAy+uzCmn*V+ON&g$fZB`*dJ8S5EpsAkSs~ru#SpG2UovAl--Cp)!Zqm}o zGyAwSkQ@+K>h1mIR+f&bJp!c@(i4A$!0}Vys`jZo3EypX&4V;GYtDEumJz z&QP1cT)nC~mrSNv@PO-UqxBXrDEOnFCwr%pQW=JHbt-opl??Z5`=C)gnkD^G>H=5+ z7ipsvxL>A(A)`?e7gnnQn}UVtcko>=b`xgt8~AQ8puiTW;} z+(uff$v;RNh_-ywId8pTK8_jD1<`e6*?&VOTYIyM)!iQG{OX#J3416x;o%Neo2vI< z@vf8W5leH9DjA;;opD)e#+u0o-)t-~R8NSc?g+(puDjQ{V&}L2v68Gt_5xdVejrxw zg}dAv4wZGDvP^BI4WMSkW2oNQ)j`TFSRl>hOGGi;!b-V-%;{+K4$>0o7KG?g#O3H- z1-BC%`}SnpG|Y&ED#=drIrH!}S#mvhyeQnjTCpyL9raJOsG80}S zDzv72nEp1#&S^(0i_@aU-gsg=b=veJ*>Qs$4NmMRw~d>c5iP%qJcGM-MQ4)p(CO|z zcBgoi$ZDA>*F=*(S7gY>iKPQ|Rsh?3-;jUUnn+H4f zj^5e#t1loJ_<`cVk0E3A2`&tp!A_xrp#fnlTr<4Ue@pr{TK9SPyG^dnym0lx&({V$ z7@lI7-^yYAZ@s;?N510?mJX}40;dzQXSOo^m2f$rXk)!S_F^(K{!uRa%E4|LkoZaB z;)GV=twGy=L${PN-g5J0`tp=UNwuFHefsya%}IBYpT9hw_FLp&tf|%9-79y~C&H-> zhtq?9_-|-+m41-LW=n&Kea_^c6Q~fpMV$IR-x5gbuV`PW%aktiNYCdqGJk-is;A9K zHB(d5nv(5Z4=&Al%e9-hd%bt$v&su{#(u^+{7julzWQ)-7w#&#waR*wOtYtYNWTV$ zcrWz@=3rqr_sY0y>=$MOBPm)rT0HtsG&Ap(*KXs~p_0%i+{E9_(5vORX?+K~bqMxJ zfSie4#4Nr~K1mtk`MZ*5G*lhV-*wfyRQFiL4z~)LRy}n6WcP{t0XLwc%3dNec~4r8 zRE%ctxeb^8oRXjvkh}4loLJ?8gjAL|xWj0%K1f_DuUt;ejS}86A_LbEp|}^SWnQVT z+@4AzJ6YX_u@iO?_arJbog>$MQVoS;c-{F*{eeiOl@qv-#O7Fu}3^dm*$2O+uj~#pSSxuKHByx zbc15#|4noH6F2#_v(Q=2*BBza>!7{Q!5$n7@v#Yfo@(fxdt|(L>=*a3AlfadzmoEn=&r53b zJoZdYDxDlm{V?*86kQHT2XT09F~hNDaD=y{@1*GTM*imkid*X@k)61Id#~ z=Dz+OIUC&^&F*R*jID*3Iwo2>*4p?Kw!t{7wH35-TPq+vraLXYfl^iZYpm#7*k}qT z>B*cgMWw2m@836;b7f;)X(v>S3x&#=zKw8-JqDldDdj7^#FvT)k9^tRC+FvN)j-naJkIiAXT|UG(i(f-%~-VvHj9vntg{Ji8US4sT&$9VdqUzWakWR_aN{vra95 z;$*!lWK)j9HZ3kEVY$|Zo%fbE(p~6OAQpR#Q=16zC0=V{Y*RSnufcZhBNr!==8CH8 z#bB5z{w&nz8G?1yHt;y`eXv>RS*U#YR;U%3_a6n%1_}i>`;Yo#{>A>nzOYJ_vuKW# z*-wt%e{}cD`}ZI3e6cqDnAyp_NTlWnvW})XquhSdOXX9&g}+qbwtoV-)MZt2?zEeF z;lP`rE#dv)4xvkaN9(ARl0I{@JD05e<{uEfT1RI^7DOsVlcLLG&5gCj8lx9%ndZhr zqlop)PICW{9?Pqhg4i+_&;;Z3wZ8rSVS(=gw*LvcRC}@=&&$6^gT1=0WAC+oFjw;0 zZ;eXka&wNE$LwiLiY<+1iM|)zA3I_0ghjf`O>^(MYuvG%4ElS&Ntvm|(Te&z?_o$bTIt{DTJ=z#5)f9(G zQpbP7-y>iJJ`Mg5oEN+o>=oJ;x`rnGFg!M#C0s1@GO!*QxTU?O+0i?GM)qjMw_gufrEgu-`TB#CAkaMcP3)Ss4u&jm)4`z#3)_H8zvUxHMMEc-=f^ zwRGNK1U7rerRvHB^&7p4e^Fpo@CfA2N&eG%J1wW`k$==xTIn5f7d!9RwP0YpXU?Pw z%s&uJ3YcFQdt+r{Rnr@<$ZF@63H@NkB{p|&Jh#wb(n4uqFKI|=D?snL>-qG+F&>x7kqo6a-Bvl zUkOcxjV1LTc}RKqh7t=Qr!JQ8wnM zl2O~}u-JOzsJV}Pf@6>mnvwHz#-8K!V|Up?e2+|C{AuzIXOWK-MuLCzMxtTXafba- zE+nt!j94BYq>NwXhBR01r?gOWQW+|XK1e^T7hpGA;9n7F9qb-70u2IX{Smz?C$5it zl>%);jS_}JD_oFpIdm`(_LuRUgs=FC+CX~SdYRTCsorDt;me0dpB_qiCpOU$^~vkH zL#)%WbdPcKB|7K z|LgnJKiHe?xH#|Q2;z;^7vkU~BJ8llAnN`yq zV-B^)dUcika3NZ2x7FHsNa~90(ZF}mnXYDr%^bCwuzOWEI=e5;;5dTlRDEJcP$D6>sJlBzYKeTZxR^ z6!oI^E9||E`UL$Sy{qq{?=8RKUlQ;FzTok|r-6=vhk-DZg-^rF!uP@}!XJjV1*k~p zuk6dLt&wWl*CVAeP2(m~*vlGjZNo=A zEVe6J22#cT$Qw};2Kt#;2cwYrCe-|!aPC(t`L#Uy2YM=*p3~IIssrI>4>?s?mDlA? zWVL%uTCdN`xI1-pOvPnt8T|bP**>v)z{Ww?-Wo{rG;K4 zce~xlQq4Ms6)Rx82eoP!bb^ZJXXZNdl3Bo-Z580$G@S}}>+Ks5`eo`PCQ+ZuhR#$| zUM%~RR-EG6ut!aTV9-Rt<|RinJDK%k)XiFT=nL<^6(H1(hNB}4qHtw-b%PWNd2l&w!+f7Jic+-EgYj~NTy zD){I7dc&;d(UWO%`e)IYS=A*L6|b{N4WO;im_mB5KoWf2U_$xu`C#Wjs_%Eb2NkIr z%iX=fPQac(Mp+Yl6CIrF&K(FA4`Wqg=b|g3$D`F^!(y#s$n#}JtMJ=&qWVxnxmAYuwVJopk?r}^?%HA;BGhFNBY2m6 zz8tK=n?|xx#asYKcoiO@-7r#eavIJ}246Q$sXs$e{)TFbKa@T+#PHl z9EPN}pfX}+t)i6A+L@j^<^J>j&u%?i`C@$9O=Frf&i&bGV;kmM#)0U}=osTyyE2}| zC?~K&oWMGeTYXeJ=x-Ss4CQ+=l=i5ooxab_MGoE{RO{X6l;VB*SPjj; zV|}8lBFiK9BH5$8qM2jS*wzfsb#7R2%iIolSBnx$Tuj=5C-)tEL3b5TJ+C#@eFVXL zhPN@3^e^#=Rfs~?vB8q9oaR@t>r|TQZ4R>UV-q*PSGR{ygk#j?SpyIB5Br|8)=j5& z@Ms9T-Q0szR?9XdXJEuAot5z5Z#;>tptY&l=Yy5cnv-F{n z+0jA93Tvjb-+dR2mPSnVAnuPYcyqZwyD1Olc49cE&vjH``V0=)Aw^a4$d#l9UMV*P zBE%W0A$)ASZp^@r%?qWZ5Y>f0wjP+%py>62$2Q)oY5!_pg+VjKIm>BhtJ{V<;)QV7 zufsu|tL(tfyc}&jkv#C>%0;s4bLriD{Dn#wMFoc3DcvPl`iiWxd2L$^0>-tV<#nry@ zbG$yOZYlByhml3N)P2X+1xB*7Bb-yTGi|wc0TTK65K{)Oc&rzqb-bpCfso5QVJ=Jm3R320si6%$~JgZ z{j@s1@BK~KTT+61gRcLIzC!D(O=cXI`F;sBhMWEwMC{{m@9zY61iuP43FgPAIE89B z&-{&iKP&51?@=uT2w~SiRFVYL9e;Fxg9JA&-OWmeq zO%|s~h2@l_y;5F!aM*;=> zwe>k_dF7?_Ccg5UcuMva>m@ zAd2SW@t*S(M))2tnOYR*I0tm$hTxF8i&b4z-Hcys z?7DV#`v&>aWAU$jXHGFSYlGFo-ia)Zz@J%;8{Z>jfEOSt;YX~f>C^zu3&q7&0PZMd z)r0B^?F*O{)BH~0TyQLV#~1z$@ZpQXle=he#cHx_8Jb8eNz=?`5^L9?^|vV`mS< z&D!o|yMq0e(^;ynZS_|PO%4AO&KmwK_@2L`eir|8L8YeL8|j-N6-R#;;goyK>*Efl zR&8>uKGl|*P$_9uY+Gz*?Csc_vAwY)MlEZzeb(vfw#KLHbDG%_)tt{cpLvOLFQp5T zu!BnH-zcS&L?vDRN}eZ;_lCOzo%ifH)+DQ`m5E^PDlln2U9K z*F0^VvNu6XnuRr<1fOOs_cNJbEkvbJ@>BRN$CZ}qZ|ZZkHq}_S=sy2fu&?unZUid? z69b?6?ok!`h@R%(75q6A3+1M+$DZJ35vjv74!r zUJQK}dwL+LU|L!8fm=jAA&qxkvtYDfTH{nVZAi>-PjR~>V{Sw)TLdG>4eCF>I_X3j9jnI<D+{q@e8aZ9fn_4?uEM& zKT#E%>Y~z0U4}=hrdCuR?)wHmnG~oW*b-P1Sn2QP^ZPts2kw2Y1Y3nGC)UffCvkAX z$Kms#vZ0p2(eRHK`1AWceYjfGJ7jE1Ta$e3`A^TXB#nMKD7L^EMP>AQ-Us%B*vIJ? zURHXUoLWEnC9J9H?m_PjnaZMScI}M59gB2P@ZI2dff@eas06rQo32h!jza`E!5wia zSQk(5q%M*i_Yvo~EX2{gh^&rWi!|r1WpZo~d+L2^G4)`r4THgP?UV9lj@MmW+7QiEWB;O&A2z(Xz+*etdYHyAV zN-6bx@so2;N+%smJ!!n}j_1bc2Y0{KKQ4Nvl-zPnX$i4r zjqpI_<0he)yqB1%((bo*F-tdJGd9E;$C9E)@cvAXPLAfmcV?SE**@&e)A&~Fc!OYS z-NpCS28wrScu4c*1)O0TbGN6dGdZD^lWRzYJWAa;K{#2LIFrn`3gRR0?rd^CcWy$7 z`;EA(3($BgTT897cqcw#F8aBdiL>j?4a7HcIb|j!#DA6C@DU?wH|l2>(#Pr-^aj3X zzDxd1+$%H-RtuVeuLHRQ#qc&x5B3bT4wp_ClHi433x9@$<_mZ(b z?dFS-+_H90>XVw&nCqkwV|C1JX?HLlM{-26MN1mBt)GZ3s|QPHKQ$FYQWYgpcl~)n zDWNN&PNC265Z$K2z$DoqlA;y&(3!YN`W7H_^F^?{$um-8e&(PF;54q1?L! zVs5OuIoUdFZ-;YJ-HlT7@fVnRFOl$#NaBakDzc-txW_uj>4g*=$e>k_C#&D7HVuO7lwS%d>4Qd#2G#5sv=6JJeu z5Ip8@O{J{&1Dkw~a>FsB1z*1Z{K1nipEi3@DLs>Q-3?34ybjJRb7gd6`l9qU(Z7td zR+3%UX@X|j;B;^gd2N;9P|-$)?uQp9bV-PWwgv*ej;bb4fU?n(%CCROTd=X3aU%QB z``t;jFPde|E9{3AjOnq%(IU|ck=v1y(ZkW?*a-8SwZm@dY(_qgIsK?~aS%F9L%1xr ziP0)2k5oROdem?=8)xNW%4qo{@iz-0rxb9rP^adI6~?RZAzs)?RG4i{?WU7F8)hxl{w zo4{jg-R$=F58Mte5BuGo3Ehe2at+YR1I8Tp1x$$ge zN~dTIdx~dZb+mUL7}KNIB4eWpA+`JLX-*qwiJg;Zj@O){UN<$Xze#9C!o|c9iGPOQ z;!aD~K9|>fkKLo3M8EZ>N&SfnEF?FTDnd~{X6yDS>!w-C?9bTs#Fm_!elC4yBr&!c zd-A?*!LQlo_IA$_tCbs8UOx8JPu=MdmZl+Z)0NRwT`HhB@^Pxc7na9Jqlf}Yvcuc~ zKeJZbOPwLqHM>Y`mqnb{3|nPH55ax9PZhnD+^C**UUS#FRfsUF&TaZ=dARZuRgDMh zf9fmoymrwGQ=@i*zT4Lz@GkK{`#4`T2v!PCh7#R1JS3qlHFriO9!}U99v6Bu_)*|5 z|8ieH&6WPO^Th(GLz1pNZS!ovi-)PBVsG0QxljArZDIdoNU@Ko`j{EJc7e4D(qv^P zFMYb{24qY9#Q%HfZo==0#S^~_KL}jZW6E)9IJco`?uYn{KV?trC3ldaS$Xw%w|>qo zPH-V3f2?iv+lU-7(q~2{;oEO*bT{`~Wt>;2i#Px#RcH82K}c1Xv2u5DZrxA)4Nppe z0GUnAr2c~(_mdk*0WY0h`LMm5xRhH&4or6r@rpad#vR5F^NRBnTJkacGB=>*jzR-G zbxIH;5G2Mjhul*6M!luQ^gh0VzL|PneJ^zfis;RK6Z|a#JGn_&%$U{Vocnv|M)=c& zFA_dUXrJ&;_+)5h@Ias|mdtwpF};G)*->IUUsg$4@HG7Fix-R2P8oZhHAEqgb*`Iz zVh1B%Qi0$vVVe;uFUa-fR&eRJ;U!t%{9w1V6qw;tqP-*A(odzYiG*WoshYCM-skj&O;pky zz*?Jc?J}+4;m)2TzF`X7s>j4~Y=omV!F~^tw8b8=*ojbA z?g!TE38I>YI7O&wx|&$G`a~5jabiwu*6V9f$#M|u^RueyPpBlk#CON{xv#a}S39Pa z@^zzn%@^zu(}Tr=Z&LqzaKH;*r@naa#GHu-5*m=8Q7w4M-`T&{cTQ^~U$o1|&ZTxp zZkNOvG22$J&J7@+#c%KF%y}hH_p1F;I`_ zoYLWs8MmFjW!ff1kxy_-R@VE%%Z;a~Cl%(;<8d3q8RG|-c)cB>5zKS38_{CXIg$SO zs(*|gfcZGdT4{gg7}zJ5;ZCKqBR;ZoIW3SP!mRE*r6pF@X#Ug}FG?q!P0I0dapVrkV1ua<(AyqmG>w*xG>mMF<~JMIez55l=HuS}2cG3}vITMIM3_ z)^NK!hwL})pRGmaC}Tiuax@yTB8#J|Viyg`ddh9)IJ>W1(Z0ahv>&nDcezPCVSn!o zayz1zu1cAR7F|t@vBs&aB%b=C?r^uFTga_R73RN)&@S!e^s*r9E~|S1RIT4&rhNcs za4@l!V_Q^~YT;WV%`WE|h2Qm|l)gd4U zM+9#K4}~6tyCx)sdxQ&xuZD_*{tCS3`&K#b)G#`w`(HkI(JT3{)LOA$tgorX@~-Qr zou8v=zoua1>7yX;nb&sr05o`#|j1I^4N^1#5DfS=l%q{WX#^@=m05G+(R+ws1}SD$9|O zoX!_mxKGS?&1U8??whY;r}Tv?_5)tpOXPHohS4~j2$2$UKjilxc$N#PH#Ljc(f-79 zmnV{Y4(z2FR_JnQE1tda!3g!Qn)zN)8+m1{+mXDfxl;P3 zj7gmzsciOiqG$-+X==?k>cwtGFUE$MYuS}T_E>8t-m(Se8mpeu-z%j2NjEjI;@kI5l!~Ut3ecyqI}N%LOPt|pQY*E7PqTg4z}^DPERTURDp@B#u`N* zN2)V!<%~pV)Sp<-t&n}&I&LYPqhfYDr#H4)HK!w^at-pF>t6A`#JBY;^s{fJp~U~n z(n&H$=6K_Xr!zp;L)}9|S9`9Uh%y$XXNpt^X47)slFc zjrwSAHV$hCxFepTH}#G3U#5~!q2LG93n_{P=L`H6xE9#TK_ zcQ|jE)YeNsmD(=t??{T#13yD$_lz^08=k#JH>w#-Hji6d?6uT5$c(q_p^H2 zUdWlX+j_dMnm_E{<;(6%(&nqZxZh}shiw)&Ub@s1Us+DHcT3_})>FHwhW8Tdv#QJO zuD!!-XH$Jr{}&*}a2o!@W712h1d(&Q@pcX-{y#=+ z=QgT@UDG=1zfkl2JY2_^)?XV2AMyk06vzDI0;dDZA$9Na@AU_%;BzyOFL<8&p^yC2 zd?&PY`652p(dMbx&(Uqs3Pyf(@^Yu7n~(T{i?(eAtj^{@D1w`aD|^LCC{KizR z=60I1)f*)jpjLSe{V)BN-d3-nWmWSlCFK&*Ft069+%vH+3levjSuQ~I_-Xt*Dc)cr zskXQc-QTEDcf=Y(J?g2kuCeiWOy(QC7_$mg3K+p%l3~@iXF}xNV}FeYq!^y{i}>Sq zVfP!@a!=jPWTYG*8>R}Kyk7?cXWa@p~^>y`c@gMUq^1n;8Rt0}${%+y# z=O68_=Wpwqr*&2$-a2QYm4NRtlR3b;VZS7@@o%i{qxfL6*cYvw_7LtBKf${F4Qljr z^Nd;CidaP<(T^cMYMlIqQiJ@0bW$@=h4K&5@-jnFFiH4S5e8@;-Tb%Qz{$ z2?2UAaVR5*xe2?E?S)o1^K+w|(bYI%yoVR)6KffFU#sy^KDBDtGl(0!K-}MWyD|2x z-|mQiqrNi_R(Ubvfxf`o_6l+N3N*fN(Uw)n8r@BnRs|$^H232T88Hj4;c$7M98y|x zim$6)Q(JQ{cVDYUJVrBY+h_Ch|EA3-#j&?_@MFr{a^#}S;-+f=uFZ+|I&N9~5 z&{v$wHAkp*SVa3sX(lyv_t=N5?hyJP+CAK%*rgB2IoQOA<)G@p5&IQqA{6>g&JMc~ zw*7hR?UFD?Utu2idlR`Yc`o-?e&fFNEhR$k%L5`w8hAHgMBe2YwTa8yMTXkXWSljD z1}O-Tqxh;rs7lDMdpPjd@=%?MnMAG#jBKH>OoEOeAP6K~B zc_3tFCe!E~bw>n=swDb86LWlt2#Ar4M=~*GW3U6hBClcvH$ZE6{&;TrzT>oHxXs8; zT8O@`=UwMLlbu!Um6x0%L>$j_my!iG8E^LpY=!5X2EJi!e?o4}eKdMi*9YG@i(7^q zmO4Zg=i|<;EVjmLymBt%w-_qxXm^;~FT=i=$}L!uTL_73j-CCNmxB>p#pre8EH*~& z19QF=F`nbeje95;Vvice&DcoT`4^R}_*%YDH{w;g#^btrR^6xmNyPzi60Xjf#87T4 zM~R}ihF|c!vKU#>l!HXlOp(X(wSqg2mE2b5Cui(;cBqCKR2;#0d&+FC!vgymzTz}8 zQpR)Z(vYm2Q;fN<8!@R9jH-Bwgghl!lb&rsh>)ryzSmb|))m zrqXz0I#Ne#I86L?)NII#-(?*8%w;lGN|R%-3{T~BqBcg61D2aR?M9II-&Te~Z`Rb_ z$eTfYe=Dj<4WVMo9_@*iU9Y9LK|42~&QxV+vjwP(eOLQk>kf5xs+vdL3a$SQ-lsG- z1o^p*Pf#pkY$HU6CK2WOIs4KEGT?5aA?gqVTau`A?hT1PYK)HWNPFJn(GJq6jU_qC zecg39=Y3Ah_ZytP2VseC#6zw8-^kQK3%!d)-W0q1nX|{4!!rjuL$S$5WO#TxvlctE z4u9fzr^pffgFLIwPDwl<4>Mw9TBA@l=g#aX)-|fY7aC=3lhDXTQ9BG*E{Kb z@f~&1t8tFMsjb%rQrG1)H)1vMFi%7)mSlut_=wYGoArK+?84DRxW>?(Euk$g#R@Fz zr6E0sh>1PH--p@3ue%qJmR005&7_UJpcLof9P^YOE~Br%qIE72|B;!WcV>LrBTwyE zEkha4uiWX_eB-H^ASl3jS!?%@!JT+;Rys?a-;k-3cqINpzQ!<8HJw6E0{2)UWGNSR zZVo4llZk)JGauv85NYh}O{oy@KKYK**(WWx71mhH>mlvOvgyI?{tLN?^19LifABOU z>M}Vx?~*f;qElf| zxFgOAPILp2rPfXd?nNfEg4WaHWaiw*tx9QhbQ||A#_cW6&fC|jVLgI!Iak(D}EK0qgGo1Opp^{R0e8@ZG@8rQ$ZE1;L>MGXpE7Dr> z@@}F-Cu0l#K;GI!G-(fdT8i39+30Oqq(Y&#%WAmu9kIwu;X7@?ZZ-h@@fE9M0n)X_ z`4{N|22a1lDfuvu3(hmV3aO0HE^PRLytWNF8lyQ)ui_OyB29genYR4ABY(fa9Y8Io zD7vZ$@_h^+2LNx{zU092qd^;|=EwHxx(FBx8`D z22OqYQ3>xsH*S^2V3E&Z4eeqLJw~2#qD81(Np|uLPBjl(i+&mYRy zBJMShI>(XB4cyXwh-9|s{R&W(V&tF((HK_gb0>sPrEP}(zKKU;#Q!7e z9N_E9zPEpTZ&JIR+MIf7Yo<(X+qP{RQ)_BZZQGq9waw(-d-#5L^8bC_o=?*x4enWI z@3j}6^*nSE{P>9}Fid}!(#X~E9s6ZZuBpss70Jx14^;o=T>GwO*BY=&rZKOyLp!D2 zBhq@u+DQx2tTMK00Ly(XJ)`~}&5|ZerJqzss=3r#{FGKoR`65%@k)E)YQtYD(M7bi*y&h2FQ;7C>mAU5{O3s%J{JaOxoXI+_dYa-rz20P|7HNDr4 zMeVLGCt+WvRh!|Xs>Ymo3D6>Nu+<4_FY~bGB4ViN*lMmb+G&GINlyGw9L~EC%2ki- zJM@!I5YyiRsg=Sh$tvAM4L?4pj!{%4e9X~pW(Kr8_PI>m@;CHnhRD0+_jE=^Dc6ag z>Z?mR|7~Uq8Z(=+o(@SQRxya-ljyndMGcr;Sq(4#R((D3eSJLBcC9BWW>?S(vvdP=bin0%~ba27;HMv zS*f-b`*rpzd$qkC8%g->W>m8N1JTxy7%U15$PCy#eusH` zC;hfC`hV@oZ@zt$Uyh%Y1ii2K_E+NeqVz@P@wKm= zWSnRNnLddF@5FiLuImsTour?Z1oruUWHL$F6RU{IGlJcmkK&C4KD)2B3!RDN`tN#0 zc2aLV&oF%nS?oh}6t3z=^uNK({-)pJOpDOsvwEhfHHqP$GE>V z!Pc>jN>FKHvm~sblX&U&oL!0WK0moWaK{_*HO1&%pWwMJn*_qpjt4`y~uCfx>VbM-}S1D%EOLE1&Es9kt zQ>mK78TN$!PeD%62}ETfw>C9h!x8fy_ToRP+;^Q3ky!U5)WvGucrC*D+_FSh^XZBwBfFkKew3Z5_O+lI!r4)+K{y`cd@oI> z=s$X~b-^^>Pz|jh3=0$RCs(y+oaCCGj)=OmK2=|Z!sbc+5*4eVoc39CRX?(j@b7>23$DlPxw!Ze5zUn1)b28?|Gb1-JG_pyrF#NA3XRo=I)XaQxu>cIM==E`oa7y0ZpBk z3DCJzSX{8lo$(0|)~-O68q#PKbr=FvEa~tMy}Ha{ermO zJL)StiR}loq6=|Wt>u&|iw_QG2Id-G^O(H>yUel|+H0`QaS&QN(dyicrN-Jr?HR12 zy>#M_V70qUT70oJEL#WvGm-T(A_iSjms77M5p*-IrZHD5EYp|0xQKK96!v>UO&~ib z;8fO4Hgeaaa21pzMms^xBrALt!{p<#BNrr_I!dfljoj*nnwZ&zA=(!04pDPFvabmI z%5Z%-D`t&8T_1zrsYA{YUw5=8c;XRSc}>yJmJCQMR|$SH0KR>dytOrN_#rC*-lsLFmKn{ z^RP%qkne4|K;+SXn$1pIi*+s&zkIV3660kcE>6oimxBG4jT5RkyR@N`&d}Qu754lP&>I0GVlZTMe$E8Lx8>_q}tIy4D+8}=dFVLP7DhlLXGmv*z$Wb%m zS%*>eSWQmxl(<;ev$1M=GDRzVaUUY*3{WIJ)Yfr+m)GKGXR&8{>Sc*w3%H@2rXtWp zNv_<68>0}cIdiFZgfr#86})76*e#+sskX4AMv`mgBx72Ow=BqNdCzXzj14w0!8DKw zkr8z3HnU<5aqYl6&jOX-3cS@&>b2v!ma=Xx;Hh0Z4c}h@k1NjWT=ZXyf>6jy4$%;7 zVFymI#rT@L#7JrBurvUpx|>z<3*>AU*6ub?}Y33h`G%45_p z3*&v~k&S*=Q)?AKs*Ynme>1uJBl3tWM0_=<5_JL1(3EOr5%Twtc9F=qmX?UvcnPOk z0j%;349^@+@4U=a&u03<=C^f#2_q8k+X8N!uhjk~%nKH^7*Cgp+&5pxvZNR_~dE!-*mw@;g2rkb3W0bREUU@asBpx z>n0=nvLJP}8rWqJ-gzUDk;&Xh6Hdlsa28}HLtYA3hmSb8A6D6pB5_W#svXoETaqc< zLCLrn9=GI>EUPN8;q7!lJ9C0B1x9-2Tu&v`V6wc=74 zI!+YbS_PL59(Xj}{%|Tm!-xmI^7}GL52>>DVDH%E<$HWDy6q|bNNa|ge)vrv?6(t59MbG>gF|bM&o?Fka|EYIi?{sFp)MXkyE*aD& zyhL`BY5X>?;**}ZGJ;o)II>HXl0_x&6t7RfFaB+5z z?4F4^i;8kif93wxa<|>>PE_Upu#2I6)ddC24(z2a?3~72wd^un5m>7(R~L5DGFHqH zbhm#H(Nb#R3a2uWnwnQ`tlbv>JDzjuJaKale&-;(S)`koesVLSmjAqj)IB4JcsDX5 zETg!yn0z3w(gDTJ?O>%mB`>PYLs3P#uUcvX=A0U9Q}Ly@v>=mDxydZc>$R~)P4cCh zOci?CW%7n0OzXtezOho4vrY=AaXF=z@-vH4``X4Cmko^B7)~Jr2A!#B=H%rJdO&Sr z1imhX`;@3+EaycE$7SN>U+%CEYo&%=5d3X%FvEqhOewplor}IjMmwb)*Nz9yB{!d~ zgnc@(ie}=8&v9~pz#?&-u$YRC$Sd;TV;XYqjKgCcC4=}tbX$>X<6N?%0N$iNI-L8N zY{`ut!ctD;FVwc0!v1rPcqO0Gnflf_s-N-5wA-UM^{@JrxFu(drxkwoE*U{Gvh94F z#dWCX*CCfI%zpZzUB}-H)&9~la{l}W^FnXVuPoFi_A}302`Ta8oXiM<7<^@S| z0L7q&WK2)V21fGCGT>vEb2j8~Ed0W1EHcP$fh|&j8Gdg)u)BXNfw;(lu< z{fZlK6nsGqFe_hI1N#l<6xzyJ{tT}YVvZ*tao%6}s8Q4;w&1VsQN2h+HeHr1e`?IG zk*PYiqz`+Zjz$qIvJ(GlgYBzA@Afnmy0p}f=Q8j1l*(u>?x-_1*{j~73mZ=>tku>! zus4=!yUCR9l3ho`@D`@W;nexYX?zAB+>O&aD|5J4m;srJk1haL)dS*{&Prj>L%U$C z$|C=u>)8>`A(w8{V5-`NcblGFE20#eT&52hj32zx61+qQyM&$CcF>5sXl=DNqQAS= zTFkW!H04xlHrj}rt-V<1z2(qr&47Ml3(lxTL^Cfzh$O(a6^KiEkrk{WR(pyy;*x_! z(6OI`QtvxAIhFe^#B1m9BDv`NuVj6EW@5CFJd|q5Rc4+F<6ozPFuba~MEf%rtPah| ziKy%7GB-&rN=>65gAk?uI1nI3pH&)vl;X-!5QO+~vOsC*H*{l`ht6aP_~ zp4l^0um7{2p<#Lo^!;vY2OX_7;3xlPrCjEJ?ptpxzx_L&xGksBA@Ttq{`5~GT4;G1lN%IjLR|}|_r-(tet)P-plxpQ`{K;sd zt2|)U9`n4{fD7u3|E$5z$ZDsxGx0V(dnv7*#mzjs22{-BJZ> zOe0G?gDnK6rWpIO7u}0}ROf>3Z>-Jw)G^nyTi$_hstC{Gc$kn*NiT`(Q^MO+Tkb*R zun#t|*Tk`@SuN$28cJ(wZeyu?Pk>8k3GvEqqT9;d3x%f9D zcw!Yq@go#i4=}AbhtszUbBslljO5HOTJ+r}ZwL|LDpto0)`x{fij)1dB9fm@j3yMmER+TRfXA#e zQT=>k?IU2B-Vm`RL9@Lguez^1o|@)%`7qw)3fQnns{dc$`$+&cDJ@l#;?#r7P@$}c z-)TxL)`h6IB^GSSSz8HfR=}nil%z_0yvH+q_P=zb7V%XQ$@=4G<#m3xagY9V#bMr0%9s8acf{j=aOxYCA;ZGR9uHi>_T{utn{4XP??Ly`uAX% z+YRT|ENL`nPcxoQ1y)Q3_L7WZ%oC>HkHJawH?MO7u}f!I-|CUImgZ#0K~*?0sCSRT zcNG5dDT;#k=*wKges}qMox8ruTDZgKpV4Q1M`!a#jQXI0;$T8}>9TvjbLB%TuXM~+ z8EwVpJcnNVrm?UL3eK<1@Istp?R;Q*&}Pmu5v-9ycd#&3#TsNn&A@H+AbJ_c9W17Y zy8>>$O;lmF@SG1(r#eZdbb)nt3lzf*DiHT!LAlA_Tl5+(g7&zC-x6PQjQ;aB&fnFX z`}6o&GhmOL42R$__D~n*tQ%AJ5}Iulh#w2Ul$#BvGr@8!w4DSyv``Zf485;-me*0T zImL<>UC*W9C>G);=i`^=bCyma!Wiuh=MD!^tsX#Czc=^W6F=UazrDaB^yTWqXS!nT zF8tew)BFDn->pjqkN~dQ z%tRC=@JH2n{X%W717}w^X2%DDg&EHMjUW;m%hMmpdn0*oGIu_MYbMuJeArm7F- zvD6Fj5Rc)@71~Gl$l$JkMmz-O_W)>stvvP3C?zci-vlBO|1pJY8kMBk{GAHIVlv&w z5m;g*{EB0+$LN@U$I@Y+2Fh$P>t;PU$X0gt0eFMxiV!w7Oy$&g1gz($J;O z&FdAqSyfs44X{s3&eqoaZNb%=)zu!0b>qJKl1&X@Uk=4)Ls@YH__!CgY|q!UV}-Qj zdz
    hN=Fq75ZfrNkvP{c`a)8=S#uxl(a*Ct>9XPIaMjA^4qX<8V1}8prbK3;uY) zbuHNYzhSNCy!QX_RM%Krf;m`dQ=Vsy3GKoCSnUveX$P^~Cf33#?tV4@Zs1yv6<5=F zU%}_r^8T8bYaL9e+hS~UlpgL`&hV@J%m=XYJ%@GeJ-$dVs7HbR5nRa$d97*Kb6H?W z7Rp=&Vt7-Fv!+UMkL9?^^W23x+h44)>Ri=$Pkba4vMTYvWnq0P0oQ8L7|zu~SVZig z?EJheWQ(y@NgcD7gd&5`+ew6VVtMBUzjrMAyZF3NL5cke!Ri(_<_cqV31(%X`XSi! z#cLPqOeng%q{j7_9r?diCREBUbH5j`-Wejwvs|ZHd9l|qR>~19c?6#)K68q%6UvI$ zV>q23;tij(zlFMh&U@3abe@z0&a$h2D$cpBDOk!uyDolr-kwB)k>t zNigLLj(uS*p~fWG_J!ht&}H}^@3zovh~=6W+~2}-NbDy53T?8R zF;=+Fze4Hl2J7$Y|6NyNEOd)id4vBI>Wa6y?*HHJ78VmfP1sGasS7@D!N4yRA;fbM zEb<|2DB(lZ|9gGV^YDuQcL(AQlXAytxqG4NBz91&T{Ga>1m|k57>fzp<=}mx1}ZeR z^7B?$HrAH^vwzl@uMkRQS@`ZOF_*Bcu(i;4%MfE_VPV03F6IX89OM}Qo<6#Vq&+7^>&_1thjT$752LHf7eC+PuTQkjP1m~xA+P7 zu%58I@Q*L}*+L;nXjH{2dg#B*t zvG9Ab>rH&-e|EYX^Y!SY$JpyJEA}ZrS?GQWb*|5>vj4A0606UV5E}_C0I`O|uFS&j z689P2wh|v@!;TqZ)`Zx5Vm%1Hd*Pd6^)to(6Fyw*wAghb{FCtLu}XkK z3&6+6;wywYhR_WWiYj6qc|1qKiyy1XBzC4ygo#zcu=v;GDMw*9!RjBY&-4NNiM95I zI~4c$miL6(rckF8Z^hny8gq%g`;gTrbWWeLF2(K>N@uYuRzk^1?7G-X_%ERYAoL+( z*OE{W5zkL(m&E#Bq1!6FOCp|i%9u5f8Ve-ld>8vfoMcI3ypZ_cRD4$0B-TI2`Mq4AwLM=mR$Hc0@h?OEN@juNNp^Rc+GeC#Dr{L9N-wIs>p@$(x4p%_zt0cAy$mAj8H@n%8gnl2)C6Q2<(8RBUOJt%Rui4#Yt$;7HE#Ogx{Z!Xr7@Tx)` z=@)nQk$a1MEACw=M0}3%Bx0qA|BKc3d&!#k-KMMNV)RYvId34PnxlTDnuB7zcs#7Yvn7@xR@_q=AI z026!Ci8D^D51~dV^OZr~f5BY}^*`a2zVe+ypW$oF-M`@Oh0@1s?p7#Rz2|+wM}HK* z{gTgli%t&yCs5dK?S*|C(!H9X?_H2A4(SVpMl-Xfkki=TYM{f6;1 zX|asWor-lWbbf?q5IT&}d@K}QUUA>TXFlh3CgJB4;MO$MaO>PdtR|$;Fc$eqA#6~9?@bo7b_{;A;0h42ldwt->{g8bPtR+J$6X60(|5cE zu@d8O$FW~06dy&bD)e^4*t{?C@ zp>i9=S`nv!P=paWT;eB&v-*T8wm93rG0&0#TbCi0{+-ty&C~kIU8TSxm10&#HeQ!l z-zkWqMXuYD{Gu426FQ?&e0S=YwUL`uRh?)%A5T9yYx5fx7n-3$(N3HsLKXZs{!b{w zeq{BgX7z}8APqlT#IFut{heo;iq93`|H8Rnp-20O@6E*TE5he<#;n0)*h2h?)$fy} z7sN1^IceVUy2TwvqF}KW8)YE}YKu-rRd|f*dYSA97$~oqGu%qbF0C${oqsCi^Yl;+ zr-55edLmB**$`iT=zXB3HUgBjOugeYo$xAhV`;GWiwZ(kGCKtv!9^-YKl#Z6u~|Y` z<%C*oMQTdD$wV{o@e8aP&U+#bOv@U5!ks)~6=%Sv*;rr6vC}~2V)D?>{Fm(M25aLB z&rc|@T<0#nvzGTV@qB^Zod65Pp>K1Ly6?gm{9Q?`q0`~I8{;>HrlruQi#XEl&yvti&Q$l`w3ijDyzG5#cCsE8!_2UIZrX#dv4{?Wwv3E;yrbIli zqO8iJWWYC=!uVTSLSFfUui4IQ%)j`VveE)@T&cjUFJ)J?B_l0_KX}CidS5Ib2Tq)A ztfq2QN2;@P|K+1YtGXwzNa6QAB3C?)Psj&mdj+|q##(E_ z{ibI{ou@ug6RR|$A~lq=?F~u*75Sehc#ZvJp|`MAN%o3RMQu*qY7+OfmMX^*Y+_O` zAA+{RW_;Kb@CL=m+aL3}65OTmL1l=xo|3O0!j3}aIfe8e?_a~VA+QmXsMl42m7xZ` zh*5m!H16OfTwJ}0sLx7UsCSl@c2j{gm}VS9#S8Wn_QrQQeX`Woi%%tKuG9qldnEQ5 zz!XSHuOz*{`cf9}Z}%@|@Ft`2&{i%eJ)~B=#pM}LCE9_KiqN`E&p9)P8NQG13~wYT zgg@kcaHYSahj!n+=UrruO{l+AA%~nz6)oDUK+Prr-ZP5NJi{xuGWI&YeRq;+nfsx`-Z6Q3FAxMSG3n1AwOb& z4dJzZ<@8A5r3F*kLAp+lWt7wb)cIpCj@(6-q!UzksYxo8)a>#bDtVcx1{LrY(|^z8 zEtfZ{ziSJXmR@DMo|Vc;EyV%lcb3fmG81|Cy^T!$t#j&1?dkMQmODtx;8fbfOszPL zKGCzODMc!i)dKQzXPb4$PAuJ1(yGnmFlu(Qo{d?yl+eVxKIgU$%frj+gXhc2k#}@8y=) zE0vWgUKjh6mBKZp!XTyFGSihtKI?*2b$>FmS_@B^kMHQi?|kPT_EyNl6u*3vPJTz} zuslRM<5UEhFxXR-#cED?mJ)g?oo{wAZ=7-$4CNJNfm_&|5d3V8@t!J&K__QWn<#yyYHm3Yd&}J~ULtVr zfAKuld(BvV0kD#fyruF^C7}wWK%V8%*MxVfrh7v^2N&`VIhq=5UHo5h=`^*&K2mCB zw_L(itzvdExspat!PU`2sI7#9nnBMpvD?h)=BAK~sh596FDPA{{IS>JgeXVced z3FV~j4!Cp=fv?>ntpne2PW}w{ZMfXSEl-zxhV#vRBfS8d_}v=;CsBlcK))s}G8+fa zTJPmOdPVJ^SIQ~LYVWF)bF-V7%rWj*?U4V3zq+=`=@Zlg^+U%TSRz3}{GkTvSR|Ks zco(fCp-3yETGIbn-{mGW-&>2M(#lmY3z)czQbz5Do={&d*RdJAWX+0;HEjfeXxh`PxVZ@b)5Eh3M%G6&yS9n^8g0p%RX*K_Ro0n!6lJCobB zBtt)F{H~RSkK_wJ_Nje2cq}-=&1)PD+ha74-qIOe>5TA3FqhDm`HjY~26geK!1?*p z87kGJ8nlYUHRH)ojTinDzUgWfshX3(T5LYCdr0j_WOB*L>6~`@c@@AwaCHQiuP3k(Uh^b+ewj1t-dJW5xmk9E+T;v827!SwcAO(>6W zuUSt*mCQuW0O__8O^lf)>@AUS=JzbSG(ZyY4k8w=+hepDHa7Wsr z<dUOsN7XKuW;9Ht#s($h&Qtt62C zhOy(L(?_}>7f`RE1@g`7Wwkcf+o{Qg^58G7cyW}`S}!!GI?1zKpIy+r7pi0>bS-(Y zu|2G||B&KyI+|BQ4N;eB?7h*S`WqTEMMUr-!TQIUuH)Nc(bfd%%rY#BsH}2*v zgP=O>jt?FUfbD~o^ROaI<>YWOyfgLMkMhmnHSyiFQp|7q^G_PUL| zEp~e|jk8b78s}A5dU;XsYhb*)67)lVuacF{8tfiamS{oQvH!F(c!Sk?%0KW`6j1iU zr7;iwt=z#{qvXIz@8392k}{9Rs8onIMWr_KD{H+zF*_btlBUC8sojR*Un+rc0Wr^I0-Al`&ZU! zW*xKYc+ItqzWl!N>Ty`?#)fj43#~(TXE%j%$I!zD8vW&>?n!X1U#)spR(HB`Kz-q* zvs{x|NBH!1OBvj6UJlfN?lDE1S!yIX_7K?rin^Eex&GDKNOzANS59M`(hl2p)Xe~h zG-H9DPzg%c z=oaU9slNJ659t-vMP4Vnlv&k$;?z{n=uMR7b}e&?J6elpG*a@oKb`H;L%D_b$tr8j za!N`o(UGX8Y{DNWSJtRs!bjmDpQ7uME~w`i2b*cGHvl#_`JI%A;V_9&Gis(|E^~ zi~1_#sg_#)!!8(X9Ng+`Qxj^(-7jV}_pshREQJx}O}A6SxO>UTVBU@n4{mpV8mrQGs$DEO0m_DZj`nwaQhneq!R<_v<~AE*L%u3=Y90ypIf7CrviFAz5PSpoTUppes6nW2`@n#oD+ zZ7?0Lb6!f7%74ty)R#Gmuw(3n%LSVUmNmNd-$&NlK+`L2Gjc)sDBGZH2==ecp@I&-r?4 zCA<&TB{P9}DwNz_E!Xgs3rp<#Tg}Ev)YdK&Iv%(aJZ;VQHmkRd{=TVtCiNbej#6G8 zCzoy5W2}%l+jgW4x@WZ3c1c~m1o9r`Pq~DX-P~t_`;mGo|ES}%R9aT0g7=>_GFUe- z*8CzBFw*&6Xa^M&1j##Ti5GCv+H2AOm~3r!+Dmg`PErjm2JU+u$Wt`pI)ylJIy^2itTxxVIl8hO2w9h_boC$sy+?ddg;Mxj%e zMIEhe(Hj$|Y&BcLi1N|wWY;6NwY4bUWq(4Wtn|UW7;0~ibLTtrL9RXFM4Snm?Rj;9 zQQnt8`@>xx+7#O8{6!vL+AHNYA*y}rb!Q6ZFX^Xx-f;B9@<8iL2yHR2GiBU{_GM?U zR2x*0kNoqM*HumiyN9Dxb-$S;ryvInU`Fc+z zPul37hm&Ei`w32>^G3=S#z>e_fZKDsAK)P6IQk+0QYrQ+v=z)dZ>L)_x6K>&Eh5j-!axZ@80wZDx=U8 zP2`5z^PNr719`DD6oy4d9tgs}1DL&oUMk|#$!0^idX9U^wOzg^zJ0_WiM+nHi+C}NbtZ_Seq4JM%P}=PDvlduKowU;L@?BzsJj65aygp1o z9h9oN^PJ~iLA9Y)S-A|8Qv#G?M#E}$)LL%$m+I*keRB;D>?dVe*{%43r(PE z&MS+1lN*4;4o4rb4Eu7qnoZgP5^s~U8y5Jc=08>vcRWg{1JxeP*S&yszMXnUZtKl< zvNNAlm3dX>Dq)#R<`~`_xxH$tS;3a8xkyc~2IV+XergE;_)Yh6-2h4ahtthI znL{`1z0v@D;~bcHH#2{-Qm!S9f?1_I>!cPl zC;7eSOb9Jv=CdkQghikQA1d9L=B=f)aGzR@?bhx#&@fk+5zIjCp@FncX%Ctn~qKq4OMCHm&FutV4kTlkt3D5++p3bc*7`CRWPn0Xa#q0F$vKPD%5A+_)}I34jO za~;1_f@!Io%01?A{itH^mEM9PtSsFi+sRCgW(BpKoSOfClfp36$p5vtiZGDl*<4@{s*`?W>!XBA( z8a&26PjO2)AKf+bUgoy*!uz+KS;`txCG@pw+AC1fS;9<8dAB_|?hLTn@s$%)$^Vej zgG9X`8?w;ZKF(yy7I;dpIGsETZ&**uu4c#1FXdNgA>C(6t{sz1)14-;W1?~dyF)y6 zIqJQACDXmZyoK(jg_&*<^4i| z_al|*bfD=rlCjkX&sbI-L}l_n=H}YMaUZ0sFrT@e+VU1MrY6dJ=@uBcD#U-Coq(Ov z%_sGgi4~YcU*pAv>9C^u0iMpD^oLfvW$m%>>V=&2@&+n?qgb&j=z5c=&!awGiwbIg zX*8MhdobRu;gC?Mz-~beD?K`31DLoAC>`XS-dHB!&NDlmjGdjD3i48_bA?b%^(z~h zOD#f;d^_s8U%+)1=K0KL-+pwvQ&ZeR?QFPo(gU{xHuEVfJOQ}d-BjuSrB=1n8SKuY z0=-SD&s=(9Wte=D2yT;i(7nmDW-Wf!UNY1)FkvmD|B;?NYZ{2ixm5XDQ#H;+9egdl zm0sLQepnOFv&sh1pL-^q=ef*b5@(6L01cn)?gCKCqq(=m%!Fp8f6$EeGnE<1hVnhC z%_HEZxlB(b0e;$JUtNafXB!=?GGKCB!wi!O{@Jv03MrR&*nP`+Is@$cF1*_Tccj;q zo{j?7$4DiK(vEs~X*xnLsMf~CqS;UZe&TNRTEt)}H^UD#idsTl`8W|!g{wV9e@IOh;uN8e+8r2gRaE{xxhpKEDW9-7eJD>%bFqgli3bkUDf(YxDK}K%jlV)0|-{>o@%6zitG5x!=Hz zE~mnN2H&@Yj>k=Y(gUYIex#AZ4ySJw#oY`ch8@9E{pvLb+jo<@n@2RW8hh4dKJvDc z9oyFdVVE1%&U1M3?kLyJ0#zJNzn~q+yKwqwIe8^_rOxQAuVYR+9TRo&(PePBpI3Bq zD)H5M;ouoa6+Nl^M4Bh}VH#vDKKVA^vk$!ZX0Vp);4ujCe526^sV=+pffj%UE(RO& zFY=fl-0vak0vSOor-(t8wx!?k4xN&j#OxoK@Cm|;G7`3iA51g+!M`?)Lw_?5m5Jy6 z9^86HPMzB@q-ABw`m(#u8Ev1lj$0ATLu_$UGrRT5Ex;--0wYZh7-{md3f?dSdzk6@ zny@bn1o4}S&sV@t=LLiLfj$446*`g0gt6FgEbrfACoBU2-&C%Jx>jFQc22X;_9-9b z--v$pF`ITwF3z;kIZp34M9B%51lh?{z;q(>>gr7O8TGDr%n=n=bC8yCGsY_ zXPI{of@u%f0_UEbY1R2mA6{pcWH&PcO%+=(uTXU6WN}?`PBj;nd z1ZFd-RSVy_fS>XgGxjx^OA9ie_6DSUeX1dQ-0E1N7U!6s-(CY-&!VfEf@fcduE=}X zfiFsREq8)>q8n2W+mtdy70a~<{jr|VxW%;M zY4qoZGF{V2|EMQ0cIokSDx^%6FV+nsJ|Cyhhic*4P3$>Cov5By|Ik`!FVsj_;?rT7 zJ<=dN$~*dVMe*7*z2WXe=0+ZIcj@hIRCJ~>J2MqlvEMDv++}7nN1^ayp&?i47B06ts#ut5y{!`y$bn>nCHTAtT z#=1p zjhSsZNryBs9m0}I33?r$;aSK>jWC{8P3_Or!w@A0dnPqgt3$NExv#&8nFlhld6PN4 zaOU#n={}+*ke`z@FY^=K_|5+jBV@wQH|H!C$erInYcHg8yMun- zA`}qv(bt`eV%mBp7f+)^k^es zHJD)Q$khF0`fz3F1P+o@$%!N?)67uSa}L_ytyWf17z>kHz04`0C&BH()6_2d1gr9Q zU+{czRj^+$BB-OccQ(3ebfM@$(NCf)pz)PCxEWp8e4*l@9-$|p_U2799V~m#t*%Vx zd}M;UzulWD>!4kMS^6+IjN&nebcfut1m{l|qK4TZ`Ula`t^kfE2RWP!9v}|qc5m{- zI((d*j($#5Ejpr9P?M>~yI5<7a*cSW08vj#@g*O;@x zs`=9TvccWh6-|rNzK^~v{sC}cHVWGn_HS6%u;0QGhvi27=dEw1(VKbmnas)#RiDEX z_7w$#4C)*rjk>T_UFDUG_J+6nEc!9BdE-=7EEL1UuT z1Ndxlwk(A4&ZA%KQ|qbO)OmEv#!2#D_^$fO z!>j5KtBfX+<6p*_I_;ksHZ{CKob7QY#Cea_&nSP;*sQ;1YVxs`1+KIR_%24{vrDTB zl@Q&k=U_Xch&MGagR41)-PpQnrZXFaPNEO6Gq@wzF8DpLF|aG}E?`h&KNeUP7!#-# zcor?M=;4>oOxLTj*Ick zHKiw4$wqAGW98&vMU5irs47nQ`d4B(v*Xv~lR5?B+b} zEf1#~oK*VcK7g4ZKK;2?=HbvtGy%4U28aA1q2!u8_y#?ci-9xv>>XIEUBHeW82vtK zSX8U1ZBgZ-@8YW>(JQ?d)IwpQRG}uJd!aFA9t$>QR#jytjUU^&nXujkCwpE{70HN# zBB}Q+Km|i*%E~V%XQq0*yaju;X9g`?*2zcS68DV4Uo`|Pl2W=u$A24m5fyaMI-<@k zM8$&aXy8Y3(t(U?+!hi)(U@~1vJ7fy_2!hsA^Qw&rqijI)q_p zcA$MkMqJ;0!CAM`jAQ;o?=CBzr)8)&-sdJtgSCQfg5!dtf)!94Jjh8`Ij{?s`{7a5 zqk2U>jv5(V0IkU0tgKq0$#`_tY-^q}lUhxw4V||vt1Ju<4`J{6lL_qmaIQZhc7I7m zwlij~I1V}Ced&dyGUwKYskQ0McUFc|U;){7Ezn)H!9^7Sv6P7ON1%LPgR?pdqG=v~ zzkxA$C3T@=-5Z9=By-*PBkB-iWQtOzYpYMxn&NF|Rajl=kPSu&$Ez7+2E4wU~T0jMA7P)9i_qn?i{hHg< zb=s2Y-=d#+3#FfS*l8Lo?3la^%WcDVMa0aW4ki}Ph2Nip1>2DABqSp#L(YAb_$qs23T*E5RY5AEA<{rfx&SD4YnhFT1fRj1kvZT>-lgGu)405#8;!@$$ptRfyU` zNv0orQtK=XJ}fhS=LGy@oA43wnWsLD9X7&~ah*K#JvnG|u%iN%m!5wUgJnHMmN=K3 z@HqX?NOV}+DraF!TLX7$N7!fvDudA){H%0=HKGq*zY&Z@mpC2M(@hB&i+m;g*HNl? zi;8u*@MmG;!bXIx<6NBR|I7EMaR5e%6=XLDPzG71&o=%cDzmk6+A6BJ%jCJB&!X|5 zMZH39YNwbz%lc&IMDtf7vKkq>9=ea+x`$GPwgk%r-v{=hO*AdgKTtGqE;>*2w5ViJ zQdGC7ve88Y>4Wc3n^nv#W;mLj@xai$F`HXQtd#Jx+_t}Cv-@@)*bqj702&Q`Z=F;b zmWIPHaP?(fZDxkF9hhI;GM%fPeZgm(Fhr5eMkM{ z!XAaa3HudRFnkoMiP^))gsEXS{Nw#gd}-0X>i{>_6#X;jL>i-x(bBlB&(t*aIj^u4 z{tI<>#hN9|n%GzQ+3XKm94-E@-4Rr|h;%)hm6I>Zg8tf5x6TK~Zcl5#NmC?6pV_l8r3 z9QrWz{hMeMj*z#qrUsJbW?-Iq3^v>=uOhqahc_$7x_iyMEudWa4smKvd|C~VRY|~f zJ_aj%m;Jbmx`9TOy)55nQ5h)AL~})WD-&>)<@S!#Zo?<#ykqad_9-VCAre(RIjfc7vN-W5s<9?g~yu#qMTcK%f~M?4!^D zyc*poIzhA!AYXU2~CD9L9?0c3S!f7o5Vx zr`MT+9zoS=05QXCqKCif`t744GMG5xs#FpSHk8BQ=X{30xQ-{wORnruS@^=tYZ0#? zvBe>J(-o+_eFT#*8FcPzBD3r;Fo+B13 zAVMt|2oG$Dt`S`X_2W!|UxB;9c_G8hY<^{p#zSZRpWv%tKh!15TZv$Au!zQ%Qj>iR zUv?I^CsWmbg3A3)x2P$#qw09{Y*cwJQ$^|pBgb%J(i}tZ`gUL z6^spZ1!wy{kOvh-8>Tkh=jA9M7emF@1=7Aa>>=8oUkn@0!;>(*H`WWXlk59(`4$@& z^ds~i{7NKMkD|=gHUy2|6JG7s=m#Zr^4QH$biH9dCaYO(b~1~Yin%HDTc{^$CX30Y zHxgys3VaAW49o~*4ZI?Dt%ln93U;T9T2Es$5ULy6NB*=SSTf|puN}7f+i~G0ctEA$ z2RscKVKkfO9w)PI3NAZ}$SMig^)Tuk{pe@gQVQa;L@)!Sp>}i)i%g(ySdI$zC05u% zD!liY8+ZU3^#Bvei2;0%#z^vR$EfRD!CtN4AM0P~U+N!*^6@a@|9$?~==VK@J93)-3l6-Q+FJdT zk<2&Fn5mzF6=a;6RjDXFWsd$P=%)T4&62{#+JnARM*ERjfv#N*>Ij>y_*M_|9p~rH zVBVk;qfu8nm@gO|SQBU-s7V$O6SAb z2$tHVu#TmpUK)hYpb}g`mpMn*)5q>lefW3q<|Z~ABz@vkEyFHL3qo)_`S&`!{$8vl z=JY=@EwhX0{~B57aU$Cz#A>N|?pweKw`F&BmH&+S&Twp=ozo>RjAJcf9Ip+N+dZl{ zqhXIp4lC7GEhko~j^$>PIUVt>@$Dc6`RG&7hJWpk2+N6*;$dGt-(Dl3QI}OVkebIG zvaW|%eK#CUVfsL=o0?qtN%WYFs$f34LwnsLV4OXeQU};R`?~cI6~1@YEc8c5QQ>-p z)^PIB!eA*B{#pfB1eXwpJPsTWybWXzb`BmSuN+U!t08EQ&L~^-3^hPar39;Pnw1cK zo$vNQ_G$)tPGw<>s{yCDjI1LxnzivurGc20-EM0zgVJDNOOdVmjnPv!Oo`1ix|EYR9FVBC36 z|FnquUfHTF1Hsl;ZKQS6I~s3{d&VqOjMktZPzr^yr2dlrQvL{kM}Ki_S01lc($^dt zjxsvx{j@^b6x0O&)>><8>4@JayLf`~V-DpDIcFL=>P5gLmj`R?!ESLGzTe|^YI^~? zakZ^JRu@EBTcSZakKC$xs3KQ2s)SF1!-K_wg|ObqV6M={P+?Fv7f`s!3NCOtnb?7l z9qMQraL{bDi-2K03Ys^oYf>@k06X9Xry-mfGw|;&tLiW|>I)md5ppGyDr#YTa4zu6 zkEjMs--gXW*8r*(wg=``1Im>BoKO!yxs?n3mu*T?YW)q-&(A4*6msb;jN%9w+R_1Z099A3&TP~ocw$4@dwx(kAG@1+ic`{x##1T!ng+;qpgNU9dB#wx7E^4M`d z+)0_?hu?wXz;GC2m%(&%5k|c>Fiel42G9r0!6+?}Zo*mpLLCjprZucm#i&~5K?B5w zukfB$Mt`dhGA4j_sf>z9GT52(=<~F%@K4P{31zU_4sC}~oXn&2Ou&gMv_vQaul3{!drQlo|WZpBIz${ga`sDN&CANI)am!-4^-Rt@!hU<{W_5z^WD^~@wCviF#LPMAqg^9nnNOUOm-w(R-L|K6 z;OI-Frx(sXosr{oA#5Z4)rpX1A2;~RaNLuhQS)?}JJvf4Yxq=_60vMks^dV=#m-H`u z<{9mEXp3E@%Kd~6lLj;7LYT}l*_-H{NLG}Y2z8^uAT91%H?3_T_7C7CS`$s*wd2G8 zI1&uv1GJAP;+31zKW@$y=}bq*A}>5KIZ?764TIVr^n*RO80YgMkPeGc-MH&blfQwk zAEOlGf1|yQQWQPLVp2A5CH1TH@D1IkV%L!F;v2Lu(qth>o7tP}bk=4Nd8C$hc^BQ3Xrv{U*w5CHlVqftvLAHveC!uNcl5ry()$~n6TtkP<=ni0=)7b(}}N^(mnuQ!&HuoPL& zGbXE+k@wJ72Fo;4X)V=b&o<+?T!l|6F$l7Jpqyr@-9To4qneRSa@|Lqr|NPRf#tdBm97h>>Jd=6t++9?;_Ji7L=(vnjHlstB+q)nYp;E9&e&#iG zH?Z2rxo?O`laM>a$9qJ&^?B|S@eI%9rqstLp!@817Tf)tBHjxrLM|g+grT;Dw})u; z6z5nw>ipB?wN!yJN@#^RKKF0$AeGpQJW&|DVb4~)FH$z;E_JkS@L&#+GfAhZ9?x(p zy6?P7bWPu~w-&SRpD^`wS0L-%G|E=cStV4#ZRzk+8`@W`qaX}Nxr06W-D*YsAL_1? z(5X$2cU$63ackpEFF0Mi0`eFoDg169P+k~I4=E*kd-XV1c6cwH zb+Cipbv`p;xZK+)Jygo-5k?h#t8(01?mV$OJN?kstHn;b!g*E9tAZZ=ZBRhfVMsnD zH<9{ys&}2We;Q7&o%rf{;0N-N6`hgR%YT44k(8!9oncM{im~V z806kow}n&*WY-z_yjP7KEzvEWMx~_^7`BJbN%x?%MrolIRJM9YLE-K6KEjjsQyNd7 z;jWa&lbL{d<4ge6bwc`!3QPqmTDM`ATh02NfiKzM{lsI=cH^OU_{1J${c=J|SL2f& zspLRmt*^9(Xd|)LhjVY9bKm=iECK|MlAFk+tg_j88_H>}a&M^r>2b6GHK$Qv{ik~~ zouAfwyS~&D{=#m`R+N4+%T?V));Vi~yAUi>L1hxB*D9FX>PvIo!tg-X2Bq3p@i6gzO2Sn@Y&gklXtqXdt1Zp z-R>Vuag3(k_?0fsD(|8bA{y=PRFO3Gqc&OZqaBfZx-YEv=5lK`c4#aAkn7_I?kF$t z7%iP$ZiqVg8MiR=X`}VG>L+i#{l-cU&Z{N(nK;sOsRHL>Ul2t7rTj3O=LCV?pFTq? zCj2H@1Q`ZgG)(QK6kz7j#*3pD&CJwxmNN$mnoze=!oFt2qI8@E*H0C8?lm2-jo=)kO^AGA>&Jm)axW{bivD}g_& zqaM;MwXmGTJ#0sVpDoPqdJfW~xl>K5r?gSZ$>*iyN-gb^mP5(!j0-IdZLyO`g_P8E z7PiO(yhulOE5hk;jUBySI<8dKHZw;shKb#ZZb7`=4tJ8%jIQG~_oI@+m|~>RJE&vi z<8IK3Yc*$<dK=3$WB zXRUtdK3?YeSC%VkagAzfH76@|o6=quR7HC-6`DsHq~_N1s#o3GR-_qiZ^83DhtYh8 z5>KnFS@I%RhH$yM8p4Mu25|ymW-e~ZGZ<&?u=Z^CZEB9c{Pph1V)A16x$giN}_o;?~+}O$o8Fmf!V6XuuZ0sb#e`{JEpm3>>N%vrgAgjy^6R4nR(hO|I&9D`PISh z8ta^y-25ZtH=i*fc9%GNs+=E>(OjQt6wo5P9OgDUix&dAn zY_D&g^0)KEIt^3lJ+pNvhn-q^NB1hdw1?kg>M49*)UEDPbAM>QWl)dkPps9Es!MM< z4>3wLr2&|cWlAmSEqv4+oVGBbhB<@4=e_VANZr8?H&+LvT-F`m)rC2{^X9Zr5p#?^ z!>gqf*9PlZjWv1`^`5ucPGqHpSvO$MLC19{Q@hLcS6Ve?1hK|*COb32PrTb}tTt9> zd9Tg0!98XTZ>l;*KSrG0Q7a&CweN?vn-849Xh8LrdV}F7b4v72Ymy@QQ-+WC zUmzn&Z`497cZB7O`tsvKRC?>aw^doFeK#_O?ey2wD!G@;z1CeIVIi(bkxxMapr!q4p7ToO#dM>-+{r-^EuPR`)Y&<0_RT8=aK~OuVP{zB%P> z)tYYYbi1m$uZ_>APgT~_6Zqv|$hSoQPf0Dk6BB`W^RG@KsezXOZ2ME`kyc5ctW+hA>Bh-8 z#%>z?9BqXfNo9>{{#iygEv5QNF6f>QMn*Lc_Hdi1owc|sohx$eU5d}!Y3(yp+5Oyp zV9hmR^}56yhTIS=MrmdQHhF2B99DMwU$3w_P@AacM7O3n{rb{!59?D@rl{WLVdaPa zaaeAnmOKn>a}T$Za~wt1|JUA|z}r~8fB*Nir*jY)GKUPIQi@7MB}0i)Dntqm6jF(j z6f&lX22zQVOp!4e%1|V8hLTxiigTQ^_r9n7yw_f>XYV8WHhllT=l6fTI{S6@zVCIf zb*;6o`&!qUuIsZ_dspfvwj%wo`TvFS-xRIA9!kXQ_VGUWTy_Y|w{A!G*2aRor*XMe z(>$zAVzzgQpJiVfYaie3Ul0g|j|6|VwlZS9(cIx&lKuX#<|)ftXzd9N4J{5<3iSzP zS!4ay(No!@bLY6z^&@yn?+15dJZGl1hl`(R&-stItDQHz_q5_hKdrg=BJFwpBuI+Tg{!n%=Zgy{tJ{{YZs2FaT?_@?Wc!-w& zZBFK?teh-+xmG^VH`FWKAbfkUzcI=)@l=dP*Sj5zN`c~5OT8C!nF`nh4#t;bIvRUq zqCEQdDCe~{^)5+#t=W1xvk)lkx0%I_h1zESRj?Q?b-U9iHe|F~U;muhZ$aZxY}vj| zh!e_&fMVO3b?^_gMj&%PU`)bR_W}?`Iwl%CzKlCU^Yv!lN7ybcbjN!y`QY&zg{*nzd)in= z`w!}!&0X4dr&auf^SWLj_*F31+>jWBPw3OmBW_9mQ|&71eiEzW1)zdN!P0mhPl%iR zhumWBC4PQ=9_J2D!gg{TYtj|8HGa&^b*gw}wHcf;^O8O{anx@}kNu-I8(ZA`dSAb* zyTW~yvoh88FvEAgKG zx;cur@Uzwk#EHfFu*A&x)SN-FMcSg!*6{6t73?Y8WRx^+^skN0Jk=#yDp4>vF*MA& zfpL2wW4b=c?|}{2S}#|-*qEkYsz1Sg-{s6S3`VStILW9OC?u=(x#nQ2zgbS-@6YCh zgD2emPIY&V_ncP7x+=6N^r^KvQQuwe)MkzT7m%%&c~@v9%?{R`W?xQFepfGMyrd2C z8e@6!F|*1Kz=+SMO$_eNI38IXaP=jLQQk6p0aj(NIPdzsK{4&aO4>N%c`Qb&>-)Td ziTlizf$Pk@-hucv@h|K_?#~`J^4h0HCbpE91}`&*B|hL3u7UV$l*LxWb!&ne5yqFf zrN%jOfv$`{Z)AQEcTdKPvis~h4<&N+qvqRz3xfl!6IxI27I&W)Vs>owwU)ZYJoM!QtW4G8p`;)CZu!Y95XlcLZm@tP3SzUqgOb$TUHZj``#jIXS&<^+)+E?dZ9{t?gA1Ir#HT121C+CIy$gar@ zR&;Co3-IE57c{PJdR4FuMlo->I+Vg~dIr^t3t7D@QN0{IIYLD>>80`b= z$YHep80)=1`d2xvqqjvjxf88hBaepPv+iVtv4lA^aWr`86R&|6jVCyRZYJ}l&uB;Ix}ds&hW@?3#<|Je zln9ud%b-0a@ZGg8>=4~-xQUPOhFZ(%8X7A$)mc9)jeqX0L^<8l8yYtInBQt2>Fvn3y9_(bT(Z6Aaw?8q=9TyA5 z4!Vo1x*6p&W(FQ*9AemuVpZe4op1aT`qNg!;Jd-kt;%|sUjGU_7fXBVk-Sd+5;H4! zGVrR=3yLlM8WF5#jn;S2j*i*yICcFm^+EwNbbYXqF~FS^ z9hNgZ_ZfSURwv|TEXddudfeKoO?MB*x<`w|KXcn^eU1B#33wF^O9YK+R^7mt#z=p? zQ_pks+XETqjcy=XH@4It&FK_FwW;pY-eLXvz?0V1dJp=3A8Wl2ytm^^qu0mFcp?2! z^DS#!;5Dqu3tQc^WA0k}22dPU+SfDJ-J%r4^!^ zeVP;OAO18H#jbw3HB6iAY{BB~5qqpt#ygb2Pt}UEHvXAa-B{x-wjXdV^}8iDdYhe9 z?sfVd!D`{rfhKx!cGoY~I(y&8YDTYc8X1}FhL<-V0;vtZUws1H3k|zR?nPDeQ9Y0Qcr+a8?>dDhFF= zjh&#~$bFv^&YEIpzhC#ww#H?6&Rq?PMJ0U)do<639&!bi50~l}`kU~9Y3IG3c$Af* z9*G9tH~tK3Y{q4oR|VR-^K;6b`XINk-QC^f*EJV~M`pH&so!MVzcZpr#Z@>%KF)V9#AR5?dhV?p$ntssBN6-J9Q^A?$cGU$YMfXs1 zYq&PU=^K44y3i?)UvZAU-OR*ed75@J{qvjtWrk;T*Y0EGQBS-FX9wUxUD2!-_{jQR z%d$Vtoflo=9W;&L6f2Wa;wsk9F2)1%N3W{$5LP@_c%D|rGJ-n;<*b8xM=Z!enNHMm z_r~6cPLA*K&Nq4n=7sJL9}FHahim!0((%JNRdY)?O^v3Zk3zews>V9JLKALLJd2*! zuC@kR+x4BuYXwf8Sn1v7Z^mPOH6!pM+NIdk-mi7F>fsH3E0#!g^&P>E!ABCnFD8Ss;b{5ZoWy9T=&VwAbnge89;`pBhhuy9bwh-LWC>rn<9w^N4i1St5-A?~$Q+m$?R1EbjlbsH=xh24 z`t~+jNA|>qY7NZI@S}h}#tU%@;R1gWxbCm8r&xniUwW~E{((Q#Xdi44+^4^dx9A7j zV%CZ8af7kVxmj^9@p$0X&_QeoI|tgE&-jC4&2u_MH+X%R?bfosOceAFpby{DJ0!01 zhXB5diMgKP^|nj7`?Rs_zKqtZByQsr0|(JxQj%6{DdZ;f-A z^SnF7KcIbU_6n_voC+<~pR`Nm7Rg-|YvJCet+tAUjqpyZqh1Q^q37c#qdB&wHxJAW z^t3+1I(uY-wK3;8cLtUw58{7#Nus8{E^vFslE^$G7JodaPi{WDuXu-61h15Gfq;KJ z+9SHmHOK0u$3 z$9O_N;LVRUjc#=M8?61SB;}>U){i{vw{^xD3CH4EE~^B5k#diF$u#vK{fTzC?S`ebmYF#0Lq@Uj1t6{!31MDp3?sHbfq@9X`piS^oHEM$KGedA&Gp#Poms|7BU{%N9t zKgIpQE@wA%2kFg26Cz83Vf_s^=xkv05O6kl_ZroMHA1fi3Yb^38yoer-Kx$BI|F~7 zPV8{_`eHPz1>VL3+C=vea5&y~uJN8_uC~hh*xZqbusXU-JH_dBrS(dQ6`V?1-k;3g z-~jyGJ0^O%lbns-5v>c@hu5-__b&TXci%{?w7Lc(#@$Z+*fy|@>X|k>J*DxF&oHmmOZ&IE#R1G-&TjAL-lKj! z)^|PYTVr9OhksY%X)Qmy7F)21y1_FNW5IWu9N1^|G$ym={~~9#RlrK+62By#zqRqn z3Yn|TN6dlxv)IzlXPG;&igE^G z0i%W1nf=Dw*{N=WRpdbJ9(IS`!}j(OJP4lFUt(RqF6alZ<2(BqHj0x#H9KM46B-&$ z1fJ6G1?yss^8>zDSF;nJo5(h&2Le_=ABUAzU%e3seBZ_w#22`|wE6f`-G$}FZdP)( z_`BWfk}Ip@u%Y=Ggo4gSef>B#K!e!F+JGH;i9`$bw?5YbMsdSHCrt#i_S3{5*59f- zcleX&W7o0H^nrP~v6-1{3HzmZC1upUO z1x0Wc7!ef%w}K_rz&Xb*?M_PE$*IJ}j1KJk9>yQ{an`O5VQtjVe#U!UKQ}Nt*a|!A z!{GO9@pjqk@jv_2F5!38pER$;GEAJ2dA`{@QN}$E0@*QFoQB%czgr&!f?>(P_eN!H zpm(R!+g`wWYCYpB{Nw7I1z0bi0|rn-Y;fAKs@mOLgvIwa)>OvlK0CV)xtBW++Q;oK zUPUD7DLh#@b=@eSmrD%v`eMto5KVolzS`KseE&Ip5eO-VyosCzSPUGC1IC)b^TA2h zWBTKanOCp^zBx9?-sC&x&7n6!KL^&BbM>}~<8Ec=344ugu@|~NF+;zP8P@AoWwQaR zhTr%(?o4|{{2Awh#8#tbU~OQBwHnRyw)dLzlU>ujCGnb3%lgJ_i6z|k>}pR#i@o6A zs=dr=-*mj2CxCWj85huZukucEBGxCIc0VI=qj8gUul0zrC-F4)EjNNayNnS}DL)?; zD4BX?tp2XUzM{49nKr^N$=Q#`yb+1Qc)@>&1GtamN;DEELg@d{R@E3ol<7ki*Z_&QE!PrEK_gj2Au z{{@S|efS9N#VU0Ut!fjt0S#E=twB3~9(;!T{Hgf(M&RaAZK=M1)y}Kwl>`1JCSW8a4$mzLz{oUB-Wa#^`cCNr4;ZiIsKj7r8AaW7GGssQU#{cD0cy|xE z4aaAF2wZxW{}I?kzX2U#ZP|-=`5~5;-_sxV#kQjt@-+)Ygty6kFzN2ZqOcP>>{s60 z!&tvvf;8L*X6G2}G2g@D=oq#qS22$eNbnb9nK1>cqaU=d@vof0^=H_%A7MqHGhX1u z>AeaXUtqx%0;8=m{!XLuPFcl?XAkJtVGA?}NqQDr>>t2k7>y@eXKWLX@jf2HgD%2; z>D78+eHpk8i?G(8f$i0L#xfh2b6yUPce{kc32hzyh<}K4%RBn7V8OT9&&JliFDJk4 zX4Q8Qo*psI`@0gGt(w>a+1`bm2|1J%$^!oPoPpN|JHco1XRnEcL(H$hdP;YHx@WuZ zV1ro~ZSXGsxMjigd5yi1?)@j^pk41|A?SIoEnR7J=-azxjH9Iwu0Jpe0$Hd3%sG2Mb%8!pQL;Y$F9u z{}t>je5M8Q+&rSU!msKIcFjt%Y90k=s+bYv{MlcuTRDe!Ha?#_IH&g~^91NYL;CT@YG{Di*k2F8^yLp_vY@; zsm&o%t#a;Qjy^A5!dcDFlk`^=tZE~}*l@#M%bJ6ab7 z4LpME$dSmn$fNk_9gVz|aa-oPOg&%4e9z@8kpJcU3-dpfe@(tqnJqG}%3PfJcD@$* zJLJDFU+s)4p;hL5t+ns5pY$R&OV6MSUj{j=oii@}Q}n9btW#H=+L_%qdvjLNtO+L< zoNSy`D%(7@{ZyZvW%w9(%DFCQbj}00&qrs+4#Zu1E0)}g-TbW4-sHcSsHz_~`Y|Id z88m|Z1BbB7yOrJgb3k(_5Lyu07L81YmAlow{a(ax&1i4I31lb zj0P59-Lp1vQZo&WmbQ<5y*?o$63nn8(eP}Jt7#q@9=xDrZxZUaIJP2ON1Ux$q#J&Qj;?~?pau4R-DOxu^m1FP| zT#x^0sn}KVyY1TOh81{$55a!-fb*1lD>M5MSbmPwpEO#U1wr6w5vUs+7>b2&&bT!5 znaqnb`)16HSdsU_tHO&ScVr&Rw=aKz0*mtBoPTq^JMwMM?3Z~R2!s9}vgIUbB+WGUHG4Xb>J7N#S*V=EfDp1+JA$BllRJNY=>d8RX`mATOS7ndM zE}Z>hR->$8S%b50&gmV!8~vPt{qQO0Vf^n#f>O6O;c44g>F>gM=bg+~EGzh0$cltA z8fPqu%n6SP%?nly{u*!sweVdXjAeeeKo=~jIvMX`#kL=BJx*W3f93_Zl~cnm60aJY z8l4|K56ppkVx?j~M5ltta7FZJ?)=<|xzlsM$vvK1H@YplEcSi;xLv?4hb3-1tY}}x ze)D_QT~~u76=vtN3%+KL;59V>kCQ6F+~9Yi5#d6Sha&SL@koJ;Un3vm=ko-pBwa#7 zLWRO+q;AHoncrqk&a9m|62CKaS*SOwreC7hTZh|*mxi7T&afJQXj#R5HGU}C82pS) zu{%NVt>k{=^aTAqBe(Xc8?&!ueRbfe3v>GB^vk&uWQ0$0=H|AKHDY$qiWziQP7dzF zTKinaN%Qn!Mgj0)8et{Al0AT(f%`-A!;kTX`iHBB<^`U!=3*h!7POw#;68AIJR_A1 zy(@?$;~D?xiRS(d-Z6Z$taw)R0oG>?{K5ytzr@q%()eC{iaw0J7JD+*3Xl4RvCgrD zvAg5t?W6WAuo0JgF_44@aMsDESRUR925U=joit_vRg975see z4!wnk+54e|^mn5|mxs3Ccjj<9`#$T+K+oW1;nk5P8L^BH(DFZr*M+iz1B1^7=g~eM z0X^W7(0QQ6_tzivA9PN|I!D7%GnO4c>=g8>dOh6IPQ!TRXoH-hr}kwJIQ3Lco!n1y z&q0UW72Auy;(hj1=ZO0ycp8sk+g_13`wG?w8`;g7Wb6cMx}5o(Sroj}b%ABU)}b4~ z;(aL4#Cp)&02WIX5O>z$H`@Wb=35x^Uy>NfXuF43%-in1g?)T0rW+#A^!t`&ZrvBk{L4dE}s(eO^Zsdg|^u4U9tT@AR7YB{bm$L-=3r_}c_E8{mQfavvt zAMoE{HSigCiL=i>0CLLXv6o^MsJa$s0KR-6Wa4qeqQ#eWOCm;Dl>72WCY_|E;T z$JD`Ab{4v{7`{@kv(|VQa|3ZQ+lAQKPh+%Rg89nz%xQL@;VRG)T7yv1k~0-bB1cDp z7lkecU8#BSS;l6MS?2}b3O*D*7+IN7G&7hv4}690kuUHXZ4mAl9v+?%-Vk0AJ~#Y) zFuzrp{lIok$9QwR7PD!mI~^00mi2BMrvbaMyG}ioy*I0L_VMgsPI)|L1Fryy`aa4tc}THNAFa8pcD(NNiMNfljMa}%vUfOL-6rhyw6bT$^yud~ zWpd6(yZxL~Ik!P>=iJ%3)uLZU$6zgYrPJ4a&32JQ z@%?@*-V4O5AMuzkV>h#h!ZpjOgKg0;c5}c2W|wR^r#{u;Ef2)nzYLP!1vHOJcv2L_ zTjmaYySC#YcR4-IUhsI@(S}E}wt;P}9x^5x*PCnb#2INFvHAz@2woieB=j;n9d@8v zU+uGTj&ss=;QX2KxUgt977Quvf5KuwrmUpaT|=&8!9B*c}7GZ61k8f^T+PN`QPfl{xg+oNB$Ceq$*s0Bx~(za3n?m&_`7 z)K(A700ZTdpb@GN(nE9cVVoVfAW$w)Huyv6>BzkqZ)FV4=#UY~7#KN+F5N4g zyD}cmxGypzR5M^2ANtdrVew~T&oII+h6mj_&H=lz-4oyS%Iv&;oST#LUe5D5cjsK5 zb6<|Z=;^!YTd`ZfoJiOe-Da$}-k#{AeXegYhM7z8vMg>5*o(X2- zH$F1B3g4iPSdeTmyRm!S#CS%3lM|@Q(KqP+5bp|4^LDs%-KSYe{>Z6{J>+$GH17mq zjh#L$wa23&qVeYTR6EPA;M~hd?UZvF_@#$Iy&aA(&!x=1+jC}FAx>hOhTp?O%tC*q z_3qGW>8-H_x*0upJ*y4R>nreXd=<;)b!I=SJh&dG@PwZeye!lzv^Q8N_+a2V;k6xn zCp13X6~vwD8I?0)k&Thfk(1ys9FBY*c`i~rvLHMwv@~$Ycqj3k>&2VK`bQ^V0s0tY z$dl-i>GlS6&C}85x!rTB&|WL$uFbt8`Z=S78nL@rckLXXU|;KQ!}|IqGOdsB_iJNTFeibOyIGr^7=!P{J6Pop zcb<19IlG*Yo8!EWhgomzSPHN=(HNYbd*I@Qb|E_(9KQlc-Us;P*KszX;hM0YdMDUM zOIWkG4nLWttj2xKNkPKHy8xJ;Bk=m!M&ES=gqHwlUZ3GtI|ww#D)@f>fLHeicx7J* z_QK1-!l6j$vEcYXu|RQ9`>qb=3zZ2sLB?(Yv1nwZN@P*ES=bNd18ME?NbQW88HdAd zgFhJwzb*Tx=R|XI{pd&W#!f3xgfd;%el?yMTbp|U_(Rn|(#Zrb<}rHT@A2Dy90Z;c zAOzHPHn}tXx3#N{dszdiV(rH=^h;KgcIb!nhmE>sRd&o?2)q)gf~RCPD{gK8DX5mY z1%K9+dL?`m4&fzx71m`%+@|yZ)zD#wom#AIRCnhxQy2|G#vZ(I_b_WJVc%{~=8fh9 z2ciV+^-<=!qumF<&zj1bep@`bJ7U?{1X-Jlk5)_i)F<%*+oheW*GA@^(fvuM_9fuO2ZjHx2V~V<{tmn06&daSM#G-~%mZ z1TYU@i--7hvF^4mwAQm)^$FnC+ot= zNM9gntyp8tVXv?a_^R*?;R@jmaP8I50nTSx9nOfv!hNy9s1yuhtJ&QD+9_-|!vFdlRuQU$3);eM z>p1b(V{Ps#Gtn*Q4+l|rx}IU|15={_ z$jzIyGQ8zGjq|ad?q$ujX0ZD((CUYEK@axa8(CTAcD*zV`^@AD4h%RJtl z$x6~m=5KGgAZTJ!eifcLQ|w1Teftsg%`I_(=u;0|%5U%=80pl-Q+XwN_D!(uo?yN4 z4e%S{jLGi+{bL5}OeI+NTZEqbjMISX7;TN0u~{sGH}P2fkM=Phe#qKs-3PW#Z;%6u z;^+QGFccgScqWh)_z1n%8w`deq1o78UlXbtY8)CA`ZlD83x|t@t1=Sahh6qstBtWH zF~HmI*u2d!^Wy?;UA&OifZ$&~zB_h+QI*Y@yPY8Uz?*dZ7Qe$R;Rr$D1%R&ZwMrtk>HaAm_kz`+rW=R1a0 zV+md&@>Jx($kcEt#&kCsUngDyx0{_(KP@m6l8y_opPzmIkA48M?H!LI{Sd0BR5N??7w#cASv z&-~#=yydsYzhR~5OIq!e_-;@voEl}dKG%^WiWh;nCeWM#hSx9(hMg*KJoXnx_cA*!?!cK zi@Jln0<1xd_gCWW8cAHA_&8CM*+ffy11qNEjW}!7qs&#NXWn4FVf}1fgNJE(R(f{^ zONZ))YJBgNDnr% zS{jYDK7LvJ)#`%1Sq7AYoo-qD3Z`%--pBD;@vX6Ou|fFP)r}pFPDJCq!}w%dG{VYn zF?v4BDT*Zk9tM7xvpuVG?!!KKP>da@hIpW_K;zBDF8=}$+V8?^YXe5{W>%sjKP zxr$wl5&8!F`xmnB|28wHuh^Mc#7@_7_NnHBxv&WIfV%K+K0RuAXN&!Y-QDg4`pRQy zyovTMH1l&{3vUDe?I5GPqMUp2HTF6iy=v^wJmpV9_dST!)FXI>K7l{)O~yOS4Lf4X z^fc(ftplIKw;{nDXu6uL7*>XB)kBYkehM`Wk7A8{X?QI&{H@_h;dbGY;a|a^`6x6X z^aSfA*N0XFw+Dt;w$Td<;cF73@UlCJ&R7UK#By+6&v*KOEAbt8C;j5r#sKOA2Xzm5^=Vi1e6m__zx*LWkCMJ4I)e)fm4KQ#wWsDAohy*Y>w zne?n9Sz(HqCOaR4*c0k+_;>@}3(i$p`dc6L!%sn_(%2oY%Kq;V|5}i`hkq zr@eF1UdVVo8y$Nk^Ras1&|c$IVQx{wX~7!dBv4WdySLKI9cHg)u(!*rOAnI8+|0#N z@ErVQi-34{u{qnUiOlWgWWBc;nRE_L1Q{?9tcEU|8!F5kb$Zyr;<9Jtd2qFcMfygX zMvjM{3hUu{q4}X-!Sg&8x}O!gW`Rv+ZR0COrWM)yG+D_%#a!l~m&H72J{U4L#)I)} zFoRdbhJvcR5VWzAVByAdtANkB2%FvSV^88`H`sRUr<`0``*8ZnZ`m0cEAI&YX)w15%W<>zRx@WUU(-nzd3_7|K0e3zlI;X@Fm>^9^lh@WqRqSI1^$C z=RquHcdvuDky8U3Vr5X@t>NnEmgk*jU|L_}bauo!H1FYC+l@W47o3k7CmXchw_FW; z*}YyzkjWlJ?j8VF;Z1DC=CgA*1eBh0*oT{96=ZekSfCZVQ<>n3eHluC-1%^LWw=b_ zdPZ(lB9$T)A{mig;fdi6;fmpmaK5mKmTMoH5PX3>t7kZcvmv|NkK(cL3cDbjCg=C? z=CBLW#`%_h?F*21HnSqTCptKKADFl|M5{$Fj5d#sVfAEiEPs4BsGLtS{&GOwDa>f% zDX=E%f{xoAO#6MTVV{SsK@9)E4U~I;@g?5ZAvhKTh4eM>4sQkRWsvb5`0EwT0_ILo zq3;AcY!jFq_2J$vc%`3{c#7U*r?pcwC(h+bOhk!KL+wFwCOcv|q4am17^4AAs zi=Uhv?At1^DzMXS1Lu1CLGa!7anA4-?GAmleg(L9pBp(wYjcm;18d?Tf%Cwt{t+Cz zo}sm&YG49>AI^khwIhuqjUv?|T4W0-sBeZBg$qX-M1FyLhl2|PFX9)|#`sbn$XjlY zpK~eg{KV~G2z2odp>bC_S39rUyW$PwTadSgu_(N%9X-U}$gbRq(O0nd9L4V51@W6% z+jt%9*zcHGG-GCdgV-H(!QdSKtuy6H~;DKF3KldE#QSTYEu(A0b zy?qIG6+_I|SP#9`EWvp-pRs0@Lz!H#Om^avT_RzzVtfp5`-i|d*hpX6m2+(NGv9x} zJ=b04ybI3PM)*|)9QqsGJKYySyetFy>>lqP*3@ri1)v#dmSxcj!$BY1K~K99KfiL; zH&$DEQ#)``uy61fm__SC=Z9N`p9l{FgZQoRmLqER3?(|XkG zZ5&{A_FdZ2#mt8mvZquA&&w6=EwqP~P7mi0W6xLOan@!-v2oyxRby250P~79QJ2y1 z3$fj?y7YG+F)yDHpUY^q6#nfOg1Yq*{qrnP?M8E^$_Mmrv%t1`lrsOQ`|S5$Y}Dm! zx(AK78DEuWbUKRucOxG4-+-?)$h-r|n~3#Nf9wIi!iMEJ{MoL-l4TQXAQyw;Fo|*7 zh3w~l1j=)9Fh6F1sxs6a#*X}A_mEqJwWrqJ)8wAX4(bm7T2L1YVk_`5ehlTn?f8<_ z^)amY4=@dDkd6CrWubKHH;(r+l;GAfTVH++mYU2@@a5x3KkvL zql>3wpAhyJf_U*fy})RE-lidiBS7X@z=*gxXZ@T6OXe{wKXzbYaRG}wXs#FU$++}jye50}QM;uxp0Vj&?sm5uJ}CMIFuRdea9$3U| z#h&_YEd(ONY>*mif-^IgzV;jDl1JFNm}fkTH=&PAZUi-A9u`Jz=xg`0sy3B1$sSlZ zS@dAd5(lxvdJ@mD7I;w9WBySUja(1E%V$78`3eiBW1P)#8+J2CKtmk|zEDMQokpSC z$|GkPpofoUO{WRibyGm8>`6QQKCqqsAwmCICD@jIxbeX+gL}Xl>mJMr3=HUjR@O{X zRWXk;n(4qP-@EZjnv31Y1nix1ux4rDukfy?@45|Su?o&A))af$E$v$Prk_BTzK`#S zZ)An+Gc?)z@sWHcFqS_Azdo=_)5EQGiZB}elo50d?-?{pE9!6#BfC1Fy7tB$SRYW1hZ}_g=(kWOM_UdaN;w(p+Lhv2J<=iI}2)2nJHaB&zf}`1KsJcB^J`&PXSE z{cQO16(ihdv54sk@2-bqow@(c-;Whd@kBi=o%(`Gwhgq);f!sr#a>3i#`Fc6nh5g<8z9 zuvH8MiqEiddLR3r>R1#^p>@vlF7sx(m!oBSalS)gXFVu?JsE92!S_RUYr8)CCda^n z{Ukn*Qx3}5H!%kNi1FKvU|a8}2Ym~B(3)Ofu)(S#XY+XLMLDDA7Ie}&u$TsGhro4x z22HvUe1i;5D{08=`+lQ2eg(%FeY}rEyv1iQSe|XMWV;X_wmsS!P>K6uX_lW|g?ZRv zJ%)8xJ#b$NVyhLvHl#X8JFSrA_V`Y}0wTj6=J&U0&%o^&__>zFj%K-Dh#vY)ko}8- z8vY7ie9f3M&$Z_8`4Zf)^#b97m65Sv`i}%5_G-|?KDKIF6U_@azk4{Nr(*0i>8v9@ zq%X!k=}p>BKBR0vGvf~6adc#_<_32YW1#w=f9=9ja0j1l_KzUHPG(K9h@Az7Za(`W zB(M({Y;y@eWfx~Q{;?Bb&hsu-2G@BLKa{cmZj}8He|M)gXhm3XtsyY zr2~+-U$FzKhRj!DzCRys^#)&eERyyPefuNq>pTGWYB3sK#cp*s_WkP6N}FO0Sedon z)!63@c_d3%m$+;o#!n^&fmMmu-;yP8^*6w+bWxjB&`-U3L@uHd^g?`*Zf2w%Uj7 zTsuEfIgVYgrku~Rhu!qn-g{V@UQVld6_0`)=(S3;&t0q&k7XscGN?+8v18-}as3o% zXVtJ;JO+~7LS|&!8J$EJwfXvPM#w9{4WCSDdvS_cP5m5BklY8>WF}T(f!!Ia zyOW%a)q++3S=dn(g>M_M{&)r6b)rwa6Un=nc}xLToQ{CSxu5@&Sk{+emeZLNY__uJ z)CQidVx)E%{zh}LLnzD2Mi*#*$y~iWnr15db=NQko22d33bJR{2F*2swmKb&TcCdd z4_DI%{KWq8A$>nw+lfcK;M#jw9gbv<)(uRu8^NS21^>eA{+FUQw;+ja_2$@-*2dOU zoNzWAtIs(|?Gnb;KjU>$m=WSVjH$aaN_>Z1q($r|onWkc7|XwGdZ2IN>0G)+UZ&(OVpg#SU1jE@bCKW52EzyLJM1xhYudt=Q4;&TerxTCVs! z3`bvrU*q-H(SmK@aaAlc8!;O1Lv7xLH(gi&-Z zF!Okh9&5Syjd%Pm|Y)Ct)Iv(sK1T| zD2-H=z%q0o-}Cyb*u#E7d)~zgTaF&rE#o{|{gwEA+`tanE%XAN z;m>pIk>A70Z*ltR{q)`^(018iDNO|lvM)1{$7#_wvg*2-e&=FFhB;VG?*PsBar&B{ z8C$$V{a*4%f~nlaug(hpr_8xqF-Ixn74SlwfBTdB9UPqFe$LEfKPy@JSplm7UoYY` zy;kh`J_ml?YvuNKA)*)9i^JoCSS{Sz*cOh}Na@`#4`X?AQyui%& zG5Bd1ThIz0(PvLX3oM3ruR`Z?H0N?qeFw5XaTgwH)j0+7J8y4(^R`;YYY=ymZ zS$q!4V~Jf49@XQ{VWs3Sx-TCy;tHfH!*2=Nb!pOFfv3RLaJe!1xEr;4m)XK5^y7AR z<98yj=OwlONOa_Ub`vJk<4uMayOQ&_OO5MjeO0j+I2ZiLi?HLpivM!7%{BT5^xFOT zAC3;|L(kq2X*7e6xI`UW4UMc#bc@JHBV&p^X3gkxL4r-;)cGg-|J@?D-4v38t>I3Jx~iuKuM z@U9c>a{#-)Qxl6>?LEv|S0+5D&PxAzj8qNEaT%Dc?HTJog*>$&Jj~3*M)w85%sqzP zLY!-Zc|alfR|W27at8(kmL(U#xi(n7KaO-h!Yr*P@;Du<_B|j;<>LPkpcNOTbzgxE zdqXr~3#9Q;v}Auq&tuV)LRXH*#%By8Uok3dPao5PnSE>IR`e!=Nzro%#jC!<<|vEu>+P{ zz3`3h=Yv|vIP6iLbp}(kH5_}8xy^9muV8gZ!@r;gEwv@@r5|$lCMU0a&Z@&YR&6(< z<#sd2@z~$2jAeggR)d-_FRzJjb$#w`WW045(&fYFy@`W(Fl18FN=RLC+Ic>B$dWns zM!4F-`!L=kCE>S_!fP0(wd5^yWR~6&9WW9pd54zv5&iWPdY{dl+Hn%DoJT=fUf@|MIE^t)v^*|apC73Ja0d2slA_*|WJh^mB? zz^c0#J$tv|ar?b+u_s8G{n2`F(S|=k4yW=xiO&??-4Zyv3Z(w8 z*>7KuE?LRS_-t_SrsJ=%4jvv)sB^HZn~av7%DZ?M3a_%qGXkrfm)SSzh3!s5dcBL$H?^3Sj+66-Tls@CMv|l6B5|@gC`d zwtE3v`4?EN=*N!cYpl6_%u2zV-~heAdwGrC<722y=WV@-?7up{4j1$U3~e~MNgLo!Z) z>z)h0L+FVr)Uhs7e+jZv2K{*fvM%g3+c1J|%jaHXw-;J;1fK!S(0XIhISQ_QitJ6p z|Kn{q){iHn;pc3^Ch|5$vd`TQn=mDlB{aWb1z;hevx%`7zAm8W!Adqs;Tz=iTll#&$=m7VHjO;y@R;+@3;Y)PV<+X$fwxgOoJ(^(2g$uGsXJ~)m)uId+cFAx7#Z%4 z6hDhJjzTKmfp>2qspF8q_u$(!dXZUtPl1yk@cfOWR13)*0?(g>!mV(xHk50?!%NXH zwfL@t7CHxRy5t_E4Dn>U-%Ht6Q`#kbmQi-0=Qi@$Ot@%o8we4WFte%MEO@bu`U?)M zAnq!Dzk%;tkfA-u&oNr*4&KCyWbKzG>8(jp{2m$GNcd;)e=OV{muwFoFm`y8U9{Km zetR?d4#p(E-{JYY{J+hUm$7tx3mKn7ZrhO7{p4|koPMSBLJt(Cx37UzUPJ4z%}4a+ zEs@AhoO1C5ecIE=*Ne$EI|!M40`5G=-P`c-1AJjeqZ1!NQ*@wT=?c#uK?C1TE3S)% zyb7+>q>mHQejb*4h0!?$7+oilZ!e3safJPkJ-pLBl<7DvC5z8VN+x>Ela%iiEeK!{ zbUsdYlos3aWSnf?z3>+dBzpvh&|^?KNUB_DXF>NDDDH=s8{zJEaBv+wUx92bL=tDi z3nAYN`27@}@&WQZiFY%P`>9;N2{+#+bOsXp^>5Pp1!-6FPJg7G|G@8eaC0@(cT)?E zT9rVDmZhFTk5)_Svy17OF68;X$sp(PEb) zkIQJ2YkrgFFOb8j$sTB4GJGk#oCjxT(I<%>YG#tdUnSe_rsO+V35{=&^S!(mL04$a zo#b&M*^*-D${6L)DRIeU*^8jJfTyK|=TejM)VMOA3zE-D6LLP+Rp3ZNCuD0Q+CXsnYvk*D?spL?+P#q94RCKKGA-o%02D;8BSuq4X(dtK z+>vDX0r>kXyc8|Qhoh$`h0yoE@+(>rCNGTt-Q=(6p=_>YT%pZ!2oa$Uslld5N{AeD ziTg~>05}&(5;7H_{)OS3;L8Q@qZqPv4qOqvS|$F)JV8ihIe2#g+`JTBa0TBN(JI8~ zvo!w|l4(WE63KChcn>9@RG3`OO}?-5_!gsOpuUqI;p6I1z8vILlo51I%A1wc7uj5i znh6~t#t!@VKMeP?;SPp*$y7q){6eVEFY^Czl4}Q({i2Y|{cv_au>`+_cG?AxxAA?5 z_(D7H;~j{Qom`(JrxTtV+mf2{=;?IlGwes(h;Pk&<39DT(Fas^F{qRD`=#lM-K=^kO_$Jo#G$ zsu9{xChtXj3h>7ACFK*REZ$*&wlDZ6Mu$Sv=I{~PR&Y^igrjg^Z&K!dN!CKt=LkIb ziI3>7#7O5?YQ7f^2x;BVbryBaBHd3(IscjXC-@wN?}85}k~xX?AbOyzB(H=7+DVRy zk+djBl=o;NefcP-#tc+^gw_gCqX=~ntxP=4q<*1f9rIJef=Nz_FcGI9oDiDgJR~z; zk`obr3&MG!DGDZ2=l|_q_DzKeFOrl#(Nl?Wlf}QtGZU%L&v%esNJyg3+Z57gCQFyY zn-`oBb(1nEq%D^_F-8%xfQ#dApKN$6d$^-X4hWeM{hN?uQTI44Q}lFEDE7}4B-$J!h-Nga8J}R3^#;~ z^p<3P}*UTFRTa z6KzvSmlzod9Vj@XLd4if%r~UeMU(RJE7EhAf1%H^l1~I@MSmdT$+j-qzmQ2Gktcqm zD&LEcTp?Yemk@HP_$G1`rIf8etZfJmik6saD}sxWWZetG3n5#APv<7vpJ;(n?-k%l zVYHqYAF6gLs3=Jj;~Du5#J7A)7F-jfJn`;D%MhcP05TzZJ5k5fcu{am>JCwM*>i}# zL)2SKme@qQh|#=|7NMENx`KGyq7N4mCRTFPii2qT968FrsJrY_L|WOxh2E0wJePFp zBU-5Fd4;4U_!j+w>>(r-vf=H^k8BsRjfavo5}cK7OK>xk)L(^=rUJ;CkWs-8sShJu zbKE;0(S8M|1O;(#kb}ru)K5Ma(kAZ3pU@CubxO*X;DV^Xq$?-?RkVEZM9H3zKGCw}pLl;Fjff%g z*Jw+!ZAeWb%Oq+jo`^DwcPZkDf2pg)`<3OCb(Sg9^`(d*bc-mPY;}@X;th$K%RB{_ z1SLUB@Ii1&JeT)UenbqJyBHS=O)I2S_Wv?mzAwQsk&`H`NF!5;yhP~fe=<+;E%_{U zhT=o2u2N@+QcI~4v1NV4TM{`*{)x9E-?4Z%g0p$^7xE!$Dq4OtDZ4f)?Ka!Wf-Ajp;+!ApVw?w%_DP%guIVnZrnfMc-;v@MalHkgnny{dj45bJeI&+CqL&kC#GmSA zMK33FQ13DQ4TzY6tBRT|N%}jNwU*K$Taf7c)jJTJkgZd2UGYq4S)nP?xi3e(B2<(k z!RSHgXQ@rZI}sdL?@YFj)V<7Id`sz2IVp)&zoL9nx|Br8+DR^nF!`srEch(AC*On6 z;OW|1j7MdN(4%6dOnwAM)kjLUq#>_mUQ(Bd+>(tmSz~!EC0n&P+3sYmMC~Q#${`6cf~E%Nd~)JWtkpQye-rctr;azKWrBvi_W z;)ZxCJ}T!_|0Z$~d5iqg>nuwk-?a!&|4ZkktY1n((&MJn6Y0q$nUBgz(v&3=?@yIN zlv4i7oKk6%1b+KcJX1B5b(MGWnQWu#TC^S&SMft}C6${XE;%KxP<+=lA$suK_S&6t9Xizh@)uB_~KjELDgHb){m- z+R5-g`j+%W48bk=z9jv0Z6iWtnbY5);!=u+s)Yo=j>(8ApvHLpm4~FD=qZjUx_NVwv}GAZiqw_7R?qX&mN}-{ zf=ZPhE0uqW->EPeQ+$hh%DT!s8KPqTK1|j_rjhAX4MiFmU!_ikC~nHs={!j1r+WIw zzam8{hxEAVp`ujkK3$SUm`tOt#ILONnQ5u`DS9G>{1Yi;c@!^1c#6J^C7y{ed8dBm z-S59s&n2BReT!5gT*-&H7GddpR-6?&Oa6&iBA)#IEs&pN@NZwK&=f5drqU^TGF)*$ z+^Orl{8aJeuZ$<+s+<-7Rah#XN+}DM9xg-DOG`~s%!aF+{yGRT;59_i4ur6or)_7`e+RG6Zv z?!~VR6E#hLn=+;3PbyBT{^=CbuN6mC8dYwUQ>yeTMCC8;We=_5C>rWl##G}3#TQkl zy!Vp2_!g<RI2n^srVv~bPlR9isY>1uHvVNt76M1s+8(U zDyA&8yq9rRh@v3Mrg$cw2`aLT;+do=zs0@!OE0;KlX@mX)SdX1bDj}xyg|9 z^m${-JN2DPp9)b|sb3LG@=}JWJk*_vDdVX-@hi9{%byBWy5RS*CEdK#e_sHg~_g6fZ@kHG8lrpV&p8hA}Dk)UIGKPF6u2tI8X{O?dxaxh& zbg2+=mHJDkm!4i;r@mF5sgP8?MVPotl{Zzg^e2*r$U8+vUZu*DdX@@1J$CwiDt>yX z;&pnMq$fiZZS`E-ix#Nv6_4}AOWnyUm5VBadaj1dmWp}yD?uYYpS&?;xlgBf_V{^Ii#thQhD&Ox@6$u&y)2K+L*=B>D0-=T zkw)@HhN(M6Oa96jdBf80R6Zh1$(DSsQl`fcPgJ;!rS3(jiYd#Jil^>n92rkaTZ)E| zh!ibJDLtLcITa$WWQtU1UJ9}V>Z#0E<|yS)(v;zOW2w6og}nJIDyiopg^DAe=FLxq zNiNISGDV7(qL7L!u4JfuH!6k7Ej^}smN)EljhEuPpdtQLIptmIThd97k)kYPse5%N zXo=RCSB_K-MC=rGNmqr)XX;MTlk`;yRSFeLK2_hpPotiser23=KBZEnQNS^%S%PwiMWcM zN+(jKU(366S~8Uik?Cca`c6+HX{EZRSx%y6(Mue!C>S^9Id7p|q6)#=y zrAjBCrsAoqR60dpgvfvOOvFw7seEKCb(e~p3K40GH->wA8Q4BlUZF{Jd0Td1V>%#!TJGtGx9PRAgH5^z>))Xw0k2mUv6 z;D4Lm`!nVFGjRmZMZEO5>i*CCs&}T+|Cu!ZGjU`utNOt|pXb@qsqhrPBuy25w%i_dk|7v|ieDz-cYT7f^?yu7LtMSj2rwCJ$rRb?& z6<^(}Ulsm8_51%S-@jKs#mzsTTBZ1ZRZk_uf3C!;w0|!4|AtuqiLa_g>A!!r+iV ze*bFxvr&~L{p0*oasD{o+44A3cwUNCe{Qu^&^gPd2%Jew@ zDQ^|`KgIjA&lNrO`)AYrH{+d+FMlVGzgiC!U;O@^{QlpkN!8$NW%>Va-DHja+vWeO z^!}4%{!@y)Z|bkc&zt)6yZ=N@JePl`r~JG3ieBE|KTqwyTN>F1rROf_r{DkY zy3Wfrb@xA8MwM&&@1HM$Or_$BJ9YiP<2Sulsq6o#vZ);ZTlook|BO~%3jgi${j;+D zlhmpvdFyv}is|wHD2?I>!0_BXv2GELqRq(47fEl-b=m%^Vd$v=&E zw$lAmx_>48?>X_GN~Kc#aV|2}AIJHpPtW$&R0;F`o;|m-(K&kx@kCJn;~42rMchA* zpSQGs99PXd)6=BqcIInYhE$p}$5OP<_Ed(asQjmrrs(|nn5i`BS1Nrf@6*eo%ASfL zuF`3wU!VO+daS>Gou2z2r9M6X)9?Q{t<2?YaZ+VS=gFC4r{bnxojI-u73ECN{jZce zo$B9Br)bFZsWPX+Q(>oHsW@l)Rpm*q+nM5tus;*$Y-#^$xGI-Qm!9|O^~-y&Qv7#+ zQ+b@3ZtA(X5^?f|ocZqev{Y)9{`WD?6mn+jsn9c}Ne@%Grc(TS^)Cnh<-orj_?H9! ma^PPM{L6uVIq)wB{^h{G9Qc<5|8n484*bi3e>w1b4*Va)0e1!f literal 0 HcmV?d00001 diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/__pycache__/__init__.cpython-310.pyc b/api/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d03a806321a546b20f20bfc68a857ae5bc38c8bd GIT binary patch literal 166 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vLKO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaM2qdE) hAD@|*SrQ+wS5SG2!zMRBr8Fni4rE0!6OdqG0054^C}scv literal 0 HcmV?d00001 diff --git a/api/__pycache__/async_quart.cpython-310.pyc b/api/__pycache__/async_quart.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f3a9048ec42ababf364e26ff46ecd2a64d177fa GIT binary patch literal 28257 zcmeHwdvILWdEb5Q?%l=WK@fb3l&&a=ASe=)s24@cl9#|zghei3V5tXed9&Di0WP=? z=-j)cuw7V^C6|z0x=!qN65Fc5q|>Q7j@zVlU zR7srbLG|~2=ia*yfTScR`6KO;d-mS*ethS9pM!dTe@emU%WvMl^f&*JqWm_!bbhws zSWfS7O!pe4L++N}_7coBSNBB&(_U6hFr+ zebw}Qnx7Mu{_4Q|Ky`3_kl)S9P<7kxASwVGE&_!zoWWyekZ^8Rd&tq zLVJCc-SfNochCGD{@pvj7r*JsZPk7A`+~gt=WpkwzQeX&Q|BK}-kCgr{AcaEoJX7!rxg2c=eYCe9%a$6@39Y}>`8mnK7=x# zNbV2++2`yr`!GsBmh21v**W`O`v~GbnLHZCasEDg+8%cvwU6O@AHMhFix%t$g7zM4 zx4SrGKV(0QHXpYiu}`4QC+tV>CEJoiMi=~SE>8%AGc=CW;1hH z>pZ>(W*v8>UUQvub=#>NurhO2{!G@I&F0U~=CYI4)0wm9v)1%!j?K(WS=rp=%=A<) zZ%v)Ho|`^zWoD_OsoaxRCih&BaW2afOmj+~$?U}RY$iWFJBQ)CMQC75_Y=Z-q2aim z-&bk~;nciBab?9vPJO;oqKVco!xeuUM|Q zQ?*jPx>9jGC;MW_S@FvCS_$A7edw9K6ZpCY0$W8LTCTaP9F??+sL!%xg>Ne8pW!-{ zMKO%*9K3V(*oAu4Ic7VToJt)Kc5G&~bfGA!GorrilsxxXaiv@+E-e-8dZ|&x%pYrb z=v?8_aY{J`@T`>z68ctF{Y0VQdZJKxiTXqXHq_?9n~!NcD3ZhQ%|fA8tU85)pDGlp zb-Pia=X8M@sK^wtBkX#65Q1h@5S91>zC0JEl2$pUYswNxrkVQZlQ&$YKuo!|1PsBug%HE#3+2)UEUsWIQugrUh6u(U=bBG)9mu>;Z`ABU zQ23;kNA+P%-i4xPm9RF*7L*c>*ATUnR>rc+=**!~y-~5P1;=u#E8gmuRbS)|VmB6# z1-C>9uDM>ZR&soUV`62`sk(vy%r}=D zuRvShHwZh9CH%pjk@)E_g8JeoX|4*~gnH3eF9i`?g-oM*dZ5eSMr8|JH0bY_35YBN z+RKKS(o^b?noJ!_rK%s!d5PEVc9&W&4h`Pr$7{MqN2 zG*If?xy;$KG8GgcleeCk&W+}+iP_BD872vH6SGq@`CCI>Z!fz_(0~dTomB~-cnBel zN6H|!T5kZ(YuF+6aI*wew1L)Z0JnA)#Z(ZA0oApw_H^XNVt$Njc72n6D2uO^>we<)4m~`qCFN97S>Vbq22+isGpo&K2n|H5Hf87Eza2qTEC23?hpciR?1M*s;Q;a zVT9s7DsLg^!fSCv(3kPrA|760oEE?l;W(|O0+-#hrhDoJo?H4JWkY8O0%qGVY|YlM zX=^e0d`(pqnP(X>tyu6Jw~cQb1aH8CeJz5ySaJU*m?IG2`TBDLbZv-XGzZS1B+3hf zWobXrNCUt*!ccRB;l~BXAt5d&h(^CfyB`G3V0bVH6hMiQ$u~Uk3S$Wg7>AKd+)Kd` z1VpQyi(-r((LL z$my#kRZJ6L0;JDn9MSzo@=&ia|G3tn`4cRcg2%sOSZ3XRwQM9l;w(3YojI050g#67&J``Y+@;x5!7?xrAOYndbTQ;*S>B`*5$-=}tH6Ix31 zP_V*X8({pgO1)I9xQ~yw3pJCoPPOhiN2g|d<4hgmM|@h8m&&zf>b&cSqnRb_$Yz@5 zftq)exc=rq_;%K*EqNDm*xJ;rp9mTi`{}JuaK(k9Z3|xq2jd&Go0ww_NQw9aWlT~r z$=(bUFF`CQE=cO<`u$xi*E{x+tp@pDQDv~pM+Q^+5EvBs&jh<-sspN_iz(zO;oJ40 zAxz-wnh0o5--eYN)0_XA`HhKUr2?n`Ofwrw(7xEHl}KU%|LN2iib8rUXp4015D6aH z7mFnASPi6>iU=-;{3KT)oRU+%L}-_gXV{QMLME22!n>6rcCqY$4MksSbUvXV=j9jYOqv9{|eFOhJHzcm1&BnP`nGqY0_xEc;b%Z z!2^uHsxGVG{qY;1Mt#|6As(nOX6sjVLXn`eI;0Zf{$s^2hiGb2QXZfe+JI3PXh%u` zZ`_*l!W}3x5wRc^C&~=!WG#rYvuITv*DWqNl3YPPklPyYTow*5Dv21*La}sl+ybyK zHY&s@qNRn#5_Z5+yIGex`FcajBc3B5f)p#`)^k7|QYLV)S0R8@>XK#($8A)=&U;dD zK&`F=o~!K1cI!Mo2&9(h6suO9WEYq4Ft*&Ka>=nO<%^E>46u`%Sy?%jcPh>jWT0SO zB~rtHB_@kTj!J>f>Lu16gn~~e>SF57nVc6_Dn&>f-SaH#-?YHQ#uE~In`F=>LSpeX z6$`Dts{Sk^E(WGIRbPi-Jf?}qkch+QgH8emVog)v<`L{;ilcS%tH#^M|ycH&v&C@C4Ous=jHK+$W%wclST zM9GB$s@BnE-=uzXZ+iB{@a8T;RjRn8*ERd%>RvVez7ZGmc+J0~9kEI}p281F<!v?rnJB;>|r(nvJY{qnhj=QEsps39>q>v)Wldz-_UKd z75kQAC*RPCmwh2jPa*wLPVd_?J&p7!PVe6`eE{jtar)qv=|e~_bNaR|(}$71%IVv; zOdmn|>zux0%k-T{|2C)Z;`H6EI7YeW4a@lC2!Wj$posvp&#c#QD30J0M@pUABgCPbsA*5 zJ9Et{3IU}l#v?PiUD&~ORwQ=71JoyAW~nJ6JWOhq>*X3@xGJzznMl&s4F^?*I3yEI zVU!)sOUj7eT|kWiPjwks*19a9HYYLih&B4AXTL%{=3|4PF6{9SCC#C7jlm+y;HMYr^-2MgOq^~LiW!+su_*8N$<>jiw}q)PlG6JZD|MiyuzXZM zM{@;bh(uxXa99B%9q4H(!yt2{1>m=ZvTZxH<^V^ud)S8_HGo+O9-AZFy*5-kHtJ~V zqt>B!)Uo#PxCl}p(8al+3*|F_MoePT{5Vw8pl47-C8J${NQQ4v0cXWG!)5VfTwFX) zlg%^UgI6QayU_w3AuM}4iW_}teK#aFtLxuEbp$FYLlrC3!lx0OCgQUK8O|R7)pgm7 zASF4XWZ{Qcb3)66rrlF$V;$^DLlN;M<#nQNiU%gf(>-I+fU&g}7iVDkGy=aK< zw7?^`)GuiuNyEF8HOv>#?)KtmLC_iJP2A_b^^w#HcA&dR8;{c4WKhqTV4u7>M6wC7*K&?vdMl&r3(k?KQR*Xep)Hz#M9dIMii8=sN9d<*F_@)iL!=O}v0iLLk8N0Dk;r)OJ94q^Rvt zKE_tWx-#9lNVtf~#1+jj-XqPCrl#I+9*k-zTms0`%4|hLGWmizpH{ zB1Vx%GmZyV8f*`h1{jYDEzfoyn--3(2Fw+x5Hvx(ZkmUKk7(dke3F8m;XHx3OEer} zm*Yx2uDxqULyEtz8S!^-F|0`Q&`i}A$dX12*6f4A5aKBYW_?zC7InX)KHu!~7;xDh z(o6yjE1cepKhIKaGhvDPWki*K7ZcFb$H&LVhQ)0t?CYR(&`uMP^bI1af|SO-K~U$% zSi?;e#{)@sYJ%3E)ej;|Pi4xgt=N8Y5)$=kY+%Pvpz%hvCXFvt3yTWSNX$~^B;F;C zXoHDwpol9WfhUo`!5`kdV9#c4AToAu%_n&V9i;Z^Q;+bPu}W)JGAIwT(8 zj0y~DP!%|qUB4`aVA3gsyNoa(ZF!jFSBgvEl>wNjPShJqjE-pwQbEiNgjH@ZY6nOs zY8)&pWd-LaR;?BA3LfbNp__2av>mu$$O*&X8Z<9)NZ6MWI4!~(MHE^>SEZng-fn>~ zqE1+guoOaDEAwocMk4yOo!%zhFA>^T)hjBdQyQjCkT|J_&~qhZh(0v)RP;Oey0q-C zARCl+5Yw)KzV9MUtoj(L!!Pu@>o7-Pud07t{i+J1gaX4URC~4>Q$Qj06IjT{z)`^T zs)3S@p_EpD?$(CU%hn+BoQTp+NV6|)s!;Ue(+$Z{Rb9Uyp; z5@^?mehOZvEJ@HHX}o?@7kelse(^F}22Cg_*wX<>kFy%2(P|?+KS6E>u$O-)#1h)e zBZg@Jw4qW<8>Xs#U}~^_KY;BAXN9OIZOj+-yd4$PDTw_7{{1R-^e@x1%ALRuNAVU* zM&;k2F0ctm8&ZuO%>%b?k_QqDO?;i2VO0LJ^u)Cf_SEqWYJ*lB)`7j^8+a&DC&Lf5 zPVc-UcjJLLy(`)k{fe;|V?~z&saY!52wj83f_?BjD8R6f zrxsa<30{RTIqf{u4caToZy?uo@>nEHBAaN`Bu)T^T_B|i8s-Yz{gF9*J2s54;jdRk{H~bHjSrXmyKmUax@T52V#r}321{mjU0O)u!X@w! zHQ>ZxPP(QvEM2(4MEo@EE!1sjIRxCloDjvI3>6Z2B25v^yEz$%c*FzPy8#)9cJP(h zVqEUw4@D0C6{@%aInc$OBsjhr5P^U=#1N}*z{i1fdobk%Bttyn!Mmm-FARd~24ZF2 zSdf=uHE{}}XJ=j*3qeM1Npw1hOvy`t0GS{_B@me;c>^(#N#w*jM$|!ApvI7%GQnz# zJ1PAHg0Yk&Gh{+ISPofFtz!2ZHl#9H4ZlS-?T|dtm3lDb_-lDx6meF zZ-mVR6UU?YjgWSY>@e;Xe+^k~NV~*eK`Lm}7a|%3uSZ6g(>XZ0WKQR^v+T%tCi67A zL{3_x?Y@talnKjIG+9!7yx>$;7~O*!bm~nZ!~wf@k&w$yV0~#DQrVV*y-Gybq;24Yc?~53D4xL?6VWu8A=H3-!Ozn1k1`#!?S4isV*$rHik<0@3m$F zQIGcQ+fg1mRK;Kj7F-BE9y!<5pmBz2EJCMJwb8Dgu=vd?r}U3kGwt%5)y!Dnqf{Nt zM6mpIC`5@SM5b9#wt!6ua!A$3rfKS$XCD4Xs4=RI=Ow7>Pl`lAQjtvNtw|Jds409| zV);HK$QzKTn61ErvpWANqwcAxTrFDs!DS^ z*JIi#FG-ruU5b}l?z8)@>d!#y@f;+zHRB5gyy5xxvWl|l<^I)Q3rcU3rM&?vd&`o8 ztN*^I9JhJ%a+g8IX*BT6e*)tnc@p!<>jFK@ooyaNbM>)*9F; zL;(?R!u=htGjM;0q-;ag6Br@d+)FbV6@fUsfimJ60{Ec7QJxDy@5Ev>OVG5N{NXd% z%w%?Ut}vB@rzpggeAXWXde3x3u)4`llh27l(8#(-BHuSubEEaKBV5~^G7k`O@ zGgRepaK^$eL|9i`q|~bj#^U}!H=dJx*-$S+^|x<97dj)O%KaVfoE-$Jtx#Y&e=qP7 z`=v`|0(s$SssXD`>N?a3CcT^dK5*TP!z&d3I?@w6^#NVJL%a6A`SMVL-s1Su+d$Ww z_CdnseSvQxoei0OlFm`E3c^oPB+rkZqDNS8U^*?4R1kh>8__$dPFxE? zTXk|Jb|ro#am8Fr1}cI-L?ImZ%-leo7%4Mvf9Dj>~i$ z@)?yZ!=Je#1gamw-rJICV$bkEL-A3E&3;)f)W!06#HNAs#F9blW)l!-RNCOTxnJ!x zn&Qg>CKO&+T3{%g>s@?Yhe~n_EpsqZT78(ZrM{YdqafWZIHdSMrAtoeM=J+{=*T4lSP@1$huS zw4;16(qY;-B6cC%w%l9m5H!@cFpDzm1+YcK zln6>Lq06TsTx&EM|L75{7UarywZRBOZF$>2IvjJ?qTbR%it5O16AcFCK8~b7Ji~Uv zriT$kHVslnD_4vVr(Q`erf#m(8b)cViY12KK+ms$AA^kp_KnxT!Q0?@$yt*XR^+P= z4|h9WR<3FIg@^jeZQ$Raq#}+rNKM))SV#KUI+6_H((5RJJn6@X%LeC;BN|X}fx9z2 zK4^v`X0eR~-s8=Yb-2a#*ipz%0&)O8u}|7jz!Q&vCpK3vfG*JI*WqTIg#=d{6V3M{hbTC%o{(vUwSnfaw;L^~tpTh+KS>N|bNdKZL&lKo@B(yVvW6Fa_`(moJ@(PEVvl%xzpmOl zyxS1&TsIN!T8|^#y`Dh0C!7sD?+s@I&sb}$I;6dB7`Ub(ZC@*eaDS^0;q9$7!aG_9 zLQAf1E4Ev~+nsWiTXB9nAZxm&BIPd35!HM5HFe#PGlf{3z_v%>5J@SA80z*(FBqha z;WPzgRY(e-vz40e?QcOfiul7V8qdA0D-Lelc}=m8 zT-5>_9p;ZDP#bQOsO!t&D$}!EXSivSL*aPN-)x{I?#ZPuSr%U*$5c>kL>TvPJaoe*qXEM3-=dw6rR+u=GnVk!~4}7&CnP%Fa zl6Ol!20wh5X2j#a8nSKL%e$54Q-rqytyuww30VW%)?Y?ix^`4xe``lXHNQoE2Y#F} zp)CchzGsh~tuGy&xBv^1Wc1)C11GR~dCq+7W&XF;h0PqOx1@DhxPv z+t(YS;>TdM_Q*Tf^>qk>F!rxB@;D|fPEjilgiZ0WqAo)*v8Kd=&*AIdjez|w1|q); zEF|eTq_q0&@W6&Hjw7ph5CKLKJf7R;Z<2sZC&AIre}GQ-W_Z|H$`S{puS5($3b(>Z8Wn@X9y)p&@~U9# z$-*N~p5s^-Ce}|AKOqg%SeElBsD&pnmH(fmSoc6NGJimtCymVZG3 z39+I@L5_kiQ2VkG=m6TXbR;s9zMan;7XV!*AJL(w zG(Ze~y*RLgH}Nc0I~eW)+LBmRe<)HfgfRnQT>eQn zJwh@+#Dn(oKI3+komt?(Lc%!2aeV|Xm$$3t%bG%tmzo;Co=T+Q>SEx{PjGwLdnJ z+E0v>_D4pa_J>AV`-619_I7$e``@WS?f11IQVyW~_q89#cMKTu_q6{pa+~(D_TPV!YC)eL-@ z%F6Z#7;P&PtUc^G21$Zj3r;acWh2`V)r!n;JL1lu25A4gCAes7Y?k5d1@e!CafBtN zLoAJ8W0PxYuapnFQHRt&2k#>sG#VX~qQ=Kz=xN@!)68DsnceJ)MEVnE(@DSc%9gH3 zp1RBq%U%pVPGE$^WF<34&> z4cSOKHGLx$t~l?-VP8&w89opDvI&M>+Lz;-)VIG$qtuw$0J#zTnv`T`07?E}03RSM z5D0PBe-N?aM-(vpwVj-pYW%MJVzc6Ij#Zf`2u$_~4A7DL0VJUl`IJ?IY|K$MCSczKgW7TSgb zxLpIMMyp^Fag>|3&H;H!j;*)hDcce(uAbb=II9PrP6vA}QaRC950FNrEutnzLpB6$ zk1~H5Vw}r$aS;yEu=ZA9?X8guBMa^5ApYYKo?fh_fCjW7+eHIt)2Ur5@MGDxOnPN%Gi8p+YCzNZV6{^M)KTt&?0NUK#EU zsb4x)h+V(Ra1E2=!Hn_Aq(J>dOdgGq&F;{ONoVrj+bbdrEFmVsxqsrQ5L#hisQ?v{ zpTmt%p$G41$bCSF%LbuA9HQeB^L8BGmiz4le5OshTExAprHdnUs$N|%cd*oV!OD7;Vrg~AIKe82I`7wqV@YDdI{OK{3 zZBt`?5+(i!FA7YP0a)+JHm4oQ!^HwsfT5OTt9)2Du4qVlz}(QLL0fsqk5r2dy_(* zIGvCm>q4wu%FO?dk^uMsnVWKFlt^faxieynKMYE0X>f6Xdd!P2Cv3wrmy_VqbUNlW zw-tZJuHv1peW!xd)$f3MNAoIKU;`8Gtpj#ll3Lj4M6E0j@;Xv*8b(qol&OhmuRU*wy_#wG~ z-qC>KehPu7*y=h6(d%(I*=TU*>BE6W18zI?ybaG}BM2PVrKJGZM)GmRHM3=cWW|cd zz^>keD+EsgCo<~rQSLC=%7BM@d4hnunLGdFlW=MP_`tFGOg4u*wK5Y=(fx?$XRLE{ zMQ zb~F;YG?LrHZ1B7UmCsoi&;9qGyzf41<{XbX`|Ozvxe&6GAx~J4<)KHBJ;5u3A1u;YVHg=oyqgmL{$ctSmv_N<}LK=2w#UrZ62pKGv_`mDTp|CrZzXTkldBc=vY@n z?D9jE|LHcvEJ>lf*gl1S(weHtJ8u{UXc`-}KuwCv4~_&^;m|>W72zymj9dq!7R+U^ zk~9e`I2(pV;p1tLwIW!NcjRQ0Le~7AGbHazJ==gjg($Zpja2zJo zAwj22ch%Zb^x^EZ27x~^&>n6&f;0zSjZnYTK2g(wC4|qkk^LQ7=BT%TPV^62GZwn! z@f$LbwA;e?!|u0Tnq}iQ`KDk7OAuiJiKOEqc%>_akhS+VA}-Tzn7YV$I_HPF=Qh|~ zg-$ZV=@}Q%MDz3^q&Pm0mfNInNP!@cTFX7elqooS$A_jsg?fB{Zglj5H2m~s2I~i) zoi}E%mSJ}-!*;WtST}LSP;50cfQLK-7X)fH?AovyL&9S_HM|lxaE^W>Y4=@I>0%PR6XO;4=qx_&a@**ql(mF6 z;0?0<5u|l_sFkpF^ecTeP_nJT*@>)}!d+Kf7q;KwkvH@u^6Hf} z4zJ@HrXJgRs#m7PL6hIIcfb<8x0Uj4^Y(fB7wNWxf7(jfxKMGeua@K#x)0#{t<>`E z%Xh5xwffeHo&Bv~jIb%U$0!wsft36^QQ6;Vr8X?gSgMr*%dx9hN`|QV1FX0omza?) zcaV1p0}B}Z6=^lZb?4G_$mw~ynz&XkxGT8TsTpf{i$_nu+BM7dz;;>|tIgdr&^2M# z5n-Y0Ws5eBI9Z3XV{D`3ng;DG$m7&%vp+-$e1@R8FWjI3-LjT|%4&3p zH7JPqmuUE{5AYegQ?lAQL`W|CKhYB%NWu+8=rCXMg6q?Y?uANu!H?t8?c)zU?8h4` zWNpK37P#~l?yPaUyhIm7gN=8chadEl3lBdSC4prGW%vn`k5CR@Z6-E8q?67HAJz3>rQZcqe-o^kuOQ1j7-$&ZU!8hU={3Np!V2k zw?~-C0uA3!Q3&Rqa;4-r=~=M4qCddB zIAaezo)j#o$+4tAyqV;%GS(mAGCk;BBnfg@Q%u*Ju@A7orTSPD3c6Jw{{i#qo<%bx zr|i@%+&(e@6Be1WaQUsi6RHp?xtQ0}>|JWA`;cN>H?PyNBf~IoHxzlbQqD-*JkG#e zAXsaQEFVUSdD9dJsgrDa8l@*5%vR&*R-*in#vv8ax2wh{oBMCt18gQ87i5gV@r2g1 zJvrCtm0RO>xi8#}wwC%3^@_FBsdgLRqvnYoqz;g+aIa>Z5{Hqo>v~P(iHuBd(=w2( z5}wo!tOMu~HyM zGRcsIE@jN^5eE=aYS8+U1Has80D-3|?u9Gy)pa~-bZ5&Ja4kSR;7aj?>&j@x915p1glDP(K$1|~67{xA+_2MyBSz`cO3y2r)O0}~;%$pZ{Tnn)KYjc-3f zUO{W&@0d6{MXnYTXQ$_~(fI}T`*^cc4|J7EkRt`IGfne$1Jn~tw%-WyB}VSXzK5{z z?!zcb*jLl&@_`VG#?>Lb?S`|#2=qoU?*r!=`T(5+h!0WB%i8tkV-gz_qWu%P*nC8+ ztBD5*tC7XK$9?}G^-$u~VbwU$y!Rut$fO1*QMg)epp8WzrluH+?!Hlt`>1Jrm4TEo zDo&ul4Y8@X7pXAWkKNqjCtHe;B(N*&Ky>?^Wy_13qWiw0D+4|R0%c^~Mxt#&_G2JW zsnThml!Sf<$1g2XyFmRjZOxsZfv#tcn~aWF&H>xwFLXe^G?kmn<2=uib^)kmU@|#p zP2i7!lqMM@b`6=j|z#T2Q3qI$MOuKy3BZ$X!`jrH7^LVy^dXyShVEKv07LVz4c zdyyk4*BL=Sd&>S{!{$y3&f6y)f z7rk+k#at$l(pEAQm7-L5Zb+}i9bM6(!iZ%ye|MCI?BptiO4(w^SC$F(hxv~*s(myG zhw15E8c&Q)l(_PeEXZ6qs_u%HDD`hqK#Y@s3y>oAQgEK){s|R}2csWHwBKKJOJz73 zLTQGZ*jVm_e}MRBl#v`L*+2I`Q`!GQ!FMUxM!_fr_fc??0+tS^=;}t z6tGPEC3^Y>1!U3ZvuNyp#y%`;ab=T?**H zByK|)IhfY*58?;72oCvHa7c6X$4v0t4)K=mqxYfP&9r&fz;5jha*ZGqc*^(B(1puJ&a6S$Rnt2;bzd@e>UWj~Ys9aCp?<@TkGzF%E~v pI2<0eWa=l%OXhy_ka<+Qzq6$A$5A;hlX@prOFfc$Fs>Qb|1YB%F8BZd literal 0 HcmV?d00001 diff --git a/api/__pycache__/codes.cpython-310.pyc b/api/__pycache__/codes.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9189fd026c870b3512952a91ab671ded30ea7ad1 GIT binary patch literal 3712 zcmaJ^Npl<55ynOm1Oe`wC{Y?}vuII-L`jxS%eIEV12EBG2ALU<;=4>Cd_xf`fS?C} zVyaS=gPnUeXK}VePC2A1{~?#`&Uuc>iP!9LlB#_D1_Vhf4#@dtdU^eNx*Po(txP5r z!DsoS8&6w{k;os3IQ$gSia7DSZ*4a~R@1V1~qO9FsyBDgP8=qr2N83EJrlmA%_5BVmrPgRD?M|)J zZ7Vx1;k6a;TKk@I(5UTrtDy~x6_~}09J5btY-H7*L*|yDn`-_g;SnMfS&jt>w%=_A zgWdi5yIpUGn5REc5RY)RBY5&6Jc^*kM3l!xj3-2#4~PUG6a#!n4DzHH;=>}zQ(~Bp zh!jtY5uOogJ}NSNOpNkzF~%pvIG+>~d`e96X)(oT#5A83Gki|W@_8}G&xv`yAkOjg zVu4=}=lPwI0j!QT|?{GoW0za<{>x5Zoh9q~2_;+1Y$CQBpNYBpQ@jYFl|_5|f> zU(|e|lpLpWCUdKyc5Pqzf~T~-{f<)G?RdTdhFZ~-9A&!RRu25uj@O2%P~Y-)q0?!t zD|^_ty`Ig|eN}kRvte0CtA>f9Q=F1!E4pSo3hrPj>`6(j+EiUjQte0}kw+fEF2PfR zUl9C~;8z5{Cio4(ZwY=!@G-&f2?ByY5Uijaj&2dG5@Z4O+XQzA)(Gwr+#~oZ!Pf}B zPVfzaZxVb+@GXLG6MTo@y96H*e2?J!1V147A;FIbeoXLFf}a!oiQvx!e}3B*a8^KkAkG@?$?^0>jo*;ZMH6Dliq)~%z+01JN4F-o{mg4bbJ4lBC>QR%& z%gL#{W}zPjll@&zcq1EX9@OWn%yxow)#$~Ald~0dy9}447#6Tw!L&BCe4ZJ>Fu74M zt42PUEHm`+yh|Ka*Uc?xj!HsCT;0%-J?W#2`iZ%*-oCD*m6=&}WVQ>s23HeB6LA~U znNf5~Co5ovAw_x3(U8M}s_EEk0)A32b{*4nb=4{|yojZRq>0Lytr~W<5@yCNBO0pf zY*(OFQH#uFPb%`|CTKa5Kmn8Cu!BX!V-uUuE8t}@|#)|2+*wLtyQg-x5nK?eKp8O&{>bX2&b>bB_`tZ1Tu9aP-3hG-NDT24A^JE$>4 zETfxd#mzGZg#u^hH&i3PrR5QPT`Ox2qLgDS56RdT+cB$L4TL6F zk{O>legp7P;ud8H%NGphY?;<$3L*j*CcYPv#bX&#Y`~AR;-5tPC+Lw9J}G4PT1_vD zj_oyC2cDm;9PaGZ{AR^(J@a-t?QHF!?$(|@r8d%S;#|mfJM~7}eZEGLtUUEs4-WkV zEZ)S`{t_S=NyHP$L{$F8d-sq!#+SJ-bB=XCeg8k2$*Av3#Iuy7Mj#vNQ)$SO{dF10hCz)=%`s4^J~a+%1k{9- zO=4ZTzitY$45(=-n*lZ2*PI141}d~U2WlMDywp4gY64W~=K`q7{<`y^rux(cP}6;C z5!4K*i_*$PP_up6B~Wvq6e(L0^Oz^j1tYec?!t-UV|h$FEgBKzIDsk9^Y<|t94cR| zclI#+b@n_=f?&VbQChpoVXNz7n(&|3F%BxZZoAWJ)?av})&|lf+1c4Us}^noJG(6( zVoV+D%8lMBg!3h@Fsfx{X*nDzdA1@a8~L~_4YBuX&4Y#q6OP~YFz3>E zdpndlM<|9!!6!Vzp&B`&GVzBAXpZ+s2+{=81TzG)1apA80`2+|!DWIg1Xl^J5qv=K z6@amP*`KFH=Li-E&J)l###aEr!1G$8>jg3Mv9A$581`fTIuZR7^Y|d{Gf1myv0zp= zi$(NFw2q!mp{ncK{(V}Wh7f{Yherzy8?8-eJzAY?xc)uTT%r>T9hBlrBD64UfV0=g znbch!7R;R-YG=24Kn6zSQPn%QdcR3D{+Y(6kBx;Iy?55h(lqr$c}|p7O0%Kj=R=UG z8jlSU^!h%2%c!Tln=GLdqJ^L4WePaiY`3h*(Z(;4^*Omv#^GdZ1kz_DowCr}Kh>*G z`epdE4Mm>=zKpBA0-%mJIy4%UfAP<_lQcdVl`DR<-)MPYf&qn_Xdv3i9 literal 0 HcmV?d00001 diff --git a/api/__pycache__/log.cpython-310.pyc b/api/__pycache__/log.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39955f26784f004b4afb77e125f78c50b7f8bd8a GIT binary patch literal 1746 zcmZWpOK;mo5awId%aUKQl_vwVs0FxDY7{6?7)2CEZrs|EVaaa63qfetA{~;;E>}U7 zt9$JqXi*?J_TFE!*PaUW7v$2;EM3(_N${KbW@d-WnOU-Cv!=n-{b^^mxv6QtBe{K9 zK%T)<{02ZYUn4p(65ZF8G<-uz(>FnyiIv*EojSgwW1N*#QrCCWs$WfOel4y0b!9h_ zhTot~YP?-}r(NrQi<)oS?=*iEZ2H?ZV!zYDx=vTGHh$KK19B5&d;(yLRpikPcYsl*Ju%IW5&_vpYF-=Id^5vT_^?%EM&x#_)FD>)k14xP6&Gp21Tb>zYP2=)L9}=NdJAi`vv7`o2ahzN>Tt z^r~M|y1B0r%db=Dbw`6ZTk(}8cNA{{x0SyFYcZdDBdf~aUixc_uP%9A@wFvyz`FIT zb;!^JZ3E=WjclpN&1Ga;@jFYts`%EDuPMF_yxjS^;+-YmQ2g$aZz_Io$?qtBf62EL ze*pZU`qFLWW3Ahfb!huBo0WZXBUXqkOsOeN=|75`NZzD}O6S5TqVU{u29lWA|l@$yMMq^&b7+0}_({tVc%9{u_ z0{)Ts3PKBE4PhN&13Z(j2jMQlJ%sxR4-g(AJVNjQq#0-8SB>wY z0pb0NUc^$`BlI;*u-?7X#WV_edRlg(=t1R!a5f7Fo6b`>&%L>blOlM%hn^lZ;z_>X zRY-LLPw^C>rZudF-q71dTW_1L?&=`nu9do3GKHXT=xQ|>UP=!IA(xPBoR?=`9#Tn&0 zm=FuWA7hAaM;51!`}& OpBaXsJAXAS>;2y;FtZT= literal 0 HcmV?d00001 diff --git a/api/__pycache__/metrics_prometheus.cpython-310.pyc b/api/__pycache__/metrics_prometheus.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..724908a216d73a378997428f0066c51893a99d87 GIT binary patch literal 5163 zcmb7I+jARN8Q+^$D_OoKah$ZkCbXrZMNUKLv}p)&94B#`$i`O4pc!Uov*$!wN7Ab2 ztejZmGG&}c!VIr4ov}N->zY@$k+M4CC)K82wxfuAxZ7Gz@0c4Q4W{W!6n@Zdv;_vky$u81xvgB@=r&i+ zq0P12cD|l(dv&i}s2AGBda*rGpJS0!)Lcnve_#JKgH(IxAy0= znKQ2SZ>3M1}66iM5E z+~k7$l5cYXgxWk6aU}f^b8Q~Vp5V;y$+*+-G1~Fd1S9T8Nhjq`QokK`!Uh+zl7Zb= zxw*2jQe9rz9Kp4^xHNdf+WqRCwWS;1xL94Qjh(viUd699xqtqAvG^59Syh>pzE!D| zU6o%>dVnufezVtZhhj(NZiT%DSF_7WtHq-<5gWV_%M>f6cCVGjU6DjwN;Tc!9lSH; zK?{{k)#QT_Z4dE!1WTO+GNtbJE-%wQic< z_jZjfm!v-}kaj^s?Jsd=%Vo|p2YZU?1mT1E{MdXcon#&>yfQz9d!W~>)EK>Gni;@+ zvMLI|NsuiP6R66Ln5O;=RkNs+Cx+aAy_u%nK=4m{V7BIYC)x=lhx{_xf8!2>029RE z+qK%dKiDY!xC4#Qn=0Uu3$17-OnJP+x=Gwg)r7uRf?YkSPK*MPnf&xm)<}%TudS$s z@vW^xdF#Iq^^BT4dgR9s{jlRNt*`n+aDFTYGDdO)g%ok4nfjgNaYf7%1!mNl;E0_; zFeZHCOJkDg$|QQk@tg)dP^MZIPmNtQoE9VzYJ6=-W|7xxcS+OMGfg{GM3JMlaU3#* z3nJ2DLDHtCkFiLQl&ce36%W~`_p3QgwBdd|S9X*mdF!Dl;gp!nE(~MI1O6m}X@p8W zH^lca|Ml8pGimch26NX+x?C)-??kY-ZJ_cy3n9ruYuV&rR_r8vWR;dYgtPPG2588|R{#+Y zG$0~tN}x<;J|pX4RRM?YEL*}uqd|*zs7F*D0Zgk0C>n6d?4KQZ;W1kr5pKheV3Xu% z_&3p^hwhU#fEC)ZQJe$&Giw(ZJ#xhSuC<@rHMRc*IrOX{&SBO)n$_0BstTfCIWNwk zQLZ+$Qk=y=yhhcxY1)oEk+K<&%C;t_%58_O9#?rSZd@FDr7G$9fg||?Zje)`T#ML> z-^=;jS+jq7E@ zAY;HqER0~Kxu7{ehlMVRq>g7;4vAs$h4qdWQCrY-gfjLK^{=7G3#f)ZhQY5CEc|TP zX80HQ8HeIA_>>%3aqVLe{gfx*Th>%%KkbDgeb;}`OM$5`n@O+5e6n8t<2Y?bvxsRAc$0kK)?IA|lr6|I#enobmKfOGVzReV@<_yB?q%nnT zBp{iKLZCTLPWXM12C39h9>tY`J_09qh<^}7-Ci))`tvn-EF#nBoTcm~WQl)j^L+TMN#d%aJ zpFGe(lDI}wOH^H_>Iy1Eif^Jp9QPy&(w#0>1?pwCy^spzL=VE~(J_%#1tRVf^6~$` zX-QzL&pm72@dgIodg+ez=g^a(Chr zs*ZH$$e@$h{}Dw}yz3c`1(QrB{a@bOrAa}n}JHAN^lZPulJ-9tAmS zVSQum?n>?U%Dv6t?!7y;Rg9N|8><`r!p6!ie|@QT`!X`IENfNn^=8s-#Rv=zgRuxG z9DbyA6u)bU+(w5aGSz7XGaFD;W>JbkiA)0$`FrCFi{+l#myA{XrEuH$-TY@p>g?yR z$2x$tWwWMnAFB?2d-iaKjOBZSQ}WVd$Tt0cj-mqrx2o)0E4BVvvYztt;$nNJ!r*k` zP9;j(I^ffhf|gH&0atvPKjSIs$d>ZRq(+m(Nc`#ACnEh&7~w5*|Xq}|M7 z_EXT6IeX>}q=BHCMPbu#1{6W62CXD5MeAm5ij)p1TmG53pFhBv0j0cZ|MX^tr~VsY zkORp|r7{*WT0p=?*+kJ#Xuu}?BKH~iyJ!5?`V^soem{hgUQ)J{Rk@(d;5bo=S6WEj z!G!MB4~ZId@0`(J(7d??~~3T%|x(xrp}Q3Bjyf8p=oVI93jUf1mK7(2n$f&YUql)3X*#5fH4` z#2vcGmOD~8T>w!o+#(p8x#erLID4d5LDY(2-gTy5{QyM{4y1%yZ$;w8s zymqfz)4^IMcw|77l}P<2u|cCe?`+2+>10pNhYutnkqIslTw|$h(d&%7?a;0$?@AVP zUL_6HiHu`-ZMubRr?KBJvUgLJCn7$KDV_VjJ5DXfC79blW@5yy%{> zS<5*kCnO{!Ziqv2e{NWs(QV@?E@Rar<*Brj^RE6QApmp9i)LbFUR zwqCbJw?|u}^_9`~nNZe;%hM1yzFxn(vAp^p)!MJ2yiGM)`=WiJ;K|eir!in)fw7cd8G74_WuA z5nW*RwlV0b{?-&so$1j=TWFGF-ek8FWLzHp3-oc>WQ;uIzVr;HF{8HJXH7OfHp_3blfbkUCS&-Wqm5&Hu{Z&InO{gSbCZMrt&JRoRsD%=} zT@xGzp}0nNU<}Hpt{QF5zpHpsb12GW>Wmhw_4ewnKoDQxq0IQDNd~U++Tth&t0V9g z5?$1Lfc*n5)`w)Gk z&Y=feIg)3)s(&)YF-wzL&7HERE%2)$-h~@4*M{RV=fi2EaWUMSBx5S_O;PSc+H#1U zLihG!RwhjjPr|UN(@e$(OQ;#1I9{wKDu9lTzeSX{z_Xl(V=f5Ii>3n~@2~k_-h9XQ z?V~qen~KHLTZ4J!$1x@x$103t7?UPLx)-C{Y^v$R7)u-r>{B%p$FwL)m;-2K2~rI- zzUs<`w@tV}wRmA`Pch%kN?I!$Cg~M+qJw%3>LsKQoihhP)`vq+2IdwTj^(@a=B$Yu zKRruZU=}$|i#Q&bZACr$FuA()(fZ)Lcn@wWznU;w)M+BH?#intIO0M_h@tDK)!Yfv zi-7bZc9GOMw~=Bj;xb;pumyhz@!hq1Ydi7io0ZXMb+oFw`6NCmj!sg!mB57mFFEC3 hZvnr7T`Vuc;TXM3kefaGw*Q%BK@I=*>{-kC^B)X+->v`v literal 0 HcmV?d00001 diff --git a/api/async_quart.py b/api/async_quart.py new file mode 100644 index 0000000..e0f2973 --- /dev/null +++ b/api/async_quart.py @@ -0,0 +1,1161 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 2nd Aug., 2024 + + OBJECTIVE: + + To provide an easy way to perform repetitive tasks in quart. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + 01. PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING ANY OF THESE + DECORATORS. + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To work with Quart: +from quart import request, current_app, g + +# To make decorators: +from functools import wraps + +# My utils: +from utils_v2.string import json +from utils_v2.date_time import date_time +from utils_v2.security import sanitizers +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.log import APILogModel +from utils_v2.api.response import ResponseModel +from utils_v2.logging.context import AsyncLoggerContext, AsyncMongoLogger + +# To work with date and time: +import time +import datetime + +# System-level activities: +import io +import os + +# To work with datatypes: +from types import NoneType +import pandas as pd + +# For Pydantic data-behaviour_models: +import pydantic + +# For hashing and shortening the hash: +import hashlib +import base64 + +# To make things human-readable: +import humanize + +# For debugging: +import traceback +import random +import string + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Chars to choose from for random strings: +ALPHANUMERIC_CHARS = string.ascii_letters + string.digits + +# To capture system information: +PROCESS_ID = os.getppid() +PARENT_PROCESS_ID = os.getppid() + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class AuthDetailsIncompleteException(Exception): + def __str__(self): + return "incomplete auth details" + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +async def data_from_request(inbound_request): + + """ + Adaptively extract the params from incoming request in whichever way it was provided. + :param inbound_request: The request that came in. + :return: A dict (could be empty) of the data/params that came in with the request. + """ + + inbound_data = {} + + # Extract data from the params in the URL: + from_args = inbound_request.args.to_dict() + if isinstance(from_args, dict): + for k, v in from_args.items(): + inbound_data[k] = v + + # Extract data from the raw JSON data: + from_json = await request.get_json() + if isinstance(from_json, dict): + for k, v in from_json.items(): + inbound_data[k] = v + + # Extract inputs from the form body: + from_form = await inbound_request.form + from_form = from_form.to_dict() + if isinstance(from_form, dict): + for k, v in from_form.items(): + inbound_data[k] = v + + return inbound_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def file_from_request(inbound_request, file_key): + + """ + Extracts ONE file from the incoming request's form-data. + WARNING: NOT RECOMMENDED FOR LARGE FILES. STRICTLY USE FOR SMALL FILES THAT WON'T CRASH THE SCRIPT. + :param inbound_request: The incoming request. + :param file_key: The key of the file that you want to extract. + :return: A tuple of the file's name and data. + """ + + file_name = None + file_data = None + files = await inbound_request.files + + if file_key in files: + file_name = files[file_key].filename + file_data = io.BytesIO(files[file_key].read()) + + return file_name, file_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def files_from_request(inbound_request: request): + + """ + Extracts ALL files from the incoming request's form-data. + WARNING: NOT RECOMMENDED FOR LARGE FILES. STRICTLY USE FOR SMALL FILES THAT WON'T CRASH THE SCRIPT. + :param inbound_request: The incoming request. + :return: A dict describing the file's name, data, and size. + """ + + files = await inbound_request.files + + inbound_files = {} + for file_key in files: + file_data = io.BytesIO(files[file_key].read()) + file_size = file_data.seek(0, 2) + file_data.seek(0) + inbound_files[file_key] = { + "name": files[file_key].filename, + "data": file_data, + "size": file_size, + "type": files[file_key].content_type + } + + return inbound_files + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def headers_from_request( + inbound_request, + mandatory_keys: list = None +): + + """ + Extract custom headers and some extra info. from the incoming request. + Raises an exception if any mandatory key is missing. + IMPORTANT: CUSTOMIZE THIS FOR THE NEEDS OF YOUR PROJECT. + :param inbound_request: The request that came in. + :param mandatory_keys: The keys that you need to have in the auth. + :return: The extracted auth details. + """ + + # Start by extracting whatever complies with the format of "X-{Header-Name}": + head_json = {k: v for k, v in request.headers.items() if k.startswith("X-")} + + # Now note down things that you want to keep from incoming requests: + head_json["Remote-IP"] = inbound_request.remote_addr + head_json["Host"] = inbound_request.headers.get("Host") + head_json["Origin"] = inbound_request.headers.get("Origin") + head_json["User-Agent"] = inbound_request.headers.get("User-Agent") + head_json["Content-Type"] = inbound_request.headers.get("Content-Type") + head_json["Content-Length"] = inbound_request.headers.get("Content-Length") + + # Raise an exception if any of the mandatory auth details were missing: + if mandatory_keys is not None: + available_keys = head_json.keys() + for mandatory_key in mandatory_keys: + if mandatory_key not in available_keys: raise AuthDetailsIncompleteException + + # Done here: + return head_json + + +# --------------------------------------------------------------------------------------------------------------------- + + +def cause_exception(): + + """ + Call this from any function when you want to raise an exception. + Example use case would be when receiving data from an API call and that field is not supposed to be null. + :return: None. + """ + + return 100/0 + + +# --------------------------------------------------------------------------------------------------------------------- + + +def describe_exception(exc): + + """ + Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire + traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it + to the dev team over some service like WhatsApp/Telegram. + :param exc: The exception that occurred. + :return: The dict that explains the exception. + """ + + exc_desc = { + "type": type(exc).__name__, + "msg": str(exc), + "tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)] + } + + return exc_desc + + +# --------------------------------------------------------------------------------------------------------------------- + + +def messages_from_pydantic_exception(exception, as_str = True, sep = ", "): + + """ + Creates a list of readable error messages from Pydantic's validation failure. + :param exception: Pydantic's ValidationError + :param as_str: Set to True to receive all messages as one string, False to receive an array of strings. + :param sep: The separator to use when joining multiple messages as one string. + :return: A list of messages of all the things that went wrong. + """ + + # Make a variable to hold all individual messages: + messages = [] + + # Interpret all the problems: + for error in exception.errors(): + loc = " --> ".join([str(item) for item in error["loc"]]) + msg = error.get("msg") + if error["type"] == "missing": messages.append(f"missing input: '{loc}'") + elif error["type"] == "model_type": messages.append(f"invalid input: '{loc}'") + elif error["type"] == "bool_parsing": messages.append(f"invalid bool: '{loc}'") + elif error["type"] == "string_type": messages.append(f"invalid string: '{loc}'") + elif error["type"] == "float_parsing": messages.append(f"invalid float: '{loc}'") + elif error["type"] == "int_parsing": messages.append(f"invalid integer: '{loc}'") + elif error["type"] == "extra_forbidden": messages.append(f"extra input: '{loc}'") + elif error["type"] == "value_error": messages.append(f"validation failed: '{loc}' ({msg})") + else: messages.append(f"invalid value: '{loc}'") + + # Return a response as per the preference of the user: + if as_str: return sep.join(messages) + else: return messages + + +# --------------------------------------------------------------------------------------------------------------------- + + +def summarize_variable( + value, + str_limit = 100, + expand: bool | int = False, + sensitive_keys: list[str] = None +): + + """ + To summarize an input value to capture the essence without hoarding to much data. + :param value: Anything that you want to summarize. + :param str_limit: The max. no. of chars of a string to retain. + :param expand: Set to True for full expansion, False for no expansion, and an integer for a specific level of + expansion. Applicable on iterables and dicts. The smaller this number, the more concise the summary will be, + and vice versa. + :param sensitive_keys: The list of keys (of a dict) to obscure when summarizing. + :return: The summarized version of the input. + """ + + # Check the sensitive keys: + if sensitive_keys is None: sensitive_keys = [] + + # Handle datatypes that you don't want to modify: + if isinstance(value, (int, float, bool, NoneType)): pass + + # When the value is a list or similar iterable: + elif isinstance(value, (list, tuple, set)): + if expand: + if not isinstance(expand, bool): expand -= 1 + value = [AsyncLoggerContext.summarize( + v, + expand = expand, + sensitive_keys = sensitive_keys + ) for v in value] + else: value = f"array of {len(value)} item(s)" + + # If the value is a dict: + elif isinstance(value, dict): + if expand: + if not isinstance(expand, bool): expand -= 1 + value = { + k: AsyncLoggerContext.summarize( + v, + expand = expand, + sensitive_keys = sensitive_keys + ) if k not in sensitive_keys else "********" + for k, v in value.items() + } + else: value = f"object of {len(value.keys())} field(s) [{', '.join(value.keys())}]" + + # When a dataframe is passed: + elif isinstance(value, pd.DataFrame): + cols = value.columns.to_list() + value = f"table with {len(cols)} col(s) [{', '.join(cols)}] and {len(value)} row(s)" + str_limit = 999 + + # If the input is some form of non-standard object: + else: value = str(value) + + # Handle strings: + if isinstance(value, str): + if len(value) > str_limit: value = value[:str_limit] + "..." + + # Done here: + return value + + +# --------------------------------------------------------------------------------------------------------------------- + + +def set_api_version(api_version): + + """ + Use this decorator to automatically note down the API version no. and propagate it throughout the downstream + decorators. Use this as the entry point if possible. + :param api_version: The version code to assign to the API. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # set the version information in the variable. + # This makes it available to the downstream decorators too! + kwargs["api_version"] = api_version + + # we are ready to call the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + if isinstance(response, ResponseModel): response.api_version = api_version + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def read_input( + read_headers = True, + read_data = True, + read_files = True, + sanitize_headers = True, + sanitize_data = True +): + + """ + Use this decorator to read the inputs from the incoming request and sanitize them. Sanitization makes the inputs + safe against certain threats like injections attacks. If you expect to take in inputs that you want to use to run + database commands, you could disable them manually. + PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING THIS DECORATOR. + :param read_headers: Whether, or not, to read the headers of the request. + :param read_data: Whether, or not, to read the data of the request. + :param read_files: Whether, or not, to read the files of the request. + :param sanitize_headers: Whether, or not, you would like to sanitize the params coming in through the headers. + :param sanitize_data: Whether, or not, you would like to sanitize the params coming in through the body or query. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Get the headers: + if read_headers: + kwargs["inbound_headers"] = await headers_from_request(request) + if sanitize_headers: kwargs["inbound_headers"] = sanitizers.for_mongo(kwargs["inbound_headers"]) + # else: kwargs["inbound_headers"] = None + + # Get the data: + if read_data: + kwargs["inbound_data"] = await data_from_request(request) + if sanitize_data: kwargs["inbound_data"] = sanitizers.for_mongo(kwargs["inbound_data"]) + # else: kwargs["inbound_data"] = None + + # Get small files from the request: + if read_files: kwargs["inbound_files"] = await files_from_request(request) + # else: kwargs["inbound_files"] = None + + # # We also make a provision for capturing an identifier + # # for the logs that we make through a sister decorator: + # kwargs["log_id"] = "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(8)) + # kwargs["log_chain"] = kwargs.get("inbound_headers", {}).get("X-Log-Chain") or kwargs["log_id"] + + # Now that we have unpacked the incoming data, + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def validate_input( + mandatory_header_keys = None, + mandatory_data_keys = None, + mandatory_file_keys = None, + header_validator = None, + data_validator = None +): + + """ + USE THIS ONLY AFTER YOU HAVE USED 'read_input'. This decorator will help you run validation on the inputs that were + extracted from the request. The mandatory keys will be checked first and the validations will be run after that. If + your validator already checks for keys, you may skip mentioning mandatory keys. DO NOTE THAT YOUR VALIDATOR + FUNCTIONS MUST RAISE AN EXCEPTION FOR THIS DECORATOR TO WORK. + :param mandatory_header_keys: The keys in 'inbound_headers' that are absolutely necessary. + :param mandatory_data_keys: The keys in 'inbound_data' that are absolutely necessary. + :param mandatory_file_keys: The keys in 'inbound_files' that are absolutely necessary. + :param header_validator: The function to use to validate the 'inbound_headers'. + :param data_validator: The function to use to validate 'inbound_data'. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # We first validate the mandatory header keys. + # Having a null value in this case is NOT allowed: + if mandatory_header_keys is not None: + for mandatory_key in mandatory_header_keys: + if kwargs["inbound_headers"].get(mandatory_key) is None: + return ResponseModel( + status_code = StatusCodes.HEADERS_INCOMPLETE, + message = f"missing: '{mandatory_key}'" + ) + + # Return with failure if any of the mandatory JSON details are missing. + # Having a null value is allowed, but is should be sent by the user on intention. + if mandatory_data_keys is not None: + for mandatory_key in mandatory_data_keys: + try: kwargs["inbound_data"][mandatory_key] + except: return ResponseModel( + status_code = StatusCodes.DATA_INCOMPLETE, + message = f"missing: '{mandatory_key}'" + ) + + # Return with failure if any of the mandatory file-keys details are missing: + if mandatory_file_keys is not None: + provided_file_keys = kwargs["inbound_files"].keys() + for mandatory_key in mandatory_file_keys: + if mandatory_key not in provided_file_keys: + return ResponseModel( + status_code = StatusCodes.FILE_MISSING, + message = f"missing: '{mandatory_key}'" + ) + + # Next we validate the headers: + if header_validator is not None: + + # Try validate the data: + try: kwargs["inbound_headers"] = header_validator(kwargs["inbound_headers"]) + + # In case some needed field is missing: + except KeyError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = "missing: " + str(exception), + http_code = HttpCodes.BAD_REQUEST, + ) + + # In case some pydantic data model fails validation: + except pydantic.ValidationError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = messages_from_pydantic_exception(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # In case some other exception was raised: + except Exception as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = str(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # Finally, we validate the incoming data: + if data_validator is not None: + + # Try validate the data: + try: kwargs["inbound_data"] = data_validator(kwargs["inbound_data"]) + + # In case some needed field is missing: + except KeyError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = "missing: " + str(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # In case some pydantic data model fails validation: + except pydantic.ValidationError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = messages_from_pydantic_exception(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # In case some other exception was raised: + except Exception as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = str(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # Now that we have unpacked the incoming data, + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def log_request_to_mongo( + attr_name, + collection: str = "logs", + api_version: str = None, + project: str = None, + log_type: str = None, + operation: str = None, + log_input: bool | int = True, + log_output: bool | int = True, + sensitive_keys: list = None +): + + """ + USE THIS ONLY AFTER YOU HAVE USED 'read_input'. This decorator will log the whole process of the API call to + MongoDB. The variable that holds the instance of 'AsyncMongo' needs to be accessible in the scope of 'current_app'. + PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING THIS DECORATOR. + :param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the + scope of 'current_app'. + :param collection: The name of the collection to write the log into. + :param api_version: The version code of the API endpoint that is being logged. + :param project: The name of the project that the endpoint was built for. + :param log_type: A hint to identify what the log was for. + :param operation: A hint to identify what was action was being performed. + :param log_input: Set to True to capture everything that went into the function, False to capture the least + info, and set it to an integer to capture a certain depth of the input (applicable on iterables and dicts. + :param log_output: The same as 'log_input', but applicable to the response from the function. + :param sensitive_keys: The list of keys to not log. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Create ids for getting logs: + kwargs["log_id"] = "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(8)) + kwargs["log_chain"] = kwargs.get("inbound_headers", {}).get("X-Log-Chain") or kwargs["log_id"] + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Set the api version as needed: + kwargs["api_version"] = kwargs.get("api_version", api_version) + + # Make variables and extract available info.: + exception = None + response = None + request_ts = date_time.get_current_utc_date_time() + start_ts = time.perf_counter() + cpu_start_ts = time.process_time() + + # Execute the function that is being wrapped: + try: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + except Exception as exc: exception = exc + + # Ensure that the response is not null: + response = response if response is not None else ResponseModel( + status_code = StatusCodes.UNKNOWN_ERROR, + message = "null response for request" + ) + + # Add params to the response. + # THIS IS ONLY APPLICABLE WHEN THE TYPE OF THE RESPONSE IS 'ResponseModel': + if isinstance(response, ResponseModel): + response.api_version = kwargs.get("api_version") + response.log_id = kwargs.get("log_id") + + # Extract the response to log: + response_to_log = "not logged" + http_code_to_log = 200 + if log_output: + if isinstance(response, ResponseModel): response_to_log, http_code_to_log = response.for_quart() + elif isinstance(response, tuple): response_to_log, http_code_to_log = response + else: response_to_log, http_code_to_log = str(response), 200 + response_to_log = summarize_variable( + response_to_log, + expand = log_output, + sensitive_keys = sensitive_keys + ) + + # Try to get the information about the request. + # There will be no data in any of these if the decorator was used to catch start-up and shut-down events. + request_method = None + request_url = None + request_route = None + try: + request_method = f"{request.method}" + request_url = f"{request.url}" + request_route = str(request.url_rule.rule) + except: pass + + # Redact the sensitive keys: + if sensitive_keys: + for k in sensitive_keys: + for var in ["inbound_headers", "inbound_data"]: + try: kwargs[var][k] = len(str(kwargs[var][k])) * "*" + except: pass + + # Construct the log: + api_log = APILogModel( + pid = PROCESS_ID, + ppid = PARENT_PROCESS_ID, + project = project, + log = log_type, + operation = operation, + apiVer = kwargs.get("api_version"), + logId = kwargs.get("log_id"), + logChain = kwargs.get("log_chain", kwargs.get("inbound_headers", {}).get("X-Log-Chain")), + method = request_method, + url = request_url, + route = request_route, + ts = request_ts, + tat = time.perf_counter() - start_ts, + cpuTime = time.process_time() - cpu_start_ts, + headers = kwargs.get("inbound_headers"), + data = summarize_variable( + kwargs.get("inbound_data"), + expand = log_input, + sensitive_keys = sensitive_keys + ), + files = { + k: { + "name": v["name"], + "size": v["size"] + } for k, v in kwargs.get("inbound_files", {}).items() + }, + exception = None if exception is None else describe_exception(exception), + response = response_to_log, + httpCode = http_code_to_log + ) + + # Write the log: + app_attr = getattr(current_app, attr_name) + inserted_id = await app_attr.insert_one( + collection = collection, + document = api_log.model_dump() + ) + + # Return the response from the wrapped function. + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def log_chain_to_mongo(attr_name): + + """ + Use this to run everything in the context of the logging decorator. Everything that is decorated with the + custom decorator will be logged if this is used. + :param attr_name: The name of the db connection (an instance of 'AsyncMongo' or 'AsyncMongoStorage') + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Fetch the attribute and get it to log the whole chain: + async with AsyncLoggerContext.logging_context( + logger = AsyncMongoLogger(getattr(current_app, attr_name)), + log_chain = kwargs.get("log_chain") + ): + response = await func(*args, **kwargs) + + # Done here: + kwargs["decorator_count"] -= 1 + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def should_not_be_under_maintenance(attr_name): + + """ + Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create + a boolean variable within the scope of the 'current_app' for this to work. An alternate to this is to set the value + in an environment variable named 'IS_UNDER_MAINTENANCE' to a string value of either 'True' or 'False' for + multi-worker deployments. + :param attr_name: The name of the boolean variable that will hold the information about the app being under + maintenance. If its value is True at the time of checking, the incoming request will be rejected. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Get the attribute and check if it indicates that the app is under maintenance, + # call the wrapped function if not under maintenance: + app_attr = getattr(current_app, attr_name) + env_attr = True if os.environ.get("IS_UNDER_MAINTENANCE", "False").lower() == "true" else False + if app_attr or env_attr: + response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() + else: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def only_whitelisted_ips(attr_name): + + """ + Use this decorator to reject any requests coming from unauthorized IPs. The list of IP addresses to allow must be + in a list that is accessible in the context of 'current_app'. + :param attr_name: The name of the boolean variable that will hold the information about the app being under + maintenance. If its value is True at the time of checking, the incoming request will be rejected. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Get the attribute and check if the request's IP is in the permitted list: + app_attr = getattr(current_app, attr_name) + if request.remote_addr not in app_attr: + return ResponseModel( + status_code = StatusCodes.AUTHORIZATION_FAILED, + message = "bad ip", + http_code = HttpCodes.UNAUTHORIZED + ).for_quart() + + # Now that we have checked that the IP is permitted, + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def limit_rate( + attr_name, + rate_limit: int = 5, + seconds: float = 1.0, + message = None, + header_keys: list = None, + data_keys: list = None, + allow_if_exception = False, + count_for_http_codes = None +): + + """ + Use this decorator to apply rate-limiting to incoming requests. + SUGGESTION: WHEN STACKING UP MANY RATE LIMITS, PUT THE SMALLEST TIME PERIOD ON TOP AND LARGEST TIME PERIOD AT THE + BOTTOM. THIS ENSURES PROPER FUNCTIONALITY. + WARNING: TIMING STARTS WHEN THE FIRST PERMITTED REQUEST GOES THROUGH. THIS MEANS THAT, IF YOU HAVE A PER-DAY LIMIT, + AND YOU START MAKING REQUESTS AT 11:00 PM AND EXHAUST YOUR LIMIT AT 11:59 PM, YOUR LIMIT WILL BE REPLENISHED AT + 11:00 PM OF THE NEXT DAY, NOT AT 12:00 AM. + :param attr_name: The name of the variable that holds the instance of 'AsyncRedisCache'. Should be available in the + context of 'current_app'. + :param rate_limit: The number of requests per unit time. + :param seconds: The time period in which the rate limit is to be applied. + :param message: The custom message to respond with. + :param header_keys: The keys in the header to consider when apply rate limits (like 'sessionToken'). + :param data_keys: The keys in the header to consider when apply rate limits (like 'sessionToken'). + :param allow_if_exception: In case Redis is unresponsive, would you prefer allowing the request to pass through or + would you prefer the request getting blocked. + :param count_for_http_codes: If this is provided, the counter will be incremented only if the response code was one + of these values. If not provided, all requests will be counted. This can be used in cases when you want to count + only when the request was successfully served. + :return: The decorator factory. + """ + + # Param-cleaning: + rate_limit = max(1, rate_limit) + if header_keys is None: header_keys = [] + if data_keys is None: data_keys = [] + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Note down the combination of values requested and the limits prescribed: + params = {"route": str(request.url_rule), "header": {}, "data": {}, "limit": rate_limit, "seconds": seconds} + for key in header_keys: params["header"][key] = kwargs["inbound_headers"].get(key) + for key in data_keys: params["data"][key] = kwargs["inbound_data"].get(key) + + # Now make a unique key from this combination: + params_json = json.to_string(params, no_space = True) + sha256_hash = hashlib.sha256() + sha256_hash.update(params_json.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # We first get the value of the counter: + app_attr = getattr(current_app, attr_name) + counter_value = await app_attr.count(base64_key, value = 1, expiry = seconds) + + # If any exception occurred in getting the count, + # and exceptions haven't been allowed: + if counter_value is None and not allow_if_exception: + response = ResponseModel( + status_code = StatusCodes.RATE_LIMIT_EXCEEDED, + message = "Please contact admin (E)" + ) + + # If the rate-limit has already been crossed, + # or when the counter was not fetched but exceptions are allowed: + elif (counter_value or 0) > rate_limit: + response = ResponseModel( + status_code = StatusCodes.RATE_LIMIT_EXCEEDED, + message = message or ", ".join([ + f"rate limit: {rate_limit} in {humanize.naturaldelta(datetime.timedelta(seconds = seconds))}", + f"this is your {humanize.ordinal(counter_value)} request in the given period" + ]) + ) + + # If the rate-limit hasn't been crossed: + else: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # If we have been told to count only for specific status codes, + # and if the HTTP code of this response is not in the list of codes, we reduce the counter by one: + if count_for_http_codes: + http_code = 200 # ... Default assumption. + if isinstance(response, (list, tuple, set)): http_code = response[1] + elif isinstance(response, ResponseModel): http_code = response.http_code.value + if http_code not in count_for_http_codes: + await app_attr.count(base64_key, value = -1, expiry = seconds) + + # Return the response from the function call: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): + + """ + Use this decorator to handle prematurely terminated requests. If your clean-up function needs access to variables, + consider using 'g' to hold data in the scope of the request. + :param cleanup_func: The function to call when the cancelled request needs graceful handling. + :param cleanup_coro: The coroutine to call when the cancelled request needs graceful handling. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + try: + + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + # In case the client closes the connection pre-maturely: + except asyncio.CancelledError as exception: + kwargs["decorator_count"] -= 1 + if hasattr(current_app, "printer"): getattr(current_app, "printer")(exception) + if cleanup_func is not None: cleanup_func() + if cleanup_coro is not None: await cleanup_coro() + return ResponseModel( + api_version = kwargs.get("api_version"), + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ).for_quart() + + # We propagate any other kind of exception: + except Exception as exception: + kwargs["decorator_count"] -= 1 + raise + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def handle_failed_request(cleanup_func = None, cleanup_coro = None): + + """ + Use this decorator to handle requests that fail due to exceptions. If your clean-up function needs access to + variables, consider using 'g' to hold data in the scope of the request. THIS DECORATOR IS NOT MEANT TO SUPRESS + EXCEPTIONS. IT IS, INSTEAD, MEANT TO PERFORM CLEAN-UP AND PROPAGATE THE EXCEPTION. + :param cleanup_func: The function to call when the cancelled request needs graceful handling. + :param cleanup_coro: The coroutine to call when the cancelled request needs graceful handling. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + try: + + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 0 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + # In case the client closes the connection pre-maturely: + except Exception as exception: + kwargs["decorator_count"] -= 1 + if hasattr(current_app, "printer"): getattr(current_app, "printer")(exception) + if cleanup_func is not None: cleanup_func() + if cleanup_coro is not None: await cleanup_coro() + raise exception + + return wrapper + + return decorator + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/api/codes.py b/api/codes.py new file mode 100644 index 0000000..5773cc9 --- /dev/null +++ b/api/codes.py @@ -0,0 +1,164 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Sept., 2024 + + OBJECTIVE: + + To maintain all status codes in one place. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +from enum import Enum, unique + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +@unique +class HttpCodes(Enum): + + """ + Commonly used standard HTTP status codes. + Can be sent after an API Call is processed. + Refer to: https://http.dev/status + NOTE: THIS LIST IS NOT EXHAUSTIVE! + """ + + # 1XX - Informational: + CONTINUE = 100 # .............. https://http.dev/102 + SWITCHING_PROTOCOLS = 101 # ... https://http.dev/101 + PROCESSING = 102 # ............ https://http.dev/102 + EARLY_HINTS = 103 # ........... https://http.dev/103 + + # 2XX - Success: + SUCCESS = 200 # .................. https://http.dev/200 + CREATED = 201 # .................. https://http.dev/201 + ACCEPTED = 202 # ................. https://http.dev/202 + NON_AUTHORITATIVE_INFO = 203 # ... https://http.dev/203 + NO_CONTENT = 204 # ............... https://http.dev/204 + RESET_CONTENT = 205 # ............ https://http.dev/205 + PARTIAL_CONTENT = 206 # .......... https://http.dev/206 + MULTI_STATUS = 207 # ............. https://http.dev/207 + ALREADY_REPORTED = 208 # ......... https://http.dev/208 + THIS_IS_FINE = 218 # ............. https://http.dev/218 + IM_USED = 226 # .................. https://http.dev/226 + + # 3XX - Redirection: + MULTIPLE_CHOICES = 300 # ..... https://http.dev/300 + MOVED_PERMANENTLY = 301 # .... https://http.dev/301 + MOVED_TEMPORARILY = 302 # .... https://http.dev/302 + SEE_OTHER = 303 # ............ https://http.dev/303 + NOT_MODIFIED = 304 # ......... https://http.dev/304 + USE_PROXY = 305 # ............ https://http.dev/305 + SWITCH_PROXY = 306 # ......... https://http.dev/306 + TEMPORARY_REDIRECT = 307 # ... https://http.dev/307 + PERMANENT_REDIRECT = 308 # ... https://http.dev/308 + + # 4XX - Client Errors: + BAD_REQUEST = 400 # ..................... https://http.dev/401 + UNAUTHORIZED = 401 # .................... https://http.dev/401 + PAYMENT_REQUIRED = 402 # ................ https://http.dev/402 + FORBIDDEN = 403 # ....................... https://http.dev/403 + NOT_FOUND = 404 # ....................... https://http.dev/404 + METHOD_NOT_ALLOWED = 405 # .............. https://http.dev/405 + NOT_ACCEPTABLE = 406 # .................. https://http.dev/406 + PROXY_AUTH_REQUIRED = 407 # ............. https://http.dev/407 + REQUEST_TIMEOUT = 408 # ................. https://http.dev/408 + CONFLICT = 409 # ........................ https://http.dev/409 + GONE = 410 # ............................ https://http.dev/410 + LENGTH_REQUIRED = 411 # ................. https://http.dev/411 + PRECONDITION_FAILED = 412 # ............. https://http.dev/412 + PAYLOAD_TOO_LARGE = 413 # ............... https://http.dev/413 + URI_TOO_LONG = 414 # .................... https://http.dev/414 + UNSUPPORTED_MEDIA_TYPE = 415 # .......... https://http.dev/415 + PAGE_EXPIRED = 419 # .................... https://http.dev/419 + TOO_MANY_REQUESTS = 429 # ............... https://http.dev/429 + UNAVAILABLE_FOR_LEGAL_REASONS = 451 # ... https://http.dev/451 + INVALID_TOKEN = 498 # ................... https://http.dev/498 + CLIENT_CLOSED_REQUEST = 499 # ........... https://http.dev/499 + + # 5XX - Server Errors: + INTERNAL_SERVER_ERROR = 500 # ........... https://http.dev/500 + NOT_IMPLEMENTED = 501 # ................. https://http.dev/501 + BAD_GATEWAY = 502 # ..................... https://http.dev/502 + SERVICE_UNAVAILABLE = 503 # ............. https://http.dev/503 + GATEWAY_TIMEOUT = 504 # ................. https://http.dev/504 + HTTP_VERSION_NOT_SUPPORTED = 505 # ...... https://http.dev/505 + VARIANT_ALSO_NEGOTIATES = 506 # ......... https://http.dev/506 + INSUFFICIENT_STORAGE = 507 # ............ https://http.dev/507 + LOOP_DETECTED = 508 # ................... https://http.dev/508 + BANDWIDTH_LIMIT_EXCEEDED = 509 # ........ https://http.dev/509 + WEB_SERVER_DOWN = 521 # ................. https://http.dev/521 + ORIGIN_IS_UNREACHABLE = 523 # ........... https://http.dev/523 + SERVICE_IS_OVERLOADED = 529 # ........... https://http.dev/529 + NETWORK_READ_TIMEOUT_ERROR = 598 # ...... https://http.dev/598 + NETWORK_CONNECT_TIMEOUT_ERROR = 599 # ... https://http.dev/599 + + +# --------------------------------------------------------------------------------------------------------------------- + + +@unique +class StatusCodes(Enum): + + """ + To be used internally within the context of your service. Customize these to match your service. + The format is: (SUCCESS_INDICATOR, INTERNAL_NUMERIC_CODE, HTTP_CODE) + Example: (True, 1, 200) + """ + + # Legacy Codes: + OK = (True, 1, HttpCodes.SUCCESS.value) + FAILED = (False, 0, HttpCodes.INTERNAL_SERVER_ERROR.value) + PARTIAL_SUCCESS = (True, 2, HttpCodes.PARTIAL_CONTENT.value) + PARTIAL_FAILURE = (False, 3, HttpCodes.PARTIAL_CONTENT.value) + + # Authentication Codes: + LOGGED_IN_SUCCESSFULLY = (True, 200, HttpCodes.SUCCESS.value) + LOGIN_FAILED = (False, 201, HttpCodes.UNAUTHORIZED.value) + INVALID_SESSION_TOKEN = (False, 202, HttpCodes.UNAUTHORIZED.value) + AUTHENTICATION_DETAILS_INCOMPLETE = (False, 203, HttpCodes.BAD_REQUEST.value) + + # Authorization Codes: + AUTHORIZED_SUCCESSFULLY = (True, 300, HttpCodes.SUCCESS.value) + NOT_ALLOWED = (False, 301, HttpCodes.FORBIDDEN.value) + AUTHORIZATION_DETAILS_INCOMPLETE = (False, 302, HttpCodes.BAD_REQUEST.value) + AUTHORIZATION_FAILED = (False, 303, HttpCodes.UNAUTHORIZED.value) + + # General failures: + DOWN_FOR_MAINTENANCE = (False, 800, HttpCodes.SERVICE_UNAVAILABLE.value) + UNKNOWN_ERROR = (False, 801, HttpCodes.INTERNAL_SERVER_ERROR.value) + DATA_INCOMPLETE = (False, 802, HttpCodes.BAD_REQUEST.value) + HEADERS_INCOMPLETE = (False, 803, HttpCodes.BAD_REQUEST.value) + FILES_MISSING = (False, 804, HttpCodes.BAD_REQUEST.value) + CLIENT_CLOSED_REQUEST = (False, 805, HttpCodes.CLIENT_CLOSED_REQUEST.value) + + # Validation failure: + DATA_VALIDATION_FAILURE = (False, 900, HttpCodes.BAD_REQUEST.value) + RATE_LIMIT_EXCEEDED = (False, 901, HttpCodes.TOO_MANY_REQUESTS.value) diff --git a/api/log.py b/api/log.py new file mode 100644 index 0000000..8c8c13e --- /dev/null +++ b/api/log.py @@ -0,0 +1,138 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Sept., 2024 + + OBJECTIVE: + + To have a structure to the response sent from the API calls. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# System-level activities: +import distro +import socket +import platform + +# For data-modelling: +from pydantic import BaseModel, Field +from typing import Any, Optional, List, Literal + +# To work with date and time: +import datetime + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Info for logging that will stay constant during runtime: +SERVER_HOSTNAME = str(socket.gethostname()) +PLATFORM_INFO = platform.uname() +HOST_OS = str(distro.name(True)) +HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class APILogModel(BaseModel): + + # To identify the machine the code is running on. + # DO NOT MODIFY THESE: + hostname: str = SERVER_HOSTNAME + os: str = HOST_OS + cpu: str = HOST_CPU + # Can modify these: + pid: Optional[Any] = None + ppid: Optional[Any] = None + + # To identify the project and actions: + project: Optional[str] = None + log: str + operation: str + apiVer: Optional[str] = None + logId: Optional[str] = None + logChain: Optional[str] = None + + # Timing metrics: + ts: datetime.datetime + tat: float + cpuTime: float + + # To understand the request that came in: + method: Optional[str] = None + url: Optional[str] = None + route: Optional[str] = None + headers: Optional[Any] = None + data: Optional[Any] = None + files: Optional[Any] = None + + # To understand the output that went out: + exception: Optional[Any] = None + response: Optional[Any] = None + httpCode: Optional[int] = None + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + my_log = APILogModel( + log = "internal" + ) + + print(my_log) diff --git a/api/response.py b/api/response.py new file mode 100644 index 0000000..02e3d72 --- /dev/null +++ b/api/response.py @@ -0,0 +1,133 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Sept., 2024 + + OBJECTIVE: + + To have a structure to the response sent from the API calls. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# For data-modelling: +from pydantic import BaseModel +from typing import Any, Optional, List + +# My utils: +from utils_v2.api.codes import StatusCodes, HttpCodes + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class ResponseModel(BaseModel): + + """ + A model for how the response should be when developing API endpoints. + """ + + # The fields that you want in your response: + status_code: StatusCodes + message: Optional[str | List] = None + data: Optional[Any] = None + seconds: Optional[float | int] = None + log_id: Optional[str] = None + http_code: Optional[HttpCodes] = None + api_version: Optional[str] = None + + def for_quart(self): + + """ + Call this when you are using either Flask or Quart as your framework. + :return: The output as expected by Flask and Quart. + """ + + # Construct the basic structure: + response_dict = { + "status": 1 if self.status_code.value[0] else 0, + "code": self.status_code.value[1], + "message": self.message or self.status_code.name.replace("_", " ").lower(), + "data": self.data, + "apiVer": self.api_version + } + + # Now add the additional fields: + if self.seconds is not None: response_dict["seconds"] = self.seconds + if self.log_id is not None: response_dict["logId"] = self.log_id + + # Figure out the HTTP code: + response_http_code = self.http_code.value if self.http_code is not None else self.status_code.value[2] + + # Done here: + return response_dict, response_http_code + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + my_response = ResponseModel( + status_code = StatusCodes.RATE_LIMIT_EXCEEDED + ) + my_response.log_id = "abc123" + + print(my_response.for_quart()) diff --git a/cache/__init__.py b/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cache/__pycache__/__init__.cpython-310.pyc b/cache/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39ed6b7db6d57b47bb261e78ca4d0972dffc17c9 GIT binary patch literal 168 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vTKO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaMNIy9- lIU`j+K0Y%qvm`!Vub}c4hfQvNN@-529mtYmCLqDW008HdDIx#> literal 0 HcmV?d00001 diff --git a/cache/__pycache__/async_redis_cache.cpython-310.pyc b/cache/__pycache__/async_redis_cache.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a0d7d1676adc6400ad5f01feb61bfe8bc3d1977 GIT binary patch literal 13364 zcmeHO&2Jn>cJJ=#`Q(QrT9zeS_O?IP^4Jt9%8Fz&f>v5tvbFY7hD2LgB8y3Ly2)v3 zW_n!RBZ`BeSGI(;59R(Yi?^G36>QjzT@RwkT-PSGjd z(^n>)hn=!hL2l|*&6#i}?`h8DU1Me1-RDf*G{jZe!dNk!X=mR(ZDqf+-#LKi0cXZ} z2+tYU*m=mExvSmR(Tj0ieyfMTOHxw z@*LN)+m>qwTh^w%Wre=A>5H4zrUydD<~9Pqeau?(nr>iuZPrA^cHL^(?Y1lC6!~kH zE?>HKY3bsn>zOevRINrBc7nzE`OVGExy=`Ydc*gd!Cc*M&3CrKhTo1KHFh#*2c39s zS8pu6arMG0Sz=4`7b=xcu%g;hRgX+Iz;pkVx_eQ$cIy>)wY$FViU)d>-xPKy_~bUK z)(tj@Ka`8)`{@jRe4u@xuNhpkd!=ookM*eL?Em0ekn_t8FR&c9?h8Bgg|%ka@c?)1 zn_vXE;I)I$Zr5GQU$epnSSYztx9bhBy>8W;b`V&{F0k;L>v+LMyl{`9p<^}trW06O zepgtI9olRy+&S6ll{L$@g6?YIzSDKvp;fn=O$*~_*;phj3_A+!mYWRBb*!~+yB>Oe zTdh}G!E1NAVK6IO2sh|9L(*AuLkt6rgx}&T*(t~D%K5mq!|AO1oI=*ix-R53_=N1>kra?6KEES6G`3 zudYTQJ8%@kTHPSDR$VaJwT0`%G=&>>MO#rDtXtTyL(A@le#;I$vU5v~HM0P!eH{y* zekeC4z7OI1=}{!1_MvuH|3Lr1_&{6BDVe#eZ|i-XQnOS?mGo0Yu7vNO;v4)F$$R>T z#)cN^vYFP`UID4Q`Xeam%jdQc8auiB+IFs=+cEpbPTtAg*J&j1yMS^tEbf&0=6!vq z%s1Ql-qeTMMx~#3Oeg=5aRp}XHEp|aL+cl=Yr6dV(CBNP*4KZk|4>ifqs>G=-!FWq zJB5#Os#Mp$BU@FAOJf|Mwp_>e)BQ-+G4Su2obm4(??6``Xo7b5fe{&VCnCKjf63&t zH|HCE%bj=JTW-_uxMKdzt$M>2tv7|g;nu@o-tKrcdwspu4e+uXdd;AA>*Ty#;(4h@ zH7u(a)MUwAXDh0lZF)fnyU{t<;DES3`?cPAS)t~Ib7z}A7B@IIm#+Mvi1q-Z7!(YM zeU|TrYsX(&hS}IBY%g$Y?(MqUp*}sxMY)^qR+PVGH@j|BP{LZBj7qgy8!A|JI3(Ct92*`i(d9XuUx-3`L1uxbbH=~Gd8_4KLq6{H@)lr+p^%!POfYpM*&A=!!n0SggGg%(xgFg= zpBmXHpZY(zPYBw4?giP6q>Qq3FZc`mcux;uNU@n1+d1qCdZ+*hEOojo0E%#TfYrxt z0PCOc=XdgbbElB*3i+KP$_rs>r`#`KSE$Hcq1c=1=V0NBL%Tu=yFw|uE3CtW|3tsW z-NDH04kfgm>=*l`vE2c!zi4+5oTJh|se}{5KdFvlACf<0hC4~`b)}pVBh^|ifS?l$ ztaEA=21Q?B!NmbkTo%N0;vkahR8)jDr7#K_HF1Q=qf8!S!p&Sf&g2OuUt?l1d6LOf zOrAzkEk|bHHrK>6EcrT;sH{fquWpEES$K@eb4;pCo<{{5cPgf34x8Mg%H*E;6W#PSZv z4lM7Pq=V?8U5LHM4r3$yxOfS_h5!?OYk7Rn;~V@8iL0&X@Z4h6aB@KC8ibl@{>QnM zoKukbqD)I^xmhlkE9D8N>P$LQ@Y3?JA6AI{u%cU1p4eS|r3|m`VfbVf=aBOVo)gYt z=LnvY&ZEv_cuqObJCD25@DHAFzIHFavd?+JdD3|bRQEfx&eP5_$Q^KwJI}f^vYzQ2 zbDl%Jha3y)G8gR|v1mh9ffj-g<|4kI(mc>6+y#_;pf3T6m;^g`>bEm$#np>SXPb6w z)v?e0C0mk2N$VXR=?Z$ID(%^)SVT+ln@r9!Imbi_=|ED?qwKF(egVlCr5p<7=sB#u zw!ts(?T-lzjiss)=_1lSfI}1BSu--dmc71bH-8Np0VgvJ4U*~?<-=~L=|)9Dz9ujk zIhK5~+#AfKs>_ypcJjAqzWZE~l^CLRh1K`s%irxui$X5>LW9Irb@U@ljU%x5;7x8x zyu!wpkN^yQi}}5J`59hHw#;as{2Lg4`LoKk;}@l~*0<_ZnB>EJ(sRHSb@aYggMCK| z_ZF$qXGtzJ`zF-M{D@`wel9qQb<5MoVAulz3A!_R`v_Pf5fB~WD_nsVyix4hR^aiM za-rz1N#8bh)0AIJ4_CQjs1NV84ZrIa99DR`;Z_H}bJM#?Pmj*&TGO}rTxD2bHti9x z8t4P%jt6gH7A*okfRB-qz7&qYTCAo`5;xRGP(r%HHAO^W5xA6bvGvxqH>@@I^GOjM zU0+Q@VGd3?rq;nIYIVQchASF_nr>?qZYg|ZK@T0nN?Q+ktZ?oTOMwFq+N8p#+YY({ ztV_o-)}O;5#-eqjfi2M$vlfQg_QP3*3Q}r8!{3DW&u9nfw%j1F;a;R3y@jZR2zMu4 z>j-zvyRD|8@>b~HmZOw(Wypsigz*jr6qf`Qa2YyYP&A@xuU&6;9rvOSPTi&xbGSBQ zlWMhk(}N5}PmI6GvN?Kw{C&deT7Z7s8X_xUjcuZ*$LkKb8w-Gd;%L#4@tTN`ab2S# z98F@=YF?aT)dOQBqRF8V<|T8YN%@FbI$}KOBYJHwtkpgS$zT#mVWyz-Z#s8KAGg>^ z^*!y}r-_~2(WS1!FPHYaYKXTvYB-ym1WXDiF*=+{^|mI5{73&IT;&+>lvcwn~201iB66Uy4KE!m@G)F1NlmC@zCvm$Am=^86auVw) z{~Ux?WBy~T0iv+3C1Wyyg{Y_8fCeLthM@|#_8jGgU%CVDL1w;N+@=$T(A(SC_TDj^ zRGDW^1&=qaMnLYL=2g{wwD~`*Lt{ zMla=500?29Fg6b;LS(3O=KZ`&+N(kav0`K!Iz`Vp6Kg!~a zW#6(L$EqW^i{P}3jKdV)as{lk+-4it`hh*#?qnntY= zMQ7ZY0em7C_ckK>hz3K3pc`ERoB_R9;0ajv2x;(D(idDpfDWpV8sePw+Lfi{MT>Qw z!^`Si5;7d?0hk5|Bc5fA5jvWV3{6F-imc3IRz7#fVF2zs4p5zk6&l%djOLX9F7(pOj$E zNq>mSs)LryDF!g3GMvD9eo|548+&LP38c(Sb9icOPg^#^q|CSz+q2}i;}aOk9|D}@ z!#ovJ9tF@>`w%9CzMAr%q*4*VL#=1VS5sLKDwjBqWE#|@6VjU>H5rio0rN6;_tA4z0z6LxWwFd09%9 zL{meP{!bK1n>sTCj5LN2hUyMs6D|)s_?QGHLUZeB zUE9`mtvA`%fD9xixPe0u$Jo~2g_+{1EOVRsrdBn3f0ByavX5gp*bXR%&0{?rkG1`7 zKnYEpix{aUa;KaloN++>Sq6Urd%hPmpreCUyNKhm=t67Ol1i>KYq?D*&}AHPWaM;t z>2kFY<%uSOsKhh9T30m1Ti}4Wh9oLqN=`E*N_=3%E0i8`jBar7x8-CGKsJIyAYt4w zil%Prh5st#rj3d|t(Wc?zwA9SD!p;Ly_e;;1o2EG5o0dE`2Rv7?ft2(4t#8Dfp8=JE z;(V6g4Xr>`k=)O&LLck4VL0X)n93TUId}KopQUR8)rq}di4r#nozwNf0j_m}fYa?r zUpYb7*~PMOl!#r*=XpCXAu!?qD1cJzWDO)Sdy~UpTJgoixPM7)iv8M8utIPIqsJ#j^O{W*ay^1Y0G1?u)sBE+}XHuxHM@RI9`*UpkC1*mFhHYntJca(X}6= zBj^dqc0i?WR{1^)={i@u39vVYgw~PJa4WMQyYh~*yM5Y9Lx(~fh^h?sm zGQ}(JL!qUpl-?~7?m=jflUB80{{dK%O2>8k7Cfu~N5%@*CFZlvP*+Ax)lCDSY8Ajh zM%{TBToRC=ann->LV5~t9#lL^co3HOs(_RZMNK*qik{4}Z`p9H->-66VK5Ha%AK>m z9bZ|1^XCVijEE`UY^|(UdPd#I@{ZjostOB>}?nctzY|?P238gT7a&VRLwC+?k#* zj7n~jI+in_XQpo8&-|sCGe569tnT3a=iex(RUDf zxX*hQR1U;gci`aW{#;6Kv9ra=0JIw46}B5 z&uWd~Rz!!p?aUOHq>rwooze>NmEuoPS3>_JbHt$0gv4O6_wKWJAoazJ)Js_}#n z98S37HeoZQj4CsnTu_To8dhDz?6LKSuzI*N=4Q>?|3c|ag)PSMYI=K_+~v!NJ*P5- z`$Cc#8PQRZX(0`!borjg!tE+6-^YvZQ&uRckd-oJWq&LqvC%s;YV_j%dL+trL+FAY zZz?dqj~`9_@v;BN9E8m$w8908rt0(Va*l>MKidJC@ z=6oA>awVbw(vYHMigT%G72hqY!>VoMq-Yto0Vz*K3wV<;&E&93o!|sXoE2bE9_d)m z;p%HU^wvDwh(u_Yd+0HsieptsnL2WUz$I`3XAclq)K9lg)K7a%NG7#W7ja{SK1f$d zb@DbtO9eJgWGvN)JZJCG#J!AF&Y&2(#mUhSrk!NeY8Aoe>n;#c+g(@ZaaCZ+nt2Nln~MjafG6moQD< z!d1-UwINIku_2gvp55TOuUt}Fog?yoYX`S(xjy48$6Z!H7+!B(8}ty*?AO12r3L^~ zZ1Mmhj3porfh7xR-H5?zvH=kVhw^}76QX8qBF3;TAuMM!Kzy6rin|>gByhfi2`yDi zSxchCnCPu8R=46ijIOu?HO`oqQf6nO{%7#mKD!A)cD8UW5(?ouQ5n(Knp}_F><{i6 zQu_Xw7D!!inVy0LG7jn%L01-fJv?fKWJ?_XFJT0i1rK`wMd|SXe_qC&@5^{c6NQ+^ zK?ysfO0CxNoom z-WGGT!%|=By{q{%Siql-{2&Yl2$O~c^?p`dKzWo(8@1L zkCk#puA-M7DLq^|P};ZuaOIiGk;4aNd!8G%7{0F!x%*(J46k3rQ+$(&%w1+~m&*gV zCMpB5^5(?fqp>`46O0p#k!AE=5__G*-eB@=CcBbV@jglRAGpK@YR>55M|jlnO+PFr z(D)eNKm~q6>;gQq!_|Y_HiHtp_?CxHiYp0Wp4XAF0{|lPKP^5=E-b?nb;>xWF?0>N zN(hjBSLgp-ur^O^s#8n7Z^-Z${>Na^sh@J}mrpIce5U@=sTb|l)ApGcPb@6FSU^*aO_4If=3>vN zR1huB!45-RDK|<82M!gNO0rb~%qTCAs938xejVC0r8s3X%<1`2G0q6VZuTV? zw&!A?%jJ$rUfrbyZ>a self.__ping_counter: + try: + await self.__client.ping() + self.__requests_since_last_ping = 0 + return True + except Exception as exception: + self.__printer(exception) + return await self.connect() + else: + self.__requests_since_last_ping += 1 + return True + + async def set(self, key, value, expiry: float = 120, raise_exception = False): + + """ + Saves some value to the cache. + :param key: The key with which the data will be stored and retrieved. + :param value: The value to store. + :param expiry: The time in seconds after which the data will expire. Must be a positive number. + :param raise_exception: If you want to raise an exception if the process fails. + :return: True if cached, else False. + """ + + await self.ensure_connection() + + try: + + # We handle conversion for unsupported Pythonic datatypes: + original_type = type(value).__name__ + converter = self.__set_converters.get(original_type) + if converter is not None: value = converter(value) + + # Here we actually try to store the data: + response = await self.__client.setex( + key, + int(expiry), + json.to_string({ + "value": value, + "type": original_type + }, indent = 0, no_space = False) + ) + return response + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return False + + async def get(self, key, raise_exception = False, on_fail = None): + + """ + Retrieve the cached value. + :param key: The key with which the data was saved. + :param raise_exception: If you want to raise an exception if the process fails. + :param on_fail: What to return if the process fails due to an exception. + :return: The retrieved data or null if not found. + """ + + await self.ensure_connection() + + try: + data = await self.__client.get(key) + data = data.decode("utf-8") + data = json.from_string(data) + converter = self.__get_converters.get(data["type"]) + if converter is not None: return converter(data["value"]) + return data["value"] + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return on_fail + + async def delete(self, key, raise_exception = False): + + """ + Prematurely delete the value from the cache before it expires. + :param key: The key with which the data was saved. + :param raise_exception: If you want to raise an exception if the process fails. + :return: True if deleted, else False. + """ + + await self.ensure_connection() + + try: + response = await self.__client.delete(key) + return True if response else False + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return False + + async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False): + + """ + To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the + counter will be created and the value will be incremented from 0. + :param key: The name of the counter. + :param value: The amount to increment the value by. Send negative values to count backwards. + :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created. + This value has to be an integer. If a float is passed, the value will be rounded off. + :param raise_exception: If you want to raise an exception if the process fails. + :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed. + """ + + await self.ensure_connection() + + try: + + already_existed = await self.__client.exists(key) + new_value = await self.__client.incrby(key, value) + if expiry and not already_existed: await self.__client.expire(key, int(expiry)) + return new_value + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return None + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + def set_complex(x): + return {"r": x.real, "i": x.imag} + + def get_complex(x): + return complex(x["r"], x["i"]) + + async def main(): + + my_cache = AsyncRedisCache( + connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0", + ping_counter = 100, + debug = False + ) + + my_cache.add_converter( + type_name = type(2j).__name__, + set_converter_func = set_complex, + get_converter_func = get_complex + ) + + value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624") + print("GET:", value) + print("TYP:", type(value), end = "\n\n") + + # success = await my_cache.set( + # key = "name", + # value = {"first": "John", "last": "Doe"}, + # expiry = 10 + # ) + # print("SET:", success, end = "\n\n") + # + # value = await my_cache.get(key = "name") + # print("GET:", value) + # print("TYP:", type(value), end = "\n\n") + # + # success = await my_cache.delete(key = "name") + # print("DEL:", success, end = "\n\n") + # + # value = await my_cache.get(key = "cnt") + # print("GET:", value) + # print("TYP:", type(value), end = "\n\n") + # + # await my_cache.delete(key = "cnt") + # for _ in range(50): + # await asyncio.sleep(1.0) + # counter = await my_cache.count(key = "cnt", value = 1, expiry = 10) + # print("COUNTER:", counter) + + asyncio.run(main()) diff --git a/cache/async_redis_cache_v2.py b/cache/async_redis_cache_v2.py new file mode 100644 index 0000000..7c82797 --- /dev/null +++ b/cache/async_redis_cache_v2.py @@ -0,0 +1,458 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 26th Oct., 2024 + + OBJECTIVE: + + To provide an easy way to cache data for fast access. This version has the change that it can handle custom + serializers by way of dependency injection. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To use redis: +import redis.asyncio as redis + +# Other utils: +from utils_v2.string import json +from utils_v2.serialization.pickle_serializer import PickleSerializer + +# For debugging: +from icecream import IceCreamDebugger + +# To make a decorator: +from functools import wraps + +# For hashing and shortening the hash: +import hashlib +import base64 + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** WRAPPERS *** +# ***** **** +# ***************************************************************************************************************** + + +def cache_it(cache = None, expiry = 120): + + """ + This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. + If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the + cache instead of going through the whole function again. + :param cache: The instance of 'AsyncRedisCache'. + :param expiry: The time in seconds after which the cached data must be cleared. + :return: The decorator that automatically caches your data. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # We first use the name of the function and the inputs given to it to generate a key for Redis with the + # simple hashing and shortening by way of base64 strings: + inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) + sha256_hash = hashlib.sha256() + sha256_hash.update(inputs_given.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Now we check if we have the value in cache: + try: response = await cache.get(base64_key, raise_exception = True) + + # If the key doesn't exist, we pass through the function and store the results. + except: + response = await func(*args, **kwargs) + await cache.set(key = base64_key, value = response, expiry = expiry) + + # Return the response from the wrapped function. + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def cache_class_methods(attr_name, expiry = 120): + + """ + This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. + If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the + cache instead of going through the whole function again. + :param attr_name: The name of the variable that has an instance of "AsyncRedisCache". + :param expiry: The time in seconds after which the cached data must be cleared. + :return: The decorator that automatically caches your data. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(self, *args, **kwargs): + + # Get the cache object first: + cache_obj = getattr(self, attr_name) + + # We first use the name of the function and the inputs given to it to generate a key for Redis with the + # simple hashing and shortening by way of base64 strings: + inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) + sha256_hash = hashlib.sha256() + sha256_hash.update(inputs_given.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Now we check if we have the value in cache: + try: response = await cache_obj.get(base64_key, raise_exception = True) + + # If the key doesn't exist, we pass through the function and store the results. + except: + response = await func(self, *args, **kwargs) + await cache_obj.set(key = base64_key, value = response, expiry = expiry) + + # Return the response from the wrapped function. + return response + + return wrapper + + return decorator + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncRedisCache: + + def __init__( + self, + connection_string, + serializer = None, + ping_counter = 1_000, + debug = False, + debug_prefix = "R-Cache | " + ): + + """ + Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools, + strings, dicts, lists, sets, and tuples :) + :param connection_string: The connection URL for connecting to Redis. + :param serializer: The serializer to use. + :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection. + :param debug: Whether, or not, you want to show debugging messages from the start. + :param debug_prefix: The prefix text to show with the debugging messages. + """ + + # Note down the configuration: + self.__client = None + self.__serializer = serializer or PickleSerializer() + self.__ping_counter = ping_counter + self.__requests_since_last_ping = 0 + self.__connection_string = connection_string + + # For debugging: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + @staticmethod + def make_key(*args, **kwargs): + + """ + Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the + inputs stay the same, the output stays the same. + :param args: Any number of args that you would like to use to generate the key. + :param kwargs: Any number of kwargs that you would like to use to generate the key. + :return: A string that can be used as a key to store values on Redis. + """ + + # Take everything into one plain text string: + plain_text = "".join(str(a) for a in args) + plain_text += json.to_string(kwargs, no_space = True) + + # Hash the plain text value, and create a key from it: + sha256_hash = hashlib.sha256() + sha256_hash.update(plain_text.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Done here: + return base64_key + + async def connect(self): + + """ + To make an asynchronous connection request to the Redis server to establish a connection. + :return: True if connected, else False. + """ + + try: + self.__client = redis.from_url( + self.__connection_string + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def disconnect(self): + + """ + Close the connection to the Redis server. + :return: True if successful, else False. + """ + + if self.__client is not None: + try: await self.__client.close() + except Exception as exception: + self.__printer(exception) + return False + return True + return True + + async def ensure_connection(self): + + """ + To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected. + :return: True if connected, else False. + """ + + # If we are not connected, we try to establish a connection: + if self.__client is None: return await self.connect() + + # Else we check if we are connected. If not, we try to connect. + # But we check only once in a while. In the meantime, we assume that we are connected. + elif self.__requests_since_last_ping > self.__ping_counter: + try: + await self.__client.ping() + self.__requests_since_last_ping = 0 + return True + except Exception as exception: + self.__printer(exception) + return await self.connect() + else: + self.__requests_since_last_ping += 1 + return True + + async def set(self, key, value, expiry: float = None, raise_exception = False): + + """ + Saves some value to the cache. If an expiry is specified, the data will be deleted after that many seconds. + :param key: The key with which the data will be stored and retrieved. + :param value: The value to store. + :param expiry: The time in seconds after which the data will expire. Must be a positive number. + :param raise_exception: If you want to raise an exception if the process fails. + :return: True if cached, else False. + """ + + await self.ensure_connection() + + try: + + # Here we actually try to set the data: + data = self.__serializer.serialize(value) + if expiry: response = await self.__client.setex(key, int(expiry), data) + else: response = await self.__client.set(key, data) + return response + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return False + + async def get(self, key, raise_exception = False, on_fail = None): + + """ + Retrieve the cached value. + :param key: The key with which the data was saved. + :param raise_exception: If you want to raise an exception if the process fails. + :param on_fail: What to return if the process fails due to an exception. + :return: The retrieved data or null if not found. + """ + + await self.ensure_connection() + + try: + + # Here we try to fetch the data: + data = await self.__client.get(key) + data = self.__serializer.deserialize(data) + return data + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return on_fail + + async def delete(self, key, raise_exception = False): + + """ + Prematurely delete the value from the cache before it expires. + :param key: The key with which the data was saved. + :param raise_exception: If you want to raise an exception if the process fails. + :return: True if deleted, else False. + """ + + await self.ensure_connection() + + try: + + # Try to manually delete the key before expiry: + response = await self.__client.delete(key) + return True if response else False + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return False + + async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False): + + """ + To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the + counter will be created and the value will be incremented from 0. + :param key: The name of the counter. + :param value: The amount to increment the value by. Send negative values to count backwards. + :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created. + This value has to be an integer. If a float is passed, the value will be rounded off. + :param raise_exception: If you want to raise an exception if the process fails. + :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed. + """ + + await self.ensure_connection() + + try: + + already_existed = await self.__client.exists(key) + new_value = await self.__client.incrby(key, value) + if expiry and not already_existed: await self.__client.expire(key, int(expiry)) + return new_value + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return None + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + def set_complex(x): + return {"r": x.real, "i": x.imag} + + def get_complex(x): + return complex(x["r"], x["i"]) + + async def main(): + + my_cache = AsyncRedisCache( + connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0", + ping_counter = 100, + debug = False + ) + + my_cache.add_converter( + type_name = type(2j).__name__, + set_converter_func = set_complex, + get_converter_func = get_complex + ) + + value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624") + print("GET:", value) + print("TYP:", type(value), end = "\n\n") + + # success = await my_cache.set( + # key = "name", + # value = {"first": "John", "last": "Doe"}, + # expiry = 10 + # ) + # print("SET:", success, end = "\n\n") + # + # value = await my_cache.get(key = "name") + # print("GET:", value) + # print("TYP:", type(value), end = "\n\n") + # + # success = await my_cache.delete(key = "name") + # print("DEL:", success, end = "\n\n") + # + # value = await my_cache.get(key = "cnt") + # print("GET:", value) + # print("TYP:", type(value), end = "\n\n") + # + # await my_cache.delete(key = "cnt") + # for _ in range(50): + # await asyncio.sleep(1.0) + # counter = await my_cache.count(key = "cnt", value = 1, expiry = 10) + # print("COUNTER:", counter) + + asyncio.run(main()) diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database/__pycache__/__init__.cpython-310.pyc b/database/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..906606ac1cf8ca05b65fd430effb1c266859207e GIT binary patch literal 171 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vHKO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaMNIxa9 oBrz$mI8{GBJ~J<~BtBlRpz;=nO>TZlX-=vg$ev;*Ai=@_0CmkOW&i*H literal 0 HcmV?d00001 diff --git a/database/__pycache__/async_mongo_v2.cpython-310.pyc b/database/__pycache__/async_mongo_v2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c0b66d1cb12936f513d6de6d40414b5b9fe4d417 GIT binary patch literal 40273 zcmeHw3vgW5dEVZ;cb@=3@F9|-9=@h!gDX*>tk_aa%d+{DD2t#%Le>b4v%c7S0W7%K zh0a|N0u0EKCC9Qmawm(IX74(j@9=iIqi0O@7+ToU+O=@~)b(h6 ztGn%)+|+KbaP9_g_tQD)xp_M0;YGOnP99A?cTzQHn{B6cz@A+4eAijoXW#k5-8uV&9Nlr}JEVEGc%tJr zXPTaydh+de9V49ed6>>)D;QXE(fdSD|Z*EvMnx zGpe&-FVA_(vz=DUcKnrg1Mh0jc8(mfokq8Lw%J`l<;6v>?T_m>oH}~!=&7Rf5+ADMXYGwfj`lJ$CxgQsa!*tr8j9JY!7dHK+lg1;_x<0$fu?chubi5Gr|Px8`nh zw?AV%ZA?|Xs(YQg z+*(*UGCg^rEYI0%cB7IS3m$Vg4&&FKLgD7ta&=u|Sa58vER@z-0cBy$Kv`Tf zQI^)qD9hc-T8Vi29IsW^ivDeDx%uk3q33hJv>dPszqS0tB&MynsJxlxpkAB4F{!qLqkBm?oJ2!`ivh`>->&-+gb2}LVdBLx2G**Z@ZPGXO?tqlj{HnFb`pO<+Z`EetY=QTl)HPKBwM>-dvd6Ki66C_PgF$uhm)f z)c%K78gq_Xct~{!@&0~iu~~O!XY0VZr3K*0{-tiS<=4;Nxj%aEe!=ki0x=xd#}`-d zjrC@`*{#>xoZlTNikpfC{^g~VqY-0=vAQ!0O5;(z2Iye?!($V_1eO;^P+*dC=Zt8Q z&l?jpQ{8~;LD6e-K+hZMCY;I_>SBy{hxJ`OF@GIbUSz8y#_A0{y+}W|VNXg6<;TQ6>(nj-+{7tuKaxNIQY>0xpirK zEo!@Xu7dm3=ZSqo-C-OLJrW0I?IW9*>_rpuFyq zY9FvCz(JDBtDAtBf?Vy{@$qrYY%l~Ac&O89okqJjX||)2+K9}$+J*5@c^22PAh@aP zS!`vo8$~eWd+MyG0F^VHV0hN+)}zj;oqT8;3lb!?9R(&-;O9_%T0$dp_@( zPgYA+%QUR;U-hC{)yL&miltGbYK-Pbjny4Ja~(F5ndJ!3WtR7#ZO-yMW*Kw48S@R0 z#|#J@R*qVUUpv!L7*U^2QFV9D>=vtHX%PS?YTx9zcd(;c&$?O53HyDO~}Y#&&3l(PVV zh+N|U#4Jow$DUM6K$i}u0F$KWEo#=an=bD;w&pYM#%o;rnA7s5it-v>6Eaw{i$>Xl zW*al!h78u3iTlaEjz#jz*tk7O_`rY6=*&`!f?C^N=`7i8&y#n7W!&!gesdPhxiYCU z&6Wp2jScZ9p0`QB!sWghlPS_c9v?t1tUsP8s0Yr)nDwIAP;?XbzRXGC%oeLaqTM|+ zn|d%arD;TZk@P^5%LILZR-96E^{3;{()0))jIYuH0Y@Gz5#WtU({gJQGu@p7c8ToU z?sV;G595~t`(;JJ789rOrqRM!ZZLNGEn}FCaN3gRhy?FMzcnTdLV}{M{Z%~lA~}Qk zeJqh9YJiQb4o2?->GtU=>*J{%vC>w!eV zbf>@0d-tGN-O@8E?(}6xOz(vy{j3kzZ!r4VNIBPt=KvC1E)_;g!G8ua0sUf;D$7gI+3tg=$62zo+>e^ z>#H3J1+2t|G4+s?L6W*)Re*$$12x-=OWkyC|6eB)B`KM&%Quv)X1ZVet1kGC&Nhjl*0&R#dMA5@}x4(@igP;9Z&gA3}kAM zlFtdplX#(rQh&PC;l;iun=lQuyZYI7XPhJEFG4LBO3qQkh-xOUKivj-vpK-f$we{g zYti*)oTXOSWO$MlB^DRK@6jqdq{(B$X=~ybu{BUcw+Vg)$sqND^mL~_Uqc!RvHsfZ zhmwuxuY?Ar`-Vv;4Xot@i_a>~ggU~aM|vCScGC0y5+0Fx9~oM`sb}7^PX;+f_{hvV z;fz=#^WMh6Y;GMw^|{>hxg)s`?)y=5-CBpzsf^mqVW;$S3Q0wps1S}70zD)d@UO1hn|WIIxCBja|mA#;8#?R^QSC_k%i+;TpsPk~{%>g#(hi zt}$-_m~ZIPHVe&>I^?`1TIr58;~aERT+6lb7r@N>xo(NPrdx(?v)9eL)-!132(H9; zD!6mETX2iKbFNCH;@vgcbV^Q;2K~so=yp86J=V4TrA69C>~3?xTZMkip8#CA`tamo5=S75NZp$2H&Zh!lJtVtokP2PGsHlM zi6n(U1iZ1mLW=0I6Lv=oh?Kj-muX>}?}$1%ZXehBRd1zb$B&tD^swD_Ao=f;PR7@k zo6!I8Nq`nuv_zVRYOvfy4?zuJ*J;BtjBf*Bl}?>?T1)-F*~&TXu6)p?l#9J zt?YZZORLRx0{j_WA!=B91cDS+$KZvdl++9I7@3Jaup^f}jbBmIW&sKxEUg<|D8>Y- zpD-VT5nTWu(xG6}w)iC!M!Y)v2lio(-w{|u#~v&Rs`s>-ez(zCSiH9e_!9fjj@;^9 zJt#7u)$x18{OR9|gQ$s2_lYs&QPTZAIB2r_o4_QDbu;<|`3o}tlgX#3hw<$Bym|zG z)Q_+@#iEC=Rqr9%O2_vR%gL6CxuQAx-Qstzu={aCwITHYnhDC$^aR%8N_RCVNAMOD zyQ^YQDa*IhJX3Z2_^#DmS@Z&n!=c{KK9fvg^ve+(1jVIpquyQZ`55&9JhzKqLCtM= zs0u@2)fhHM@C);yS$x$htnRt|lw^jVLR-wM`B@}dE=bnj7zU{_Xj)k}AA$MHf?fLQ zypd~{-=6cgFM|W~-=p{?Xum=X{V9Rj>;$E-77fwfnVJ<;;_d_%55o%p1t%Did(Czo){LIH3x?Pv z`4{f}Py~s0R;rjmt6zUlJzOhKs3CMtkwK|VEGSA(s?98G#xZEbhj2*E_&(kx8>(%i zshMW_dL2f4*r44djAA_)uGb%1a$5S1+RNu}XR)7!7?!AA2g5LzzyzjgxOx{al@}H5 z;N2DVZa#Z2i}$iP$l@3a@=^g(^%##5RFor{nuYqbaNxfWMLB1}KA8Sj{dW1LavuNQ zYF390bGT}l)$}i~;dcwJ?H)0twFVl>{E(pV$NymrbL36>cy7j^L*$wARGwZXcdsx| zW6HuA&uebUEyKH12=7+hD$a`G*^oPovr>4r$sNI28E3lYW_J|l6`XC6YrJ}^I5dr^ zs(wnIyv^P2UWX@#GS_yvJ8^9|stJ!5>+f@Sx!2?RrttiBcei^3&PRARyf*IM=-z~D zo2ABH?X{X34bN=%&A6v~#P1U?*3{PUF8x@>)Ha;yZ=f&BnA-kvVENQ_z}@{pHK8OK ziox+E)gb(d4T)yKdC&?FZu4`-r$8C#UJ?($dAJ23bHL3*Py@B5`8co~HR}h(Gt2yY z9tvq4o+g-s>U9k>0x7I;A7)VV8e?@^(w*2FGI6ZIoW4m>djwPvEf=3C@>zYfY%GJEI3LCb#3b-R5PaXJ#Gy4f*LxhY5%f?8vQMj}9cKw$ z3dvt`OVhvP945*R?44Bk*ic#oA#dTE_Qfv`Tk}0wFgx%rOoDudCw6m2Do2J+m&BE} z&^CGov9Rsa$k$0rFIqiOP7~fW@9_rw;?n3Na}fGGwakf2rcd%Tjj?eBZDaM} zXVbFfi~|o^Y5>x7Tz}C_{Iz%k9VF~KM^pFE?rC8Hz*P#rG6Tb;k+IBlGfqTi8d^78 z%pb&RL+h$fhq6=6%+!uBAeOj9WGQKph4ziKm_piD(_%`HhXN85M7m5-M*;UA#beWe z7DcCTS~3~um%~3JlRQY88c#$VsQX%$5UD6ALlaxB=?Tv}3}S$VCSj#i0CXy9^!UaL${K`91(t#C92bQ`u zMqx?^cs3;+-kKU(XAW+TFp3Uwg{am2y*N5id&xpAcCYJ_3 z%X9l$zB-W^Un?IJi5=94w4oEWsK`1D3=iQMeq>m^Vo}8hfsACGWm3jvh1ei5qG-%3xELVUxKu*|&k1$;+5w#O+K!4*K)jJSSYs}!D= zyJsX;fH)B~n@m{bu?p>m8mZ(ta7$o{*ilmxe6>sLh|q|=k{oKF1;17+2SAJnhc*!1 z1-v0Sinm{cO_=!E4k8C(J!^CjPB)~lcoHm;@SJkObCd^wOCVkSEuL=3WqulW{|SLT z6ezc9fwJ5yP^N+j#*roN-&(lfi%TW zAwf`yS#$~&K66P__}jSqd%dWz8-B4KL2E~f_J^pzZe>)fGzWN*{@`9*pj5QFC5sDd zaFRWj4<)ToWrDJlU%4xV2Uds&@Dyk8Kr2fL<*%kCtz%G;Aa%od08eR0!5u{ZtcX{P z28ac$Sh7mnB((CAh)!a)3i;?rkQ?-im^C_X`NvGZ0bBR-`qn{hW#Czn|nKp#LXMv+ksK%&Z8QDS1Z zx~7|N0;$%p+6Ug^T`q*D)tx;E26H9CDoSqQxnjlyT7j5V2`v*audHKTP-vMztTUpj z%j&%`vQ?j`@>w7-jzr8VI^P_OY?z=0#)s+o+CZs!W8mDG=3+=UbRbeZBputwJxGJ_ zuY2}lvvG!~+(sY-=@q>76A&;VNJ(sx!{tm_gFz@WwwepgFaQNH;^UD*8NKFvpb9YL z=pKe99b3VFBn+B9^mM3_CWu!p8F7?yprXxa`p?mhM(Iij3RcH7-50)#pBIRu@Do}b zIFy)9EQ#WL2+4>HCnC_ofr#*ykE240>I|iXS5h7IcUb%ai_f$89E*>#_#}&SECyIi z=$8aVBQLgrkN!BGgZj4H%8%yZLCeE~milL1D3+=(S(`I-X?u#b$Bi{;G>Lxrtuala ziKdSxnd_z1H}}vTP#vzm2w8i1tvnBx%NfhxgCk3gxrOx-C{PJh1-11N7=K~q7-&%? zR{1JIi;C*qF)gY*fmKgJi*OXuqPx?y=;1{$MVLUq`6GcSVG1GY^m6v(Jmbt*lBW7< zr|l(Jxun@N%&C#dqyyQz!9JwzCbYn8(4M}mF;UzGSvSvywWuc@VAxC=)wOsQsa??s zNQ4e!gPaz<&MUA(LN4$7va1upzh~oXY{*GIeBv0qm&K(;j-^I8^5n_$8I@GefT*M+ zaI(v?QLMj!%_|tvReP1wY7c(%6$DH5_tAO^vAK?BV@>rJarXnn>%BMtUhkl$n&9;C z#T`YfT2iIG*CLK9dG71mwp9RpC;Dcf`dU=I|8iD@Zhw1o$B<4n!$@*wA_cX#(HISdq2t|c)A27yBKof zMDEY8yelR5TWgWrKOUoMzucd9F1!(WZK4t$upglv;rr6Yb;Ltm&ouK7z!y8$VU!1! zqd{g+8HSFhRTHtfB29F{^ainLL%##XjIgozdcxsCkw=66HE}_&IK-%3-d8(FTsUuu zsE(ipS-=~+VG>$VOFL_}MXG)sp7U_VFQojG(+f%?G6g&rN?=3X5D!gH#_8;cGT4)tmSzAc)-TVb^b#fOv- zacH0rad%9a`n7<9^Jtnf0jFGu<~@o)w1_Ja7W(icxupg2t0{WWXZHJNXdx?5M4-@19X4e6{{l^H2Ck}K!$Ere?)ae2Z0x>% z64m>33KJfM-C14^J|2ZeF<>LFZ2BF*yBYX|=k*b6dDhB%!NsHSjl`M#j+l%0hi?>| zFTwuuSAp!j5lw00#E!!TbvTEXj>LAhNgun~be2MkULRbqf$X0HGzW6Te~B9h0sB6q zyCLjKD93JN82;-^n#Gb%ZHy0t!Z`@vec>EE_>jSXD-Y+eoXTK1Hfs;NlqCDe!BZ2* zC+<4{S2Usq;q3VEI^5BsCWf-7UjE6Y#ac?{LkAyxFk`U3B9mK~_3rf+jo*MsrHxxp z0lqU{rJ)dn2^H$si64p7c?oo4pr6_hoxY5_9?>a`j6xWonZHapNU9$jhJz?Z+C!wg z_!mptozUyory^a6cJ_mn|E^AeX6R#PVYCjbaj6y~RylHnLD$NrH{addu^@Q@aq zvUU;;Q~!O$ zWhVx_?!*d_`GS;1xKa}NskCx1B|nwpV8FXBAwOBpu^#Rbg(jVKRI{_h^hY1IZuEE(ue z$sZ`=F-$QVvdbVvE`LYX<-SnG!6CZ^l?~bL4OPr63VA5XvIaD8&q@Y6j1MvvhD(TO z|GN^9r>QV(lCFWMt-dA}n~Pqn+4eN3M`sL^i4Ytub)7TbxP5|aJ^<^20{ZX5_W{rYSg8^o?nf4C9CLSdeRfvh{(<>jcm+_ zgB2vHRo*Nk40I%LXHn3{I~1`|`7G$))Oic&VNeV@QbHjZ5~&!qtQd2tzQBS} z5{gb&@qLJuRM9A+s2!>=v!EEDNJQj=f=%I4^oG}JMa$9YE>Tag7-O-F;wZ#bpKA%q zxna{Rng)`azKsr)>IHKj%9qWzO1U>G<<3|Va#j7d`L5`!`jUCapkrz8+veT!_a*br z_?d5;cSY^uO*OfH!Mt6r$=TKm(#r-qnE8ofw|uUe-5@mr69O0ROpFkw^9~au+=^u= zQDOwPk)%b4dfEHsOqEawQzhs=_a;lYJxC=|JdNC1&a5|Zoe5& zZiuy|`7)A`WAQVywjsQ@5QX}|09Z!SSUyof@Nx1?F$zLJc(Ju!*3ZFESY7c_?!KWl z9rRaHQ?Z~+cOTkB0CE)cS8_fJvJ}fIM;*Tjh9W|Vg#K}i7;+cp#zIC-ozxy_rd$uQ zFJo(?lzecWGo3w+AU<+B;U+5kapowcZJkSNANvD_ z%r8d%G@hYqJRVq5C~E__6uFOk3j7e1G~-XN7?oi|MB-}V-wy*AQs~)}_aC*7AF=N{I&t(Ac8WM+A9~atOC`=7>l>W} zokCrL=w%bV?7fgnrkT!26cXeP6foid34k!22K(FBM8> zo5g=(@jEOi$JB1?s;{#6T^3(skqA_>0&zfiMlj?#{t5y?NO1C?{rx<%I`4B z*d!p50T8P3vbDSTf0A?47`{*}_k@q7a(70YBlVVzwVyx1tNWM&^IuG8rU2rwIr=+s(tXUhC?b!8x2ec$Ve$R zC-HQ!hUUT23m*g{<}u`Md>hlbduRo^=`z-LkP$$lmzTk)#27tND4A9LC4C*gIw|vO z@LJ1AVcivEg=86F$l5v7Fu=vaL}r|0At_|DDVR(~bK+QFE>Elf0D^)n#PChTaI2}K z5JqStNM)B-0sd+_-bc85DRl(7OsS_Sbq{S1cK1I5&LK(hQiMBHg5}LZu7!FhxOD#><+U`OxC3 zxG2ki1;0M|Ss%$}zB9>R;+m50jJdZHr(}}dOy@5tgo|siKt~qW@Qz&JYI9Iz z=nIlLnd!cDxUcX!jukX8Zi=;!nwY+&7p?06(-1I%s~5Ji5*X|_&F2^8clGf$8~v*4-G;bn0fSKb-|s_rBDZMc`Pr_E+HAf zQZn@=j9_E?(BDNb24GM*Tv{G2H3xTymzrBT$lEPMJO@^w;&aI4Prh=_qO$}UG%b`^ zt{yv1601*X<)M_-ryN^-j>ba#AgfOyEyO2S2MPF!mYpZa$3>AL?$MN!GR*)v!#x{_ zg)z$YV z7Z`u1{vM0ZvG@fRpJ(xL7JXI?Vdz4fk|(8NA|{_>#X+=cv(OsqZJN>lCaKk4JPk&F zeeYs)>y=$r^29|(nr^5;&?aG zdzQG@2n-Ska6JbL_nO%Rox)-{>mKGMQgp8v(!GLuI79ce@glNzSDimiOW``=LbLnG zb|$7s#&A%2=p!a+!joj*pvJ84Bs2>K&c52om`9|`YjJZlQoVG11v0cZ95pDrnVzvr z*LGZ(V$?>9yoQBDvw1>=8_N58~kFpB>X zZ+(MBg4m+}%G$GyLNJ0G^$Nq-?^K5#!l+(B_?L+9<7hJqv9b_0i0F)XvHI{U+Y4Bm z^5@n_Cg5tkAL}pmnvL=NOL%u5o->f+oqc$YeBdEA=X>xRI}4>KKuNtv@SO5Zgy)cN zA}<&5d;o4w4aDt}m%#0kWuW--!GF zokb!tgy}94DkAX|S4c&~25@OAa`PYnP2`5#3JD##js~`VI#M1)NACFE=t%OSj4h#z zv4|xG#+vj|5{CUtf*UulwH+6Z57Ln)cAJ2#P>d7Z$dVZU76x`I>^2cu8NmH@4q=_( zCR!AeCa79wG9u=ExWt_QsU20}oY6rG_Fo!;}@rLB$f@Xqn zf`3d@F{%Cm%35fb7SUUlg7vWEjUx4fo5D#M%#MGIXcUUyvLvRON7#_K6^AblTqL@` zAAv6UhK=!OU^W?M*p_TEku8q#BpQE!J+TWBsqCMmzWPBZXZJ<0~Me*4IHh6-h6^HvIn&xAZ2D`_!Sbg zj8OFU?{O}Y*00cCfSnn&zkpjAq_6r<;~$j>sB(Z5gFv0_`GJHMsS! zi~R}t2;(PsU!)EsSb`^$sk6m(C&Bj;oFlsp#7htS$bSg)j3i3;{A4E-swLPcx)X2g z+-PAp$9P{Bi8DZq`SKi>EYbHB`ohqXgSZrOb>t36;>v5GuWi8tem6B6_dK`etM###@72U=T7+^;fz{W_P7@HIkfGm}^1ZJdmj zmq^G*Q7_;~X+)tH<9CVBJw*38UXv)jI8-m#q8kuphyKN;`$b}OxK2tidIvm#J(!%@ zTrx6h^tIE6yY%;c1;76NCeQMYm2SRfgJ~VuBSJ}$14lu0sw2*;7%ED+7 zn|-0#521L!mI-I%6}_2r_?vFl3Uy#h@AQznzfe1l}Hboff?Q)dzMc{q3cNX~Yd-)dv|2oF2kZ zTzfBev1Wwns;4?N8s?&%xRUKHn8X~eXfRQ5cAfWH2SREqWXk^$ZoQcAL5WQGG~|XH zi65<4LRn?wo65QXAfnK6Bx-u=8*G6k$Yf{~GWGtTf-d_|TjR_aj~+#F16q|`aCYZb z@9r0_LUc&PsX;s9MC{}XXfG%^{z|)njU$U#ae;u{Anz|NWQ6&IrBH4R3No}oq3if( ze4Wizc#CwF{frq*(u<2J1_Nh#gxC-mOIu~{Vj%a%3?^wOlf9{ecg3piCvnopU<_n$ zdXt#Hb4pe(Y9WzXq_GMr&ecRF1f=w~x1@4MZW+#X=B4G+OEei4Rb`+zz9;v9)&ui_?+N#$ah) zgv}hg79-K}qmU9Mq6ATB@r!8*8wWg5w>12T7(Lc#A#Gf`n{b*m zeZ7HDynYff?`^w*NN_RMhZ_LKuf-T*9gH*=*U<0c=q9g1xV!46U#0N|Qns!fWtI)Q zt5ef+sV5}s#U#W&elM*E16Q_j3?zU{3&7XH(=(RFBtlvbkDR|j!_&jKa~&C|sY1LV zo%W7}c)ZUJF=e0Z!!(wGWnVU{>EmuOpmO*%r%NP`35a+SiY^wN-vJ>NIS*VyG>SpD z!o592AK3~_Yll(bVPIphyZ_pN5cF`Twcp|!zsf@1_qC`4vYZcr^%B)S&{DBw7Vgt9 z+Fd`khwSkkXaTl~8^C5#F^#5CzG&?%{&^%*;{4?v7tR0{)c1yrwWx$Q#VqJEI2pi# z3aeva`1HU^B0*t61NLMK{kQu=W<-4`iF*te(#MupYvn<(q$tWO+S}Dq2V<5rD9X#o zZ^)8PV)akBv3)O3ilI||eBO-=Fna2fxAdAGuEl}389Poh(qHNwb7Qc&6cN;b zMP8g07&f4M{nSqVFW@PXcL-ghY#Uipg_>;_$`$n_qGvY_x;h{>JB*DJIwoW-k)6Md zHw`3W+el)1UicxBu{%&B@zH^=C^ucPu|+E6AWmdiO%$u)ZANGr85sR1a)7Noy z5PpG+<{qA7k+Cdp4)1-(e5XFLE|{kIs;-Y`2VceVW$R_rtiEdQkB_)i`63rC>t)mW zIBfH!xXl9I`#ybj>&x;D+|+)-X}0V28g>mwvzAFBK#Xxk$ui&qEI-WAR zSbsEc;y2GN;K2y2bu2fT2F{h=>QxlM zjVDe{+;_5mk|i09986{=z3vc zrS49rri!zKWpMbk&k|NgacVvZAE#l7$)?WE&VG$c`I>Crso5zhh_MmMk^dhH9 zytA3-e*Ns7O X{*B%n9pye!e)F(dHb-x@e*6CcWE!x7 literal 0 HcmV?d00001 diff --git a/database/__pycache__/async_mysql_v2.cpython-310.pyc b/database/__pycache__/async_mysql_v2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eda566ebe369f03473a9788317e20671b96756a1 GIT binary patch literal 9102 zcmd^F&vP8db)H|lJG)pc01~1|nPFNiNyOR|Kq|5;1<{fq5Hu-KAWcA42%8xXW_p1E zc4rnnvjlcCD?uTz%8!eyOgmLTl8X+h+>)G=|6)$5N>yAsrJO@tktsUg>)9W(04yuF zRAo`q(=*eryQklK-}_z5k5h|l{FQL)<{v0 zl{Ki?HE=fEoSVlv=N4QO=R7aCMYr@=d89(AV{RGuite~u!MRkNC{7lqil^M)cTc<1 zkJYuY;&ZrS?ipTopXcN5Sw3FNyXV{&9_Q97nP=bN6MXWVGH7GE=iT2!n<@9gic)I6=L$?2hm@Jm0R@+pNKDKO&L9$Cc+l z47NE7qd>qQnj&zx+Y+z{+jp7Zk?{CK7|$keFgqL!{O-z~wo3en9@%M-l8{fLI~Le{X`%nu?)BAm4Ybd_nuFa^B(?4YOlrY&q3d=S*F z(ChGdIa$BeSm%NTHRc4q&z;B%e7U#{+w&tFj68#cj`d`Vjc}eV_-%$-izX%oo6ycI zTgJrjV(u}i?cG%Fu2dq{XE-iZYu-jn*ffjOKR3DH!#+MIY+UY`(%x9iR;yZ)Zwg-X z_L7R{JN1^!7Xv@ydr{R)rmUfHS{4nPoF2KA&Lw$aUyk1L+!x0QQv3xq0&lK?-q`z35+YB0f&V?b?gC-YqciYaU zEgE-4umvTBb9U3S?2Qe}4IH`jbFIj$ht|W_=K26PXOm}Ijdr+G$Nkx6J25TG^S#Kj z{upC~Wfb|TyqZ@lI8N!E)5GPO?YDGPwCew;PRAmSklxZ&;GpWVobzX@n8GbtPm43S zObV9e^15a1kqjY(p>|G<3?v_qj6=;x)Bi#;F^W&r&WnHwYu5%6xX<^07%RaQCk5XMaA}aRi&${N@pTg4oWe4Lu7+I zwYiU4SN%w(wrOow-30)3{DBiKg@_*T`As-Q#&8KZ;HP$kz$2aI1c!3qjnbxv<~$Fu zX$gNG-r{GhJTdz^Of+}BswV~5B)Kdq_MWU5^&n`b>s}obV_2d@ZwAf8w7o!TJQ+*p zBP~^sszrrb6{J;oK~mY^kp=VeBa8YH6ZEvWwCC_9d7qfXbybrpO^SSQo2)ad-QFVn z32uiMP-y#F;gptF4g8_}(lDlLiH>NVtc$cSHGdJ6Pf~5GEk%&V z)I$YYRd2!!T@_lwQPnzsH_*>2uv?-K_d@_Q0GMDft118|03#$A9Y`(OK5+)p5CXzF z2Xu3WwE-S-s0xR$6WG3KeImII^l!>Flkc zh5ZFuNsrU^nb|p?@t+cxT`fXTj=&lpX<|B75A?{0 zwMWt;b_}|XHS!Lslb3h?Yq(y4kHrDpOA19n&S@c*7ZHvq9trGjdd?=u0eo)L zrf5Do0jC}yY|_luIl?XT5pt+i(4Wn4glaA+c_ADyvV8|G=puSI?0QnT-mB#8W|RAF zlB?ANJA%9Fm_SlWCu33F@FLzgs^l@tl1-=u(Y%ll&nTMed9@7Z>72Sa$iwu& zWX~yosqGN6Qw6!ZTo=S~ZT3oHxIA=@E>!6nkt`=@G~fJ3>W?~~Rys48^{Mg5*|do6 zT&o9gG{G-i`EN&OXk5Q>mT)k2Dg zd8Z}9KqM3B(Mm1S!s9TRh(OFTcanViG$}Yiz18qT@oN8#)3=l%vN)LjXY_tApwRXW zLo*DWBC}DN{MAcpNv+^o`(Ef9z_n&+yG%00{lv z$8o%6tgk48wyW8;^rWiUb1;wr>Z}G5p??=|hfeyuy9qAK!=WqpDR7_hTfW`s^I+5n zwg}9QCI%^PH2`tLDDf4Yru>%hh-xXufTX7h$VvTY`pTl2uy=dO1EdkfC_R0QxDF{=WHa@;AoRng1}RI_ENGJSzJ+p~5e4;PG#XGZa9Q?{U6a2qCFD%B1n9qM$*Q+1z0yWHM+$}jrB*sxCh_vmO5pSLX9lh z)jsXnau<}(p9U$^+px3l7)YTRk9CWZ2&&z&kH&m#me84KV2lkCkdW`Fia#D1z(;JZ z^esLHW=;cRcgCZ9Y(7$59or!5uHKWm>l!{bSJL^#g|mvMxcQ$O-Ey24KaR_Bu?94Z z%Q#F|N3Kmr;aukG02#hpJk;FMKyuLXz&xaIKKUc9JKn9t<8k?*c&I#7#h=HOxa^Lh zpYdzT&c~6O&P}e!>Fz{4fms^jv)GI$&_=d&%eXs!R{5FoSD@EBKN-##&nwpy%=e$N z^Q}(UPXIha4~5M)I5t#VvU<`g;xP9qH)TGD-S+R=$9?Q*?oFmI{g({iX2a||^-iJ>Bp@CYd^2FKO=)bxA$m7qe_#ZbbRWhVAWbThLZeorwGba<)2~ zq0Of1aJ2vX5R#_(020v#fo%Cv%A$}t%(7v&A9;X{M(PvkWEqQ;)26j`tX5Hf=9kpgV*vLsSnnF2r7qU%Iz(xxU5YuS{u2*ZzDaaWj z-^9)dh^Mqonyj2*0E@$792+-wO zDb;ufC@B`HAYAR7yIbe9*#x~sSUG09NG{pT@=~>P_T)!PRcu@2L4TuGhzu0lsrCkV za8LE%PuNivw`hQOQ6yThy@9j*iQ70Onh3=WRK$BIs-?srH6%G<``80rq6aTgaS=t5 z3+tRW6TQy;L`RC6$qGA4DF+Ice3HV|HPpV>U_dpnMFh zP{h}SE(OPY=Z_FrM~UP+-9lU#A(Ah3&DbO&*$(Zi8mmXrWTT<9;b^L zV)5Uq`?%Y?zXs{KYudkuCLiu)dhMBRd@2!2a2Q zFN?Qf`zcp0@I6`qE~&C`={`e|l;qy?k`RHAdt?&3s-+Z|9BL-pl2;|;NbN(*uGO$* z%U$t9x@Q1Vn&LV&Bu*}&<|ufK^~Sbf7?c8qucIZH^dy0Wp=uiO=}G(n5ca?M<;-u4 z{mBA>hPhwS)3f%hk~2p)5i*@hTy}Ekwf`e7J#hNvlbv2VJPf?hxqr;*7p045v&Waa z#ke?{JtChZw=WLd9(xYr8ld6py8Z1@Xa0)J(;jU(aQi=n6PGe>Uy@E->K1x#U;3yd z-5%$j+y87tLXS_9pp}slaC<}i4LN;WkZw=zpSpdb9dd%N$wVbAet`|_ykFCvZg z@P$;GcrB>cgI%Hye9xxumGc8E`Thq>_wLWLi?^0nmhP>xTg$6=SQB4G0++vUhcIO} z)4ROP952F0RSN37%6_nL`-7zwHuG|LnWabdjf;bB@j-TBp8are&G4JKboJ_$D_5>w zy38&u^IdkG*Ec=7NGsLLM_PXyjnmdQ+jtXvfpw|%DyW#hVf*%_V-nvPNGu|S@NL=q zs@(u5U$))WrRrdW2ZLIVod54Jk|<0Hs#GJ=7`E2K$cq2~PzSCFT~fde@qKWFt);;H z{~cxp2ZD$iir)gLeNZE-KLV;^lVmBSPtkNNgQmojhL9>ex+EYKTU2~Cj+Xn7M`J$7 z0O`3bkP?*sP9S|g!>+QuE?13|+kTIxa}&jls;avXP>T?B+Bd5LWc<^#yWj!+@{Bb<4G=R0SrWg zZVW#s$Qo@jiad%R;RtETOvTj97fjuJPAipEvtpjpW{7}lYU#hV+nQ=T)imvyrfXkn zM(L^c8t(j>?&Rei?WvZl{95)%Z%JQ@Evq^$KlKF;UQ!R+VC+ph+7#cSu^AOr6!=(+ zP;G~a)^r8^YXGtZ?3(t!UCySyC9tQ^O9xEP;q)P^k$zMyM#6SDvHB#B9ewzre0ez) w;gi$}ub!CK(&c{>7NCSPC=A7Xktj4)fDZhX(*vZtazXi6`PjUj*NkWX1LU`xh5!Hn literal 0 HcmV?d00001 diff --git a/database/async_firebase.py b/database/async_firebase.py new file mode 100644 index 0000000..a8065a1 --- /dev/null +++ b/database/async_firebase.py @@ -0,0 +1,504 @@ +""" + + AUTHOR: + + Khushal P Soonderji + Bhushan Thakkar + + DATE: + + Friday, 19th April, 2024 + + OBJECTIVE: + + To provide an easy interface to work with Firebase. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +# --- +import sys +sys.path.append(".") +sys.path.append("..") + +# For async operations: +# --- +import asyncio + +# For system-level activities: +# --- +import os + +# My async utils: +# --- +import async_json_utils + +# Firebase: +# --- +import firebase_admin +import firebase_admin.firestore_async +import firebase_admin.auth +import firebase_admin.db + +# For debugging and logging: +# --- +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncFirebase: + + def __init__( + self, + credentials_json_path, + app_name, + max_connections = 5, + debug = True + ): + + # Debugging print: + # --- + self.__printer = IceCreamDebugger(prefix = f"FBase ({app_name[:8]}) | ", includeContext = True) + if not debug: self.__printer.disable() + alert = f"Firebase session starting with max. {max_connections} connections." + self.__printer(alert) + + # Initialize the instance: + # --- + self.__semaphore = asyncio.Semaphore(max_connections) + credentials = firebase_admin.credentials.Certificate(credentials_json_path) + self.__firebase_app = firebase_admin.initialize_app(credentials, name = app_name) + self.__firestore = firebase_admin.firestore_async.client(self.__firebase_app) + + def __del__(self): + firebase_admin.delete_app(self.__firebase_app) + self.__firestore.close() + alert = "Firebase session ended." + self.__printer(alert) + + def __user_to_json(self, firebase_user): + + user_json = { + "uid": firebase_user.uid, + "email": firebase_user.email, + "emailVerified": firebase_user.email_verified, + "displayName": firebase_user.display_name, + "phoneNo": firebase_user.phone_number, + "photoUrl": firebase_user.photo_url, + "customClaims": firebase_user.custom_claims, + "disabled": firebase_user.disabled, + "providerId": firebase_user.provider_id, + "providerData": firebase_user.provider_data, + "tenantId": firebase_user.tenant_id + } + + return user_json + + async def create_custom_token(self, uid): + + async with self.__semaphore: + + try: + custom_token = firebase_admin.auth.create_custom_token( + uid = uid, + app = self.__firebase_app + ).decode("utf-8") + return custom_token + + except Exception as exception: + self.__printer(exception) + return None + + async def authenticate_token(self, token): + + """ + To authenticate the given session token. + :param token: The session token generated by Firebase on a successful sign-in. + :return: Either the retrieved user information or a blank dictionary. + """ + + try: + user_info = firebase_admin.auth.verify_id_token(token, app = self.__firebase_app) + return user_info + except Exception as exception: + self.__printer(exception) + return {} + + async def create_user( + self, + uid, + display_name = None, + password = None, + email = None, + phone_number = None + ): + + """ + Create a new user. + :param uid: The id to identify the user by. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + firebase_admin.auth.create_user( + uid = uid, + app = self.__firebase_app + ) + return True + + except firebase_admin.auth.UidAlreadyExistsError as excp: + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def delete_user(self, uid): + + async with self.__semaphore: + + try: + firebase_admin.auth.delete_user( + uid = uid, + app = self.__firebase_app + ) + return True + + except firebase_admin.auth.UserNotFoundError as excp: + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def get_user(self, uid): + + async with self.__semaphore: + + try: + user = firebase_admin.auth.get_user( + uid = uid, + app = self.__firebase_app + ) + return self.__user_to_json(user) + + except Exception as exception: + self.__printer(exception) + return None + + async def list_users(self, users_per_page, page_token = None): + + """ + To get a list of users. Firebase has a limit of 1000 per call of this API. So we use page-tokens to fetch next + pages of users. + :param users_per_page: How many users you want to list in this API call (Max. 1,000). + :param page_token: To be used in case of pagination. + :return: The list of users and the page token to be used for the next call. Will be None in case of failure. + """ + + async with self.__semaphore: + + try: + users = firebase_admin.auth.list_users( + page_token = page_token, + max_results = min(users_per_page, 1000), + app = self.__firebase_app + ) + next_page_token = users.next_page_token if users.has_next_page else None + users = users.users + users_json = [self.__user_to_json(user) for user in users] + return users_json, next_page_token + + except Exception as exception: + self.__printer(exception) + return None, None + + async def create_document(self, collection_path, document_name, document_data = None): + + """ + To create a new document with the specified data in an existing collection. + :param collection_path: The path of the collection (can be a sub-collection). + :param document_name: The name of the document you want to create. + :param document_data: The data that you want to populate in the document. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.collection( + collection_path + ).add( + document_id = document_name, + document_data = document_data or {} + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def delete_document(self, path): + + """ + To delete a document. + :param path: The path of the document. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).delete() + return True + + except Exception as excp: + print("FIRESTORE DOCUMENT DELETION EXCEPTION:", excp) + return False + + async def set_document(self, path, data): + + """ + To overwrite the data in a document. + :param path: The path of the document. + :param data: The data you want to update as a dictionary. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).set(data) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def update_document(self, path, data): + + """ + To update the data in a document. Provide only the fields that you want to update. + :param path: The path of the document. + :param data: The data you want to update as a dictionary. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).update(data) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def get_document(self, path): + + """ + To get the data in a document. + :param path: The path of the document. + :return: The dictionary of data as found in the path specified or None if the operation failed. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).get() + return snapshot.to_dict() + + except Exception as exception: + self.__printer(exception) + return None + + async def get_document_fields(self, path, fields): + + """ + To get only specific fields (keys) in a document. Like how projections are used in Mongo. + :param path: The path of the document. + :param fields: The list of fields (keys) of the document that you want. + :return: The dictionary of data as found in the path specified. + """ + + async with self.__semaphore: + + try: + if type(fields) is not list: fields = [fields] + snapshot = await self.__firestore.document(path).get(fields) + return snapshot.to_dict() + + except Exception as exception: + self.__printer(exception) + return None + + async def create_collection(self, collection_name, document_name, document_data): + + """ + Creates a new collection in the root of the database. Note that Firebase doesn't allow creating new empty + collections, so we must add one first document in it. + :param collection_name: The name of the collection you want to create. + :param document_name: The name of the first document you want to put in the collection. + :param document_data: The data that you want to put in the first document of the new collection. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.collection( + collection_name + ).add( + document_id = document_name, + document_data = document_data + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def create_sub_collection( + self, + document_path, + sub_collection_name, + sub_document_name = None, + sub_document_data = None + ): + + """ + Firebase allows you to create collections inside documents. This method is built for that. Note that Firebase + doesn't allow creating new empty collections, so we must add one first document in it. + :param document_path: The path of the document in which you want to create a new collection. + :param sub_collection_name: The name of the collection you want to create. + :param sub_document_name: The name of the first document you want to put in the collection. + :param sub_document_data: The data that you want to put in the first document of the new collection. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document( + document_path + ).collection( + sub_collection_name + ).add( + document_id = sub_document_name, + document_data = sub_document_data or {} + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def get_collection(self, path): + + """ + Get a whole collection's data. + :param path: The path of the collection. + :return: The collection. + """ + + async with self.__semaphore: + + try: + snapshots = await self.__firestore.collection(path).get() + return {snapshot.id: snapshot.to_dict() for snapshot in snapshots} + + except Exception as exception: + self.__printer(exception) + return None + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import time + from shared import constants + + user_data = { + "firebaseUid": None, + "roles": "ClientAdmin", + "sessionToken": "siddhesh_20240419" + } + + async def test(): + my_firebase = AsyncFirebase( + f"{constants.PROJECT_DIRECTORY}/utils/cred/firebase_certs.json", + "myFire", + max_connections = 5 + ) + print("\n\n---\n\n") + results = await my_firebase.get_document("testCollection/testDoc") + print(async_json_utils.to_json_string(results)) + print("\n\n---\n\n") + results = await my_firebase.get_document_fields("testCollection/testDoc", ["sampleMap"]) + print(async_json_utils.to_json_string(results)) + # await my_firebase.create_sub_collection( + # "myDeepCollection/deepDocId", + # "subCollection3", + # "subDoc", + # {"sub_key": "sub_val"} + # ) + # await my_firebase.create_collection( + # "rootCollection", + # "subDoc", + # {"sub_key": "sub_val"} + # ) + # await my_firebase.delete_document("activeSessions/9d402f1502dfd55a4326fa7fc8e6cb7d") + # print(async_json_utils.to_json_string(await my_firebase.get_collection("myDeepCollection"))) + + start_time = time.time() + asyncio.run(test()) + print(f"FINISHED IN {time.time() - start_time} SECONDS.") diff --git a/database/async_mongo.py b/database/async_mongo.py new file mode 100644 index 0000000..c052fac --- /dev/null +++ b/database/async_mongo.py @@ -0,0 +1,843 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 28th May, 2024 + + OBJECTIVE: + + To have one central place from where all async database connectivity happens. + + REFERENCES: + + 01. https://motor.readthedocs.io/en/stable/ + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For async behaviour: +import asyncio + +# For datetime handling: +import pytz +import datetime + +# MongoDB: +from motor.motor_asyncio import AsyncIOMotorClient +from bson.objectid import ObjectId +from bson.json_util import dumps, loads + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class MongoFindException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo find operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoInsertException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo insert operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoUpdateException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo update operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoReplaceException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo replace operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoDeleteException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo delete operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongo: + + __db = None + __db_name = None + __client = None + + def __init__( + self, + database_name = "myDb", + max_connections = 5, + debug = True, + debug_only_errors = True, + host = "localhost", + port = 27017, + connection_string = None + ): + + # Basic config: + self.__max_connections = max_connections + self.__db_name = database_name + self.__host = host, + self.__port = port + self.__connection_string = connection_string + + # For debugging: + self.__debug_only_errors = debug_only_errors + self.__printer = IceCreamDebugger(prefix = f"Mongo ({self.__db_name}) | ", includeContext = True) + if not debug: self.__printer.disable() + + # Rate/access control: + self.__exclusive_semaphore = asyncio.Semaphore(1) + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + @staticmethod + def generate_id(): + + """ + Just generates an '_id' in MongoDB style. + :return: The '_id' in MongoDB style. + """ + + return str(ObjectId()) + + async def connect(self): + try: + if self.__connection_string is None: + self.__client = AsyncIOMotorClient( + self.__host, + self.__port, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + else: + self.__client = AsyncIOMotorClient( + self.__connection_string, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + self.__db = self.__client.get_database(self.__db_name) + if not self.__debug_only_errors: + server_info = await self.__client.server_info() + self.__printer(server_info) + except Exception as exception: self.__printer(exception) + + async def ensure_connection(self): + if self.__db is None: + async with self.__exclusive_semaphore: + await self.connect() + + @property + async def client(self): + await self.ensure_connection() + return self.__client + + @staticmethod + def dict_to_dot_notation(input_dict, parent_key = "", separator = "."): + items = [] + for k, v in input_dict.items(): + new_key = f"{parent_key}{separator}{k}" if parent_key else k + if isinstance(v, dict) and v: + items.extend(AsyncMongo.dict_to_dot_notation(v, new_key, separator = separator).items()) + else: + items.append((new_key, v)) + return dict(items) + + @staticmethod + def normalize_date_time(document): + + """ + MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it. + This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was + timezone naive, UTC timezone will be applied to it without changing the time value. + :param document: The document that you want to normalize the date-time in. + :return: The document with normalized datetime. + """ + + if isinstance(document, datetime.datetime): + utc_tz = pytz.timezone("UTC") + if document.tzinfo is None: document = utc_tz.localize(document) + else: document = document.astimezone(utc_tz) + + if type(document) is list: + document = [AsyncMongo.normalize_date_time(item) for item in document] + + if type(document) is dict: + document = { + AsyncMongo.normalize_date_time(k): AsyncMongo.normalize_date_time(v) + for k, v in document.items() + } + + return document + + @staticmethod + def __from_json_string(json_data): + return loads(json_data) + + @staticmethod + def __to_json_string(python_data, indent = 4, default = None): + return dumps(python_data, indent = indent, default = default) + + async def list_indexes( + self, + collection, + session = None, + raise_exception = False + ): + + """ + Lists out the indexes of a collection. + :param collection: The collection whose indexes you want to list out, + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The list of indexes or None if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + indexes = None + + # Try to list the indexes: + try: + responses = await self.__db[collection].list_indexes(session = session).to_list(None) + indexes = [{key: value for key, value in response.items()} for response in responses] + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not indexes: raise MongoException(hint = "list indexes") + return indexes + + async def create_index( + self, + collection, + keys, + options = None, + session = None, + raise_exception = False + ): + + """ + Creates an index on a collection. + :param collection: The collection to create the index on. + :param keys: The keys (and sorting) to implement the index on. + :param options: Additional config. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the execution. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the insertion: + try: + options = options or {} + keys = [(k, v) for k, v in keys.items()] + response = await self.__db[collection].create_index(keys, session = session, **options) + if response: success = True + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not success: raise MongoException(hint = "create index") + return success + + async def insert_one( + self, + collection, + document, + session = None, + raise_exception = False + ): + + """ + Insert data into a collection. + :param collection: The collection you want to feed the data into. + :param document: The data to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_id = None + + # Try to make the insertion: + try: + response = await self.__db[collection].insert_one(document.copy(), session = session) + inserted_id = response.inserted_id + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not inserted_id: raise MongoInsertException(hint = f"{collection}") + return inserted_id + + async def insert_many( + self, + collection, + documents, + session = None, + raise_exception = False + ): + + """ + Insert a lot of data into a collection. + :param collection: The collection you want to feed the data into. + :param documents: The data to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_ids = [] + + # Try to make the insertion: + try: + response = await self.__db[collection].insert_many(documents, session = session) + inserted_ids = response.inserted_ids + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not inserted_ids: raise MongoInsertException(hint = f"{collection}") + return inserted_ids + + async def update_one( + self, + collection, + filter_json = None, + update_json = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update one document. + :param collection: The collection you want to update. + :param filter_json: The selection criteria to locate the document to update. + :param update_json: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the insertion: + try: + response = await self.__db[collection].update_one( + filter_json, + update_json, + upsert = upsert, + session = session + ) + success = False if response.modified_count == 0 else True + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not success: raise MongoUpdateException(hint = f"{collection}") + return success + + async def update_many( + self, + collection, + filter_json = None, + update_json = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update many documents. + :param collection: The collection you want to update. + :param filter_json: The selection criteria to locate the document to update. + :param update_json: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + update_count = 0 + + # Try to make the insertion: + try: + response = await self.__db[collection].update_many( + filter_json, + update_json, + upsert = upsert, + session = session + ) + update_count = response.modified_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not update_count: raise MongoUpdateException(hint = f"{collection}") + return update_count + + async def count( + self, + collection, + filter_json = None + ): + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + if filter_json is None: filter_json = {} + count = await self.__db[collection].count_documents(filter_json) + except Exception as exception: self.__printer(exception) + + # Done here: + return count + + async def bulk_write( + self, + collection, + requests, + session = None, + raise_exception = False + ): + + """ + To perform various individual operations in one go. You will have to import individual actions like "UpdateOne" + and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method. + :param collection: The collection you want to run the requests on. + :param requests: The array of requests (operations) to be performed. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + response = await self.__db[collection].bulk_write(requests, session = session) + count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not count: raise MongoException(hint = "bulk write") + return count + + async def find_many( + self, + collection, + filter_json, + projections = None, + skip = 0, + limit = None, + sort = None, + session = None, + as_json_string = False, + indent = 4, + raise_exception = False + ): + + """ + Finds one or more records that match the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter_json: The filter criteria. + :param projections: What parts of the matching data you want to fetch. + :param skip: The no. of records to skip before picking next ones. Needed for pagination. + :param limit: The max. no. of records you want to fetch. + :param sort: The sorting rules to apply. + :param session: The session if you need to do this in a transaction. + :param as_json_string: Whether you want it as a JSON string or a Python dict/list. + :param indent: The indentation to use if you want it as a JSON string. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # Assume defaults: + if sort is None: sort = {"_id": -1} + if limit is None: limit = 10 + + # try to query the data: + try: + results = await self.__db[collection].find( + filter_json, + projections, + session = session + ).sort(sort).skip(skip).limit(limit).to_list(None) + if as_json_string: results = self.__to_json_string(results, indent = indent, default = str) + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not results: raise MongoFindException(hint = f"{collection}") + return results + + async def find_one( + self, + collection, + filter_json, + projections = None, + session = None, + as_json_string = False, + indent = 4, + raise_exception = False + ): + + """ + Finds one record that matches the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter_json: The filter criteria. + :param projections: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param as_json_string: Whether you want it as a JSON string or a Python dict/list. + :param indent: The indentation to use if you want it as a JSON string. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to query the data: + try: + results = await self.__db[collection].find_one(filter_json, projections, session = session) + if as_json_string: results = self.__to_json_string(results, indent = indent, default = str) + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not results: raise MongoFindException(hint = f"{collection}") + return results + + async def replace_one( + self, + collection, + filter_json, + replacement_json, + upsert = False, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter_json: The filter criteria. + :param replacement_json: The data to put in place of the existing document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # try to query the data: + try: + result = await self.__db[collection].replace_one( + filter_json, + replacement_json, + upsert = upsert, + session = session + ) + if result.modified_count or result.upserted_id: success = True + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not success: raise MongoReplaceException(hint = f"{collection}") + return success + + async def delete_one( + self, + collection, + filter_json, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter_json: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + result = await self.__db[collection].delete_one(filter_json, session = session) + count = result.deleted_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}") + return count + + async def delete_many( + self, + collection, + filter_json, + session = None, + raise_exception = False + ): + + """ + To delete many documents from a collection. + WARNING: sending {} in the filter would mean deleting ALL the documents. + :param collection: The collection from which you want to delete many records. + :param filter_json: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + result = await self.__db[collection].delete_many(filter_json, session = session) + count = result.deleted_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}") + return count + + async def aggregate( + self, + collection, + pipeline, + limit = None, + raise_exception = False + ): + + """ + Perform an advance query on the data. + :param collection: The collection to perform the query on. + :param pipeline: The pipeline of actions to take. Must be a list. + :param limit: The max. no. of records to retrieve. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to perform the aggregation action: + try: results = await self.__db[collection].aggregate(pipeline).to_list(limit) + except Exception as exception: self.__printer(exception) + + # Check results and return: + if raise_exception and not results: raise MongoException(hint = "aggregation") + return results + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/database/async_mongo_storage.py b/database/async_mongo_storage.py new file mode 100644 index 0000000..94df2d8 --- /dev/null +++ b/database/async_mongo_storage.py @@ -0,0 +1,358 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 28th May, 2024 + + OBJECTIVE: + + To have one central place from where all async database connectivity happens. + + REFERENCES: + + 01. https://motor.readthedocs.io/en/stable/ + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +# --- +import sys +sys.path.append(".") +sys.path.append("..") + +# For system-level activity: +import io + +# For async behaviour: +import asyncio + +# MongoDB for File Storage: +from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket +from bson.objectid import ObjectId + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongoStorage: + + __db = None + __db_name = None + __client = None + __store = None + + def __init__( + self, + connection_string = None, + max_connections = 5, + host_name = "localhost", + port = 27017, + database_name = "fileStore", + debug = True, + debug_prefix = "GridFS (M) | ", + debug_only_errors = True + ): + + # Database Initialization: + self.__host_name = host_name + self.__port = port + self.__db_name = database_name + self.__connection_string = connection_string + self.__max_connections = max_connections + + # Debugging: + self.__debug_only_errors = debug_only_errors + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + @staticmethod + def generate_id(): + + """ + Just generates an '_id' in MongoDB style. + :return: The '_id' in MongoDB style. + """ + + return str(ObjectId()) + + async def connect(self): + + """ + Initialize the database connection. + :return: Nothing. + """ + + if self.__connection_string is None: + self.__client = AsyncIOMotorClient( + self.__host_name, + self.__port, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + else: + self.__client = AsyncIOMotorClient( + self.__connection_string, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + self.__db = self.__client.get_database(self.__db_name) + self.__store = AsyncIOMotorGridFSBucket(self.__db) + + @property + def fs(self): + + """ + To access the features that have not been wrapped in this reportlab directly. + This could include things like streaming files chunk-by-chunk. + :return: The file-store instance. + """ + + return self.__store + + async def write_from_memory(self, file_name, file_data, metadata_json = None): + + """ + Save a file (from RAM) to Mongo. Suitable for smaller files (a few MBs max.). + :param file_name: The name of the file. + :param file_data: The data of the file (held in RAM). + :param metadata_json: A JSON of metadata information that can later be used to search files (RECOMMENDED). + :return: The file's id as a string (if it gets saved) or None. + """ + + if self.__store is None: await self.connect() + + file_data.seek(0) + file_size = file_data.__sizeof__() + + file_id = None + try: file_id = await self.__store.upload_from_stream(file_name, file_data, metadata = metadata_json) + except Exception as exception: self.__printer(exception, file_name, file_size, file_id) + if not self.__debug_only_errors: self.__printer(file_name, file_size, file_id) + return str(file_id) + + async def read_to_memory(self, file_identifier, by_id = True): + + """ + To retrieve a file (in RAM) based on the provided identifier. + Suitable for smaller files (a few MBs max.). + :param file_identifier: Either the name or the "_id" of the file. + :param by_id: Set to True if you are fetching by the "_id" of the file. + :return: Either the file (in RAM) or None. + """ + + if self.__store is None: await self.connect() + + file_data = None + + try: + if by_id: grid_out = await self.__store.open_download_stream(ObjectId(file_identifier)) + else: grid_out = await self.__store.open_download_stream_by_name(file_identifier) + file_data = io.BytesIO(await grid_out.read()) + file_data.seek(0) + + except Exception as exception: + file_data = None + self.__printer(exception, file_identifier, by_id) + + if not self.__debug_only_errors: self.__printer(file_identifier, by_id) + return file_data + + async def delete_file_by_id(self, file_id): + + """ + Tries to delete one file by the id. + :param file_id: The id of the file in the database. + :return: True or False based on the success of the operation. + """ + + if self.__store is None: await self.connect() + + deleted = False + + try: + response = await self.__store.delete(file_id = ObjectId(file_id)) + deleted = True + except Exception as exception: + self.__printer(exception, file_id, deleted) + + return deleted + + def __format_metadata_json(self, metadata_json): + + """ + NOTE: ONLY USE WHEN SEARCHING FILES BY METADATA. + MongoDB expects dot-notation while searching for files by the metadata. We are making a function to search + files assuming that the conditions are to be applied to the metadata itself. So this function add the + dot-notation to the right places to conduct a successful search. + :param metadata_json: The JSON to format. + :return: The formatted JSON that has the right dot-notation. + """ + + formatted_metadata_json = {} + + for key, value in metadata_json.items(): + if not key.startswith("$"): key = f"metadata.{key}" + else: + if type(value) is dict: value = self.__format_metadata_json(value) + if type(value) is list: value = [self.__format_metadata_json(item) for item in value] + formatted_metadata_json[key] = value + + return formatted_metadata_json + + async def find_file_by_metadata(self, metadata_json, limit = None, skip = None, sort = None): + + """ + This method only lists the files that match the criteria mentioned in the metadata JSON. + :param metadata_json: The JSON that describes what you want to find. + :param limit: Max. no. of records to retrieve. + :param skip: No. of starting results to skip. Useful for pagination. + :param sort: The sorting conditions to follow. + :return: A list of (JSONs of) files that match the conditions. The list can be empty. + """ + + if self.__store is None: await self.connect() + + files_list = [] + + try: + limit = limit or 10 + skip = skip or 0 + sort = {"_id": -1} if not isinstance(sort, dict) else sort + formatted_metadata_json = self.__format_metadata_json(metadata_json) + return await self.__store.find( + formatted_metadata_json + ).sort(sort).skip(skip).limit(limit).to_list(None) + + except Exception as exception: self.__printer(exception, metadata_json, len(files_list)) + + if not self.__debug_only_errors: self.__printer(metadata_json, len(files_list)) + return files_list + + async def find_file_by_id(self, file_id): + + """ + This method allows you to get the file's info from the id of the file. + :param file_id: The id that was assigned by Mongo during upload. + :return: The file's info or None if the file doesn't exist. + """ + + if self.__store is None: await self.connect() + + file_info = None + + try: + formatted_metadata_json = {"_id": ObjectId(file_id)} + file_info = (await self.__store.find(formatted_metadata_json).to_list(1))[0] + + except Exception as exception: self.__printer(exception, file_id, file_info) + + return file_info + + async def get_file_name(self, file_id): + + """ + Returns the file name if the id of the file is known. + :param file_id: The id of the file as assigned by MongoDB when the file was stored. + :return: The file's name (if it exists), or None. + """ + + # Ensure that we are connected: + if self.__store is None: await self.connect() + + # Ensure that the input given is of 'ObjectId' type: + if type(file_id) is not ObjectId: file_id = ObjectId(str(file_id)) + + # Fetch and return the file name: + files_list = await self.__store.find( + {"_id": file_id}, + {"filename": True} + ).sort({"_id": -1}).limit(1).to_list(None) + try: file_name = files_list[0]["filename"] + except: file_name = None + return file_name + + async def aggregate( + self, + collection, + pipeline, + limit = None + ): + + """ + Perform an advance query on the data. + :param collection: The collection to perform the query on. + :param pipeline: The pipeline of actions to take. Must be a list. + :param limit: The max. no. of records to retrieve. + :return: The array of matching records or null if there was an exception. + """ + + if self.__store is None: await self.connect() + + results = None + try: results = await self.__db[collection].aggregate(pipeline).to_list(limit) + except Exception as exception: self.__printer(exception) + return results + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/database/async_mongo_v2.py b/database/async_mongo_v2.py new file mode 100644 index 0000000..e5311c4 --- /dev/null +++ b/database/async_mongo_v2.py @@ -0,0 +1,1736 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Original: Tuesday, 28th May, 2024 + Modified: Tuesday, 17th Sept., 2024 + + OBJECTIVE: + + To have one central place from where all async MongoDB activity happens. + + REFERENCES: + + 01. https://motor.readthedocs.io/en/stable/ + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For system-level activity: +import io + +# For async behaviour: +import asyncio + +# MongoDB for File Storage: +from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket +from bson.objectid import ObjectId +from bson.json_util import dumps, loads + +# My utils: +from utils_v2.string import json + +# For datetime handling: +import pytz +import datetime + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongoBase: + + def __init__( + self, + connection_string = None, + max_connections = 5, + host_name = "localhost", + port = 27017, + database_name = "myDb", + debug = True, + debug_prefix = "Mongo | ", + debug_only_errors = True + ): + + # Basic variables that will be needed later: + self._client = None + self._db = None + self._fs = None + + # Database Initialization: + self._host_name = host_name + self._port = port + self._db_name = database_name + self._connection_string = connection_string + self._max_connections = max_connections + + # Debugging: + self._debug_only_errors = debug_only_errors + self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self._printer.disable() + + def enable_debug(self): + self._printer.enable() + + def disable_debug(self): + self._printer.disable() + + async def connect(self): + + """ + Initialize the database connection. + :return: None. + """ + + self._printer("Connecting...") + + # If a connection string is given, + # give preference to it: + if self._connection_string is None: + self._client = AsyncIOMotorClient( + self._host_name, + self._port, + maxPoolSize = self._max_connections, + minPoolSize = self._max_connections + ) + + # In the absense of a connection string, + # try to connect through the other credentials: + else: + self._client = AsyncIOMotorClient( + self._connection_string, + maxPoolSize = self._max_connections, + minPoolSize = self._max_connections + ) + + # Debugging print: + if not self._debug_only_errors: + server_info = await self._client.server_info() + self._printer(server_info) + + # Now we connect to the database and the file-store: + self._db = self._client.get_database(self._db_name) + self._fs = AsyncIOMotorGridFSBucket(self._db) + + async def ensure_connection(self): + + """ + Call this at the start of every function to automatically connect to the database when the user of the library + forgets to explicitly connect to the database. + :return: None. + """ + + if self._client is None: await self.connect() + + @staticmethod + def generate_id(as_str = False): + + """ + Just generates an '_id' in MongoDB style. + :param as_str: Set to True to convert the generated id to a string; and False to receive it as an instance of + 'ObjectId'. This is useful when you need to pre-assign ids to files. + :return: The '_id' in MongoDB style. + """ + + generated_id = ObjectId() + return str(generated_id) if as_str else generated_id + + @property + async def client(self): + + """ + returns the client to perform activities that have not been explicitly wrapped in the class. + :return: The client's instance. + """ + + await self.ensure_connection() + return self._client + + @property + async def db(self): + + """ + returns the database connector to perform activities that have not been explicitly wrapped in the class. + :return: The database's connection instance. + """ + + await self.ensure_connection() + return self._fs + + @property + async def fs(self): + + """ + returns the file-store to perform activities that have not been explicitly wrapped in the class. + :return: The file-store instance. + """ + + await self.ensure_connection() + return self._fs + + @staticmethod + def from_json_string(json_data): + + """ + Converts from a JSON string to BSON. + :param json_data: The JSON string to convert to BSON. + :return: The BSON interpretation of the input JSON string. + """ + + return loads(json_data) + + @staticmethod + def to_json_string(data, indent = 4, default = None): + + """ + Converts from a BSON to JSON string. + :param data: The input BSON data. + :param indent: The no. of spaces to put into the string for pretty print. + :param default: The default function to apply to data that cannot be converted directly. + :return: The JSON string from the input data. + """ + + return dumps(data, indent = indent, default = default) + + @staticmethod + def dict_to_dot_notation(input_dict, pk = "", s = "."): + + """ + Converts an input dict to dot notation format. Can be used as a utility to perform searches. + :param input_dict: The dict that you want to convert to dot notation. + :param pk: Parent Key. DO NOT TOUCH (meant to be used during recursion). + :param s: Separator. DO NOT TOUCH (meant to be used during recursion). + :return: The dot notation representation of the input dict. + """ + + items = [] + for k, v in input_dict.items(): + new_key = f"{pk}{s}{k}" if pk else k + if isinstance(v, dict) and v: items.extend(AsyncMongoBase.dict_to_dot_notation(v, new_key, s = s).items()) + else: items.append((new_key, v)) + return dict(items) + + @staticmethod + def normalize_date_time(document): + + """ + MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it. + This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was + timezone naive, UTC timezone will be applied to it without changing the time value. + :param document: The document that you want to normalize the date-time in. + :return: The document with normalized datetime. + """ + + if isinstance(document, datetime.datetime): + utc_tz = pytz.timezone("UTC") + if document.tzinfo is None: document = utc_tz.localize(document) + else: document = document.astimezone(utc_tz) + + if type(document) is list: + document = [AsyncMongoBase.normalize_date_time(item) for item in document] + + if type(document) is dict: + document = { + AsyncMongoBase.normalize_date_time(k): AsyncMongoBase.normalize_date_time(v) + for k, v in document.items() + } + + return document + + @staticmethod + def read_to_ram(file_path): + + """ + Reads a file into a BytesIO object in RAM. + :param file_path: The path to the file on disk. + :return: The file in a BytesIO object. + """ + + with open(file_path, "rb") as file: file_data = file.read() + file_in_ram = io.BytesIO(file_data) + file_in_ram.seek(0) + return file_in_ram + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncMongo(AsyncMongoBase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + async def list_indexes( + self, + collection, + session = None, + raise_exception = False + ): + + """ + Lists out the indexes of a collection. + :param collection: The collection whose indexes you want to list out, + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The list of indexes or None if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + indexes = None + + # Try to list the indexes: + try: + responses = await self._db[collection].list_indexes(session = session).to_list(None) + indexes = [{key: value for key, value in response.items()} for response in responses] + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return indexes + + async def create_index( + self, + collection, + keys, + options = None, + session = None, + raise_exception = False + ): + + """ + Creates an index on a collection. + :param collection: The collection to create the index on. + :param keys: The keys (and sorting) to implement the index on. + :param options: Additional config. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the execution. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the insertion: + try: + options = options or {} + keys = [(k, v) for k, v in keys.items()] + response = await self._db[collection].create_index(keys, session = session, **options) + if response: success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def count( + self, + collection, + filter, + session = None, + raise_exception = False + ): + + """ + Counts the no. of documents that match the given filter condition. + :param collection: The name of the collection to count in. + :param filter: The filter criteria that the documents must satisfy to be counted. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The count if the operation was performed successfully or None is something went wrong and the exception + was suppressed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # try to query the data: + try: count = await self._db[collection].count_documents(filter, session = session) + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def insert_one( + self, + collection, + document, + session = None, + raise_exception = False + ): + + """ + Insert data into a collection. + :param collection: The collection you want to feed the data into. + :param document: The data to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_id = None + + # Try to make the insertion: + try: + response = await self._db[collection].insert_one(document.copy(), session = session) + inserted_id = response.inserted_id + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return inserted_id + + async def insert_many( + self, + collection, + documents, + session = None, + raise_exception = False + ): + + """ + Insert a lot of data into a collection. + :param collection: The collection you want to feed the data into. + :param documents: The list of documents to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_ids = [] + + # Try to make the insertion: + try: + response = await self._db[collection].insert_many(documents, session = session) + inserted_ids = response.inserted_ids + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return inserted_ids + + async def find_one( + self, + collection, + filter, + projection = None, + session = None, + raise_exception = False + ): + + """ + Finds one record that matches the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The matching record or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to query the data: + try: results = await self._db[collection].find_one(filter, projection, session = session) + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Check results and return: + return results + + async def find_many( + self, + collection, + filter, + projection = None, + skip = 0, + limit = None, + sort = None, + session = None, + raise_exception = False + ): + + """ + Finds one or more records that match the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param skip: The no. of records to skip before picking next ones. Needed for pagination. + :param limit: The max. no. of records you want to fetch. + :param sort: The sorting rules to apply. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # Assume defaults: + if sort is None: sort = {"_id": -1} + if limit is None: limit = 10 + + # try to query the data: + try: + results = await self._db[collection].find( + filter, + projection, + sort = sort, + skip = skip, + limit = limit, + session = session + ).to_list(None) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return results + + async def update_one( + self, + collection, + filter = None, + update = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update one document. + :param collection: The collection you want to update. + :param filter: The selection criteria to locate the document to update. + :param update: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the update: + try: + response = await self._db[collection].update_one( + filter, + update, + upsert = upsert, + session = session + ) + success = True if response.modified_count > 0 or response.upserted_id else False + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def update_many( + self, + collection, + filter = None, + update = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update many documents. + :param collection: The collection you want to update. + :param filter: The selection criteria to locate the document to update. + :param update: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + update_count = 0 + + # Try to make the insertion: + try: + response = await self._db[collection].update_many( + filter, + update, + upsert = upsert, + session = session + ) + update_count = response.modified_count + response.upserted_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return update_count + + async def find_one_and_update( + self, + collection, + filter = None, + update = None, + projection = None, + return_updated = False, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update one document. + :param collection: The collection you want to update. + :param filter: The selection criteria to locate the document to update. + :param update: The values you want to update. + :param projection: What parts of the matching data you want to fetch. + :param return_updated: To choose whether you want to retrieve the original document or the updated document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + document = None + + # Try to make the update: + try: + document = await self._db[collection].find_one_and_update( + filter, + update, + projection = projection, + return_document = return_updated, + upsert = upsert, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return document + + async def replace_one( + self, + collection, + filter, + replacement, + upsert = False, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param replacement: The data to put in place of the existing document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # try to query the data: + try: + result = await self._db[collection].replace_one( + filter, + replacement, + upsert = upsert, + session = session + ) + if result.modified_count or result.upserted_id: success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def find_one_and_replace( + self, + collection, + filter, + replacement, + projection = None, + return_replaced = False, + upsert = False, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param replacement: The data to put in place of the existing document. + :param projection: What parts of the matching data you want to fetch. + :param return_replaced: To choose whether you want to retrieve the original document or the updated document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + document = None + + # try to query the data: + try: + document = await self._db[collection].find_one_and_replace( + filter, + replacement, + projection = projection, + return_document = return_replaced, + upsert = upsert, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return document + + async def delete_one( + self, + collection, + filter, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # try to query the data: + try: + result = await self._db[collection].delete_one(filter, session = session) + success = True if result.deleted_count else False + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def delete_many( + self, + collection, + filter, + session = None, + raise_exception = False + ): + + """ + To delete many documents from a collection. + WARNING: sending {} in the filter would mean deleting ALL the documents. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + result = await self._db[collection].delete_many(filter, session = session) + count = result.deleted_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def find_one_and_delete( + self, + collection, + filter, + projection = None, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete a record. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The document that matched your criteria or None. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + document = None + + # try to query the data: + try: + document = await self._db[collection].find_one_and_delete( + filter, + projection = projection, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return document + + async def bulk_write( + self, + collection, + requests, + session = None, + raise_exception = False + ): + + """ + To perform various individual operations in one go. You will have to import individual actions like "UpdateOne" + and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method. + :param collection: The collection you want to run the requests on. + :param requests: The array of requests (operations) to be performed. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + response = await self._db[collection].bulk_write(requests, session = session) + count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def aggregate( + self, + collection, + pipeline, + limit = None, + session = None, + raise_exception = False + ): + + """ + Perform an advance query on the data. + :param collection: The collection to perform the query on. + :param pipeline: The pipeline of actions to take. Must be a list. + :param limit: The max. no. of records to retrieve. preferably apply the limit from within the pipeline. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to perform the aggregation action: + try: results = await self._db[collection].aggregate(pipeline, session = session).to_list(limit) + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return results + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncMongoStorage(AsyncMongo): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + async def easy_upload( + self, + source, + file_name: str, + file_metadata: dict = None, + file_id = None, + chunk_size: int = None, + session = None, + raise_exception = False + ): + + """ + Easily write one file to MongoDB's GridFS. Ideal for directly uploading small files without having to worry + about any internal mechanisms. + :param source: The actual file, supplied as either a path string or a file-like object, to be written to the + database. + :param file_name: The name of the file as it will be stored on (and retrieved from) GridFS. + :param file_metadata: Any metadata to later search the file by. + :param file_id: Any custom id to be given to the file. TRY STICKING TO THE ID GENERATED BY 'generate_id'. + :param chunk_size: The chunk size (in bytes) to use for storing this file. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted file or None if the upload failed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_id = None + + # Try to write the file: + try: + + # Input pre-processing: + if isinstance(source, io.BytesIO): source.seek(0) + elif isinstance(source, str): source = open(source, mode = "rb") + + # If no file id was supplied: + if file_id is None: + inserted_id = await self._fs.upload_from_stream( + filename = file_name, + source = source, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + + # If a file id was supplied: + else: + await self._fs.upload_from_stream_with_id( + file_id = file_id, + filename = file_name, + source = source, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + inserted_id = file_id + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return inserted_id + + async def easy_download( + self, + destination, + file_id = None, + file_name = None, + session = None, + raise_exception = False + ): + + """ + Easily read one file from MongoDB's GridFS. Ideal for directly downloading small files without having to worry + about any internal mechanisms. + :param destination: The path on the local disk or a buffer in RAM to save the downloaded data to. + :param file_id: (RECOMMENDED) the id of the save file. + :param file_name: The name of the saved file. NOT RECOMMENDED because you could have many files with the same + name. The best way to tell files apart if from the id. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. The contents of the stored file are written + directly to the destination. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to read the file: + try: + + # Input pre-processing: + if isinstance(destination, str): destination = open(destination, mode = "wb") + + # If a file id is supplied (preferred way): + if file_id is not None: + await self._fs.download_to_stream( + destination = destination, + file_id = file_id, + session = session + ) + if isinstance(destination, io.BytesIO): destination.seek(0) + success = True + + # If a file name is supplied: + elif file_name is not None: + await self._fs.download_to_stream_by_name( + destination = destination, + filename = file_name, + session = session + ) + if isinstance(destination, io.BytesIO): destination.seek(0) + success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def count_files( + self, + filter, + session = None, + raise_exception = False + ): + + """ + Counts the no. of documents that match the given filter condition. + :param filter: The filter criteria that the documents must satisfy to be counted. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The count if the operation was performed successfully or None is something went wrong and the exception + was suppressed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # try to query the data: + try: count = await self._db["fs.files"].count_documents(filter, session = session) + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def find_one_file( + self, + filter, + projection = None, + session = None, + raise_exception = False + ): + + """ + Finds one record that matches the given conditions. This does NOT return the file itself, it returns the record + that describes the file. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The matching record or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to query the data: + try: results = await self._db["fs.files"].find_one(filter, projection, session = session) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Check results and return: + return results + + async def find_many_files( + self, + filter, + projection = None, + skip = 0, + limit = None, + sort = None, + session = None, + raise_exception = False + ): + + """ + Finds one or more records that match the given conditions. This doesn't return any actual files directly, it + returns the records that describe the files. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param skip: The no. of records to skip before picking next ones. Needed for pagination. + :param limit: The max. no. of records you want to fetch. + :param sort: The sorting rules to apply. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # Assume defaults: + if sort is None: sort = {"_id": -1} + if limit is None: limit = 10 + + # try to query the data: + try: + results = await self._db["fs.files"].find( + filter, + projection, + session = session + ).sort(sort).skip(skip).limit(limit).to_list(None) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return results + + async def update_metadata_for_one_file( + self, + filter: dict, + unset_data: dict = None, + set_data: dict = None, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for one file. + :param filter: The conditions to filter the files by. + :param unset_data: The fields that you want to discard. + :param set_data: The fields that you want to add or update. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the update: + try: + + # Input pre-processing: + update = {} + if unset_data is not None: update["$unset"] = self.dict_to_dot_notation({"metadata": unset_data}) + if set_data is not None: update["$set"] = self.dict_to_dot_notation({"metadata": set_data}) + + # Actual update happens here: + if update: + response = await self._db["fs.files"].update_one( + filter, + update, + session = session + ) + success = False if response.modified_count == 0 else True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def update_metadata_for_many_files( + self, + filter: dict, + unset_data: dict = None, + set_data: dict = None, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for many files. Practically the same as 'update_metadata_for_one' except that the scope of + the modifications is far wider. + :param filter: The conditions to filter the files by. + :param unset_data: The fields that you want to discard. + :param set_data: The fields that you want to add or update. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # Try to make the update: + try: + + # Input pre-processing: + update = {} + if unset_data is not None: update["$unset"] = self.dict_to_dot_notation({"metadata": unset_data}) + if set_data is not None: update["$set"] = self.dict_to_dot_notation({"metadata": set_data}) + + # Actual update happens here: + if update: + response = await self._db["fs.files"].update_many( + filter, + update, + session = session + ) + count = response.modified_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def replace_metadata_for_one_file( + self, + filter: dict, + replacement, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for one file. + :param filter: The conditions to filter the files by. + :param replacement: The new metadata to put inplace of the old one. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the update: + try: + + # Actual update happens here: + result = await self._db["fs.files"].update_one( + filter, + {"$set": {"metadata": replacement}}, + upsert = False, + session = session + ) + if result.modified_count or result.upserted_id: success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def replace_metadata_for_many_files( + self, + filter: dict, + replacement, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for one file. + :param filter: The conditions to filter the files by. + :param replacement: The new metadata to put inplace of the old one. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # Try to make the update: + try: + + # Actual update happens here: + response = await self._db["fs.files"].update_many( + filter, + {"$set": {"metadata": replacement}}, + session = session + ) + count = response.modified_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def delete_file_by_id( + self, + file_id, + session = None, + raise_exception = False + ): + + """ + Deletes one file by the file's id. Deleting files is different from deleting simple documents because, in files, + you also need to clear out the chunks (which are stored in a separate collection). The built-in mechanism of + Motor only provides support to delete by the file's id. + :param file_id: The id of the file that you want to delete. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to read the file: + try: + + # Delete the file: + await self._fs.delete(file_id = file_id, session = session) + success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def delete_one_file( + self, + filter, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Find one file that matches the given filter: + file = await self.find_one( + collection = "fs.files", + filter = filter, + session = session, + raise_exception = raise_exception + ) + + # If we have a list of files to work with: + if file: success = await self.delete_file_by_id(file["_id"]) + + # Done here: + return success + + async def delete_many_files( + self, + filter, + session = None, + raise_exception = False + ): + + """ + To delete many documents from a collection. + WARNING: sending {} in the filter would mean deleting ALL the documents. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The number of files deleted (can be zero) or None if something failed while searching the files. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # Find all the files that match the given filter: + files = await self.find_many( + filter = filter, + session = session, + raise_exception = raise_exception + ) + + # If we have a list of files to work with: + if files is not None: + tasks = [self.delete_file_by_id(file["_id"]) for file in files] + results = await asyncio.gather(*tasks) + count = sum(results) + + # Done here: + return count + + async def get_upload_stream( + self, + file_name: str, + file_metadata: dict = None, + file_id = None, + chunk_size: int = None, + session = None, + raise_exception = False + ): + + """ + Returns a GridIn object so that you can perform your own upload using the built-in writing methods. You must use + the 'write' method to write data to the file by passing it either a string of bytes or a file-like object. When + the file has been fully written, you must call the 'close' method to finish the operation. In case you need to + cancel the operation, you can call the 'abort' method to delete all the already written data and stop uploading + new data. + :param file_name: The name of the file as it will be stored on (and retrieved from) GridFS. + :param file_metadata: Any metadata to later search the file by. + :param file_id: Any custom id to be given to the file. TRY STICKING TO THE ID GENERATED BY 'generate_id'. + :param chunk_size: The chunk size (in bytes) to use for storing this file. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The upload stream that implements the 'write', 'close', and 'abort' methods, or None if something + failed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + stream = None + + # Try to open the upload stream: + try: + + # If no file id was supplied: + if file_id is None: + stream = self._fs.open_upload_stream( + filename = file_name, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + + # If a file id was supplied: + else: + stream = self._fs.open_upload_stream_with_id( + file_id = file_id, + filename = file_name, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return stream + + async def get_download_stream( + self, + file_id = None, + file_name = None, + session = None, + raise_exception = False + ): + + """ + Returns a GridOut object so that you can implement your own download logic using the built-in 'read' method. + Once the reading is done, use the 'close' method to release the resources used by the stream. + :param file_id: (RECOMMENDED) the id of the save file. + :param file_name: The name of the saved file. NOT RECOMMENDED because you could have many files with the same + name. The best way to tell files apart if from the id. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The download stream that implements the 'read' and 'close' methods, or None if something failed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + stream = None + + # Try to open a download stream: + try: + + # If a file id is supplied (preferred way): + if file_id is not None: + stream = await self._fs.open_download_stream( + file_id = file_id, + session = session + ) + + # If a file name is supplied: + elif file_name is not None: + stream = await self._fs.open_download_stream_by_name( + filename = file_name, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return stream + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + from shared import constants + + async def main(): + + # Create an instance of the database connector: + my_db = AsyncMongo( + connection_string = constants.MONGO_DATA_CONNECTION_STRING, + database_name = constants.MONGO_DATA_DATABASE_NAME, + max_connections = 10, + debug = True + ) + + # Connect to the database: + await my_db.connect() + + # # Get the documents to migrate: + # documents = await my_db.find_many( + # collection = "scriptData", + # filter = {}, + # limit = 50, + # projection = {"_id": False} + # ) + # # print(json.to_string(documents, default = str)) + # + # # Adjust them: + # adjusted_documents = [] + # for document in documents: + # script_id = document.pop("scriptId") + # adjusted_document = { + # "scriptId": script_id, + # "desc": "no desc", + # "content": document + # } + # adjusted_documents.append(adjusted_document) + # print(json.to_string(adjusted_documents, default = str)) + # + # # Insert the adjusted ones to the new collection: + # response = await my_db.insert_many( + # collection = "_scriptData", + # documents = adjusted_documents + # ) + # print("RESPONSE:", response) + + + asyncio.run(main()) diff --git a/database/async_mysql_v2.py b/database/async_mysql_v2.py new file mode 100644 index 0000000..f4cac6e --- /dev/null +++ b/database/async_mysql_v2.py @@ -0,0 +1,424 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 30th Aug., 2024 + + OBJECTIVE: + + To be able to access SQL-based databases from python in a simple way. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# MySQL Database: +import aiomysql +import decimal + +# For data-crunching: +import pandas as pd + +# For time-keeping: +import time + +# OS-level operations: +import os + +# My utils: +from utils_v2.string import json + +# For async activities: +import asyncio + +# For debugging: +from icecream import IceCreamDebugger +import traceback + +# To work with datatypes: +from typing import List + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMySQL: + + def __init__( + self, + pool_size, + *args, + **kwargs + ): + + """ + A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as + JSON-like structures (list or dict). The format for the results was very specific to our use case for serving + Bicree's requirement. This may not serve your requirement at all. + :param pool_size: The number of connections to maintain n a pool. + :param args: Any arguments to pass. Not used. + :param kwargs: Pass the connection configuration from here. + """ + + # Set up the variables: + self.__args = args + self.__kwargs = kwargs + self.__min_pool_size = 10 + self.__max_pool_size = max(pool_size, self.__min_pool_size) + self.__pool = None + + # Set up the debugging tools: + self.__printer = IceCreamDebugger(prefix = "MySQL | ", includeContext = True) + + def __del__(self): + pass + + async def connect(self): + + """ + Establish a connection and create a pool of connections to call from. + :return: None. + """ + + try: + + self.__kwargs["db"] = self.__kwargs.pop("database") + self.__pool = await aiomysql.create_pool( + minsize = self.__min_pool_size, + maxsize = self.__max_pool_size, + loop = asyncio.get_event_loop(), + **self.__kwargs + ) + + except Exception as exception: + self.__printer(exception) + self.__pool = None + + async def ensure_connection(self): + + """ + Tries to ensure that a connection is present. + Can be called before every function to make sure that our pool is established. + :return: None. + """ + + if self.__pool is None: await self.connect() + + @staticmethod + def __parse_row(row): + + """ + Converts from the custom objects of 'aiomysql' to types that are supported by Python. + :param row: The row from the result. + :return: The parsed row which will have types that are closer to being native to Python.. + """ + + parsed_row = [] + for item in row: + if isinstance(item, decimal.Decimal): parsed_row.append(float(item)) + else: parsed_row.append(item) + return parsed_row + + async def fetch_all(self, cursor): + + # Make a variable to hold all the result sets. + # Needed for when the procedure responds with many "tables": + all_result_sets = [] + + # Iterate over all result sets, + # and process them one-by-one: + while True: + + # Process the current result set: + this_result_set = [] + result = await cursor.fetchall() + if not cursor.description: break + columns = [desc[0] for desc in cursor.description] + for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row)))) + all_result_sets.append(this_result_set) + + # Move to the next set, + # or break out of the loop if all done: + if not await cursor.nextset(): break + + # Done here: + return all_result_sets + + async def call_procedure(self, procedure_name, procedure_args): + + """ + To call stored procedures and retrieve all the responses. + :param procedure_name: The name of the stored procedure that must be called. + :param procedure_args: The args to be sent to the stored procedure. + :return: The raw result set as received from the database. + """ + + # Make sure we have a connection: + await self.ensure_connection() + + # Make a variable to hold all the result sets. + # Needed for when the procedure responds with many "tables": + all_result_sets = [] + + # Call the procedure and get the results: + async with self.__pool.acquire() as connection: + async with connection.cursor() as cursor: + await cursor.callproc(procedure_name, procedure_args) + all_result_sets = await self.fetch_all(cursor) + + # # Iterate over all result sets, + # # and process them one-by-one: + # while True: + # this_result_set = [] + # result = await cursor.fetchall() + # if not cursor.description: break + # columns = [desc[0] for desc in cursor.description] + # for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row)))) + # all_result_sets.append(this_result_set) + # await cursor.nextset() + + # Done here: + return all_result_sets + + async def call_procedure_and_get_json( + self, + procedure_name, + procedure_args, + retry_count = 1, + backoff_seconds = 0.5, + backoff_multiplier = 1.1, + return_exception = False + ): + + """ + The method to call when you need to call a stored procedure and retrieve the response as a JSON-like object. + This is custom formatting based on the structure created by Mr. bhushan Thakkar in late April (2024). + :param procedure_name: The name of the stored procedure that must be called. + :param procedure_args: The args to be sent to the stored procedure. + :param retry_count: The max. number of times to try in case one or more attempts fail. + :param backoff_seconds: The time to wait before making the next attempt if the retry count is more than 1. + :param backoff_multiplier: The factor that dictates how much to modify the time delay by when waiting to retry. + :param return_exception: Whether, or not, you would like to return the exception object if something goes wrong. + :return: The formatted response and the exception (if asked for). + """ + + # Note down the start time: + start_ts = time.time() + + # Try to get the data from the database: + results = [] + exception = None + for _ in range(retry_count): + try: results = await self.call_procedure( + procedure_name = procedure_name, + procedure_args = procedure_args, + ) + except Exception as exc: exception = exc + if exception is None: break + await asyncio.sleep(backoff_seconds) + backoff_seconds = backoff_seconds * backoff_multiplier + + # If the results are blank: + if len(results) == 0: + formatted_results = { + "status": 0, + "message": "Please contact admin (NE)" if exception is None else "Please contact admin (E)", + "seconds": time.time() - start_ts, + "data": {} + } + if return_exception: return formatted_results, exception + else: return formatted_results + + # Extract the very basic success or failure indicators: + formatted_results = { + "status": results[0][0]["status"], + "message": results[0][0].get("message", "ok"), + "seconds": 0.0, + "data": {} + } + + # Handle the remaining keys of the zeroth result set: + for key, value in results[0][0].items(): + if key not in formatted_results.keys(): + formatted_results["data"][key] = value + + # Format + for index in range(len(results)): + if index > 0: formatted_results["data"][f"rs{index-1}"] = results[index] + + # Note down the time taken: + formatted_results["seconds"] = time.time() - start_ts + + # Done here: + if return_exception: return formatted_results, exception + else: return formatted_results + + async def execute_one( + self, + query: str, + commit: bool = True, + return_exception: bool = False + ): + + """ + Runs one command / query in SQL. + :param query: The query / command to run. + :param commit: Whether, or not, you would like to commit the execution. + :param return_exception: Whether, or not, you would like to return the exception from this function. + :return: Either just the result or the result and the exception. + """ + + # Make sure we have a connection: + await self.ensure_connection() + + # Start by assuming failure: + rows_affected = None + results = None + excp = None + + try: + + # Get a connection and execute the command: + async with self.__pool.acquire() as connection: + async with connection.cursor() as cursor: + rows_affected = await cursor.execute(query) + results = await self.fetch_all(cursor) + if commit: await connection.commit() + + # SQL-specific errors: + except aiomysql.MySQLError as exception: + self.__printer("SQL Exception", exception) + excp = exception + + # Other errors: + except Exception as exception: + self.__printer("Other Exception", exception) + excp = exception + + # Done here: + if return_exception: return rows_affected, results, excp + else: return rows_affected, results + + async def execute_many( + self, + query: str, + data: List[tuple], + commit: bool = True, + return_exception: bool = False + ): + + """ + Runs many commands / queries in SQL. + Consider the following example: + QUERY: "INSERT INTO pincodeMaster (pincode, city, state) VALUES (%s, %s, %s);" + DATA: [ + ('110001', 'New Delhi', 'Delhi'), + ('500001', 'Hyderabad', 'Telangana'), + ('600001', 'Chennai', 'Tamil Nadu') + ] + :param query: The query / command to run. + :param data: The data to substitute into the query string. + :param commit: Whether, or not, you would like to commit the execution. + :param return_exception: Whether, or not, you would like to return the exception from this function. + :return: Either just the result or the result and the exception. + """ + + # Make sure we have a connection: + await self.ensure_connection() + + # Start by assuming failure: + rows_affected = None + results = None + excp = None + + try: + + # Get a connection and execute the command: + async with self.__pool.acquire() as connection: + async with connection.cursor() as cursor: + rows_affected = await cursor.executemany(query, data) + results = await self.fetch_all(cursor) + if commit: await connection.commit() + + # SQL-specific errors: + except aiomysql.MySQLError as exception: + self.__printer("SQL Exception", exception) + excp = exception + + # Other errors: + except Exception as exception: + self.__printer("Other Exception", exception) + excp = exception + + # Done here: + if return_exception: return rows_affected, results, excp + else: return rows_affected, results + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/dataframe/__init__.py b/dataframe/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dataframe/splitter.py b/dataframe/splitter.py new file mode 100644 index 0000000..7725905 --- /dev/null +++ b/dataframe/splitter.py @@ -0,0 +1,118 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 22nd Oct., 2024 + + OBJECTIVE: + + To provide an easy way to split dataframes into chunks and divide the workload nto more manageable batches. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + USAGE EXAMPLE: + + for sub_df in DataFrameSplitter(df, chunk_size = 50): + + # Do your task here: + pass + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DataFrameSplitter: + + def __init__(self, df, chunk_size): + + """ + Use this to process your dataframe in batches. Useful for when you need to send out alerts at intervals or need + to maintain checkpoints. + :param df: The dataframe to iterate over. + :param chunk_size: The max. no. of rows to pick at once. + """ + + self.df = df + self.row_count = df.shape[0] + self.chunk_size = chunk_size + self.offset = 0 + + def __iter__(self): + self.offset = 0 + return self + + def __next__(self): + if self.offset >= self.row_count: + raise StopIteration + end = self.offset + self.chunk_size + chunk = self.df.iloc[self.offset:end] + self.offset = end + return chunk + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/date_time/__init__.py b/date_time/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/date_time/__pycache__/__init__.cpython-310.pyc b/date_time/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9b4dd9fc9461d005c17ecda7dfee64a72effb570 GIT binary patch literal 172 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vXKO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaMNIxa9 oBsIPy6R0~rJ~J<~BtBlRpz;=nO>TZlX-=vg$f9BT#DQ! zW|vk-P(gvDN2BNVBprRrztB^!MS))SR-_lB#{xwWplN?I%cYjGlLjaXv@7lGZ@y>V zyx)6oXwS~36g-*VTyOj(qbOgCO!$c)a~H2eRYjpnRiP@?+Gh|dsC=qACF>K_r2M9;(==Y4VbSARI(4E}=Oju1os%dDG%ryK zXhEWBphbyhfR-eh1zMJ94(Pl@=YTFqG!OKOL<>L{C0Yc!B+(MkWr>!7UX|!P&=rX; z0KF#BD?mSz=pxY7q?Wuy-=UZ3t6!>BLT}J3^fjQ@=}mf-5}<2zg}zR&0lfkA27MFg zO`4`}eW_H_^lf?_-?!M?G;^TwuaXMY%JR)*2Y*bZ2%h}o%7d+)H9sQ?9yWVU(`b`z zQnqc2GTt)%%KQ0BA=GYzo7CvvB)8smT+-@UK&!Xj_nU6r`=qd5DSjGmU$IG-+fPi& zh+z?CIDK+x^oeVeLz^FvL(^>%*X*#YKg3R9qp(vbtryCnIZL_xXf0d2rAJ%&`!7^J z7iv}V)a*dZW;2?XpoYuDuwGK6l(k(WGc!n}X)fK#rTYS{r1w_>)ES{{x|8+GWV^j_ zyAeV=iDbLIK}?LdKZvJkx`T>%dJkVZSV-@!q|ua;^nrF{X*)IhaIg@pplsOsw$W?bqt^F(`{=opugficVq+|eIr}}{ z7);C2cg(&)o8OFOQwd~K6J{WIVgSNutmAbmNCp>%Dph+5pLUgFb*P-EM-eOT!aNL> zp?abnMTU{%=ukURca^ddRoqxVqbf&HRT)g}Dtj;)N72t=FhBr_4%I^?qZym9IFbm^ z&zdcS>k9oet!~dHf$|Ut&a@grSq(zXdYy4*xukCMj^Soy_qDFUjn24-?)O+Dl_nb( zhK9jGSWX+dInmp(d%VVUe_X$Bt;a)i5xY)YjOdS^CB-_iY&Rely(y={N*u<)sCJv| zGtzSyotT9($2Bd(HEnBRDt}tB$v_aUJ?hAr+vC<~MZT06J-aWzZ7CD?QbKV4GQ61o z6=&-uuLsq;K-|z3qVnnX8cmgj`>lG z*s?<7c&C=p(oNC?O+HmGDn}7HOA4bG-09W~c%;$PQ!q)EPoezLFlaT~nroeF&9@d% zx;Ipg7f(<#jDV3UuS;g)>n;r=L-h;E#j=u_Hhvvqmunmd)88itrbUG!`y7MW0dg7d zLdm3Z2j@xZ-0l!_l3=ov{={ha*cvIoX+YDBfLJ!L;I7LWC`Qh~$xE7w`NrdNhPfP^ zwB*19H^gBNYzt!x;s!ZM{BE}`-Id_qc)=kSWJ%%Y#d4)s`j}LTn}zCDsX&UQjV-cU zeDsLiE0C>E3p=|z#Y&}6nwV4SkWW6xpVz1N$i6ruMmlhvV0DvOL{LiDqw&V2R9?bW z$-NL!ArhaGNzf0@lczleh8A0qMVl+s=EPc?*asuzXZ zckq?G8yOHMv+OO106`a9*EhCyHuIITmol+T$2F`P^CFO$7pu2z!}T<)>&2~JXP0G0Ufgx$H#P3zc*#o8+naT5 zz1t5C-J3HU-KKmi*ZDCL5Lu9F&INKoCLrnL zKxqX=ASPX(HdOhgp>`ZUfwLPTs)H+=3COl1tu^!pni}{Iu{~lq1paFj5fOi96RH>p z;oqpg?+(JcANGL|w6DNxWVgu#LvIQ$!cpBMeY=OS557~Z&F;Cv%gR;xJzk>1XO^2A zabwJ3>6g!Jbfn*zDSjCO<*y_0qTRkb;O~gM5Vsex>_abUI=0WeOhnds$?@uWRQ+afQ-%0s> zY(8O*kjjfEXIccdWo(cp4sE>R|AxG!74@T^A0YB)nlIs1Y%!xrIe0UHHPo?#XDSzU zXT^FcELPKTPeuD%Au&OJEupEJ8vm9Hj(n$Z`7P9Dv>zl;MB@MR0_E??{hXB~md-yw z&G*W&=hjZ;_y^JYCnC9n{iBM=!4x&E&2g6ibygQml~eodO*9{W7_hFO%ZNHBxy{ICb>9C=><7()Q!Z zxOjr5fg#g@BdOD1$wcuZlog#mTmNAjYM9>JitaKXbB@x0JdH!idk-cdQ8 z!UfUS*5`q?uKTwb#}6`vIt4vZFoDA9%A9ifoYqO6; zIOc34i7E*qZt020yqZ9oP@~hS`4mp``~S2Hnwt7Y>Qd@PY9Y0dSV$}-UWsebZ~p}x Cu7TnJ literal 0 HcmV?d00001 diff --git a/date_time/date_time.py b/date_time/date_time.py new file mode 100644 index 0000000..6e57fd5 --- /dev/null +++ b/date_time/date_time.py @@ -0,0 +1,271 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 21st jun, 2024 + + OBJECTIVE: + + To provide an easy way to work with time. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For date and time handling: +import pytz +from datetime import datetime, timedelta +import dateparser + +# To handle date-time objects from a Numpy array and Pandas Dataframe: +import numpy as np +import pandas as pd + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Date-Time Formats: +DATE_TIME_FORMATS = ( + "%d/%m/%y", + "%d-%b-%y", + "%d-%m-%y", + "%d.%m.%y", + "%d/%m/%Y", + "%d-%b-%Y", + "%d-%m-%Y", + "%d.%m.%Y", + "%d/%b", + "%d%m%Y", + "%Y%m%d", + "%Y-%m-%d %H:%M:%S" +) + +# Useful Timezones: +TIMEZONE_UTC = pytz.timezone("UTC") +TIMEZONE_IST = pytz.timezone("Asia/Kolkata") +TIMEZONE_ET = pytz.timezone("America/New_York") +TIMEZONE_CT = pytz.timezone("America/Chicago") +TIMEZONE_MT = pytz.timezone("America/Denver") +TIMEZONE_PT = pytz.timezone("America/Los_Angeles") +TIMEZONE_JST = pytz.timezone("Asia/Tokyo") +TIMEZONE_CET = pytz.timezone("Europe/Paris") +TIMEZONE_GMT = pytz.timezone("GMT") +TIMEZONE_AEST = pytz.timezone("Australia/Sydney") +TIMEZONE_NZST = pytz.timezone("Pacific/Auckland") +TIMEZONE_CST = pytz.timezone("Asia/Shanghai") +TIMEZONE_KST = pytz.timezone("Asia/Seoul") +TIMEZONE_MSK = pytz.timezone("Europe/Moscow") +TIMEZONE_BRT = pytz.timezone("America/Sao_Paulo") +TIMEZONE_GST = pytz.timezone("Asia/Dubai") +TIMEZONE_SAST = pytz.timezone("Africa/Johannesburg") +TIMEZONE_AST = pytz.timezone("Asia/Riyadh") + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def translate_date_time_string( + datetime_string, + source_format = None, + destination_format = "%Y-%m-%dT%H:%M:%S" +): + + """ + To convert an input datetime string to a different format. + :param datetime_string: The datetime string to translate. + :param source_format: The current format of the string. If not provided, dateparser will be used. + :param destination_format: The format to convert to. + :return: The converted datetime string. + """ + + try: + if source_format is None: datetime_obj = dateparser.parse(datetime_string) + else: datetime_obj = datetime.strptime(datetime_string, source_format) + return datetime_obj.strftime(destination_format) + except Exception as exception: + return None + + +# --------------------------------------------------------------------------------------------------------------------- + + +def parse_date_time(input_value, timezone = None, date_formats = None): + + """ + To take any kind of input and interpret the datetime from it. + :param input_value: Either a string or an integer or some form of datetime representation. + :param timezone: The timezone to apply to the interpreted datetime. EXISTING TIMEZONE INFO WILL BE OVERWRITTEN. + :param date_formats: The string formats to consider when parsing a string input. + :return: The parsed datetime or null. + """ + + datetime_object = None + date_formats = date_formats or DATE_TIME_FORMATS + + # It could either be in seconds or milliseconds from epoch time's base date (January 1, 1970), + # or it could be days since Microsoft Excel's base date (December 31, 1899). + if isinstance(input_value, (int, float, np.number)) and not np.isnan(input_value): + if input_value > 9999999999.0: datetime_object = datetime.fromtimestamp(input_value / 1000.0) + if input_value > 999999.0: datetime_object = datetime.fromtimestamp(input_value) + else: datetime_object = datetime.fromtimestamp(input_value * 24 * 60 * 60.0) - timedelta(days = 25569) + + # The input can even be a pre-formatted date: + if isinstance(input_value, str): + datetime_object = dateparser.parse(input_value, date_formats = date_formats) + + # If the type is a datetime object, then return it as it is: + if isinstance(input_value, datetime): + datetime_object = input_value + + # If the type is the native datetime format of pandas: + if isinstance(input_value, pd._libs.tslibs.timestamps.Timestamp): + datetime_object = input_value.to_pydatetime() + + # Process the timezone: + if datetime_object is not None and timezone is not None: + datetime_object = as_if_timezone(datetime_object, timezone) + + # Done here: + return datetime_object + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_current_date_time(timezone = None, as_string = False): + + """ + Returns the current time as a datetime object. + :param timezone: The timezone to apply to the returned datetime. + :param as_string: Whether, or not, you want the output as a string. + :return: The datetime object/string representing the current time. + """ + + if timezone is not None and isinstance(timezone, str): timezone = pytz.timezone(timezone) + now = datetime.now(timezone) + return now.isoformat() if as_string else now + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_current_ist_date_time(as_string = False): + + """ + Gives out the current time in IST timezone. + :param as_string: Whether, or not, you want the output as a string. + :return: The datetime object or string representing the current time. + """ + + return get_current_date_time( + timezone = TIMEZONE_IST, + as_string = as_string + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_current_utc_date_time(as_string = False): + + """ + Gives out the current time in UTC timezone. + :param as_string: Whether, or not, you want the output as a string. + :return: The datetime object or string representing the current time. + """ + + return get_current_date_time( + timezone = TIMEZONE_UTC, + as_string = as_string + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def as_if_timezone(datetime_object, timezone): + + """ + Ignores existing timezone info and applies the intended timezone. + The time stays the same, only the timezone marker changes. + e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-09 00:00:00+00:00 + HINT: IT PRETENDS "AS IF" THE TIMEZONE WAS THE INPUT TIMEZONE. + :param datetime_object: The datetime object on which the timezone needs to be applied. + :param timezone: The timezone that needs to be applied. + :return: A timezone-aware datetime object. + """ + + tz_object = pytz.timezone(timezone) if isinstance(timezone, str) else timezone + return tz_object.localize(datetime_object.replace(tzinfo = None)) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_timezone(datetime_object, timezone): + + """ + Converts from one timezone to another. The time is adjusted by computing the difference between the two timezones. + NOTE: THIS FUNCTION ASSUMES THE INPUT WAS IN UTC IF THE INPUT WAS TIMEZONE-NAIVE. + e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-08 18:30:00+00:00 + :param datetime_object: The datetime object on which the timezone needs to be applied. + :param timezone: The timezone that needs to be applied. + :return: A timezone-aware datetime object. + """ + + if isinstance(timezone, str): timezone = pytz.timezone(timezone) + if datetime_object.tzinfo is None: return datetime_object.replace(tzinfo = TIMEZONE_UTC) + return datetime_object.astimezone(timezone) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/image/__init__.py b/image/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/image/scanner/__init__.py b/image/scanner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/image/scanner/scanner.py b/image/scanner/scanner.py new file mode 100644 index 0000000..28fe69b --- /dev/null +++ b/image/scanner/scanner.py @@ -0,0 +1,410 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 19th Sept., 2024 + + OBJECTIVE: + + To provide a way to easily scan documents like printouts and visiting cards. + + REFERENCES: + + 01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085 + 02. https://github.com/JaidedAI/EasyOCR + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys + +from diffusers.utils.import_utils import candidates + +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To work with image processing: +import cv2 +import imutils +import numpy as np + +# To run OCR: +import easyocr + +# To download images from the web: +import requests + +# My utils: +from utils_v2.string import json +from utils_v2.ai.object_detection.yolo import YoloDetect + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DocumentScanner: + + def __init__( + self, + layout_detection_yolo = None, + whitelisted_yolo_classes = None, + ocr_languages = None + ): + + self.__yolo = YoloDetect( + model_file = layout_detection_yolo, + debug = False + ) + self.__whitelisted_yolo_classes = whitelisted_yolo_classes or [] + self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"]) + pass + + @staticmethod + def open_image(image): + + """ + Opens an image from various sources. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :return: The image as a CV2 object. + """ + + # When the image is provided as a string, + # it could be either as a URL or a path to a local file: + if isinstance(image, str): + if image.startswith("https://") or image.startswith("http://"): + image = io.BytesIO(requests.get(image).content) + else: image = cv2.imread(image) + + # If the image is provided as a io.BytesIO buffer: + if isinstance(image, io.BytesIO): + image.seek(0) + image = np.asarray(bytearray(image.read()), dtype = np.uint8) + image = cv2.imdecode(image, cv2.IMREAD_COLOR) + + # Done here: + return image + + @staticmethod + def polygon_area(points): + + """ + Computes the area occupied by a shape described by the array of points. + Example input: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]] + :return: The area of the shape. + """ + + n = len(points) + area = 0.0 + for i in range(n): + j = (i + 1) % n # next vertex + area += points[i][0] * points[j][1] + area -= points[j][0] * points[i][1] + return abs(area) / 2.0 + + @staticmethod + def show(image, title = "Preview", wait = True): + + """ + Just a quick wrapper to show the image in a window. + :param image: The image that you want to show. + :param title: The title of the window. + :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many + windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the + very last one. + :return: None. + """ + + cv2.imshow(title, image) + if wait: + cv2.waitKey() + cv2.destroyAllWindows() + + @staticmethod + def enhance(image): + + """ + Applies a form of contrast boost to make the edges more easily visible. + :param image: The OpenCV image that needs to be enhanced. + :return: The enhanced CV2 image. + """ + + return cv2.detailEnhance( + src = image, + sigma_s = 20, + sigma_r = 0.15 + ) + + @staticmethod + def get_edges(image): + + """ + Gets the edges in an image. + :param image: A CV2 image. + :return: The image with the edges detected. + """ + + # Denoise the image: + processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + processed_image = cv2.GaussianBlur( + processed_image, + ksize = (5, 5), + sigmaX = 0 + ) + + # Detect edges in the image: + processed_image = cv2.Canny( + processed_image, + threshold1 = 50, + threshold2 = 200 + ) + + # Close small gaps in the edges: + kernel = np.ones((5, 5), np.uint8) + processed_image = cv2.dilate(processed_image, kernel, iterations = 1) + # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel) + # processed_image = cv2.erode(processed_image, kernel, iterations = 1) + + # Done here: + return processed_image + + def scan(self, image, min_area = 0.125): + + """ + Looks for a rectangle in an image and flattens it out. No colour filters are applied here. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_area: The minimum area occupied by the document in the image. + :return: The flattened image if any, else None. + """ + + # Open the image and read the data: + image = self.open_image(image) + if image is None: return None + height, width = image.shape[:2] + image_area = height * width + + # Enhance the image to have better visibility of edges: + # processed_image = self.enhance(image.copy()) + + # Detect edges in the image: + processed_image = self.get_edges(image.copy()) + + # Find the contours in the edges, + # and sort them in ascending order: + contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + contours = imutils.grab_contours(contours) + contours = sorted(contours, key = cv2.contourArea, reverse = True) + + # We try to get clean and passable contours. + # For our purposes, 'clean' is when it can be approximated to exactly 4 sides, + # and 'passable' is when the approximation comes close to it: + clean_rects = [] + passable_rects = [] + for contour in contours: + perimeter = cv2.arcLength(contour, closed = True) + approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True) + if len(approximation) == 4: clean_rects.append(approximation) + elif len(approximation) <= 32: + candidate_contour = cv2.minAreaRect(approximation) + candidate_contour = cv2.boxPoints(candidate_contour) + candidate_contour = np.intp(candidate_contour) + passable_rects.append(candidate_contour) + clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True) + passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True) + + # Now we decide between the best candidate: + document_outline = None + if len(clean_rects) > 0: + candidate_rect = np.array([p[0] for p in clean_rects[0]]) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + if document_outline is None and len(passable_rects) > 0: + candidate_rect = cv2.minAreaRect(passable_rects[0]) + candidate_rect = cv2.boxPoints(candidate_rect) + candidate_rect = np.intp(candidate_rect) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + + # If there is no rectangular contour, + # we exit with failure: + if document_outline is None: return None + + # Since we did get the best candidate for the document, + # we figure out which point n the contour is which corner of the document: + document_rectangle = np.zeros((4, 2), dtype = "float32") + sum_points = document_outline.sum(axis = 1) + document_rectangle[0] = document_outline[np.argmin(sum_points)] + document_rectangle[2] = document_outline[np.argmax(sum_points)] + diff_points = np.diff(document_outline, axis = 1) + document_rectangle[1] = document_outline[np.argmin(diff_points)] + document_rectangle[3] = document_outline[np.argmax(diff_points)] + (top_left, top_right, bottom_right, bottom_left) = document_rectangle + + # let's compute the dimensions of the document in the image: + left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2)) + right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2)) + top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2)) + bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2)) + max_height = max(int(left_height), int(right_height)) + max_width = max(int(top_width), int(bottom_width)) + + # We compute the destination of the transform: + destination = np.array( + object = [ + [0, 0], + [max_width - 1, 0], + [max_width - 1, max_height - 1], + [0, max_height - 1] + ], + dtype = "float32" + ) + + # We apply the transform that flattens out the document: + matrix = cv2.getPerspectiveTransform(document_rectangle, destination) + flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height)) + + # Done here: + return flattened_image + + def extract_text( + self, + image, + min_confidence = 0.5, + margin = 0.05 + ): + + """ + To run OCR on the input image. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_confidence: The minimum amount of confidence for detected text to be considered. + :param margin: The extra pixels to include when cropping into a section for OCR. + :return: The extracted text. + """ + + # Make a variable that will hold the final result: + extracted_text = {} + + # Open the image and read the data: + image = self.open_image(image) + height, width = image.shape[:2] + + # Run OCR on the entire canvas: + ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence] + extracted_text["fullDoc"] = " ".join(ocr_result) + + # Get the doc layout in the image: + document_sections = self.__yolo.predict(image) + document_sections = document_sections["boxes"] + + # For each section, we run the OCR process individually: + section_results = [] + for section in document_sections: + if section["class"] in self.__whitelisted_yolo_classes: + section_width = section["x2"] - section["x1"] + section_height = section["y2"] - section["y1"] + x1 = max(int(section["x1"] - (margin * section_width)), 0) + y1 = max(int(section["y1"] - (margin * section_height)), 0) + x2 = min(int(section["x2"] + (margin * section_width)), width) + y2 = min(int(section["y2"] + (margin * section_height)), height) + sub_image = image[y1:y2, x1:x2] + ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence] + section_results.append({ + "text": " ".join(ocr_result), + "class": section["class"], + "className": section["className"], + "x1": x1, + "y1": y1, + "x2": x2, + "y2": y2 + }) + extracted_text["bySection"] = section_results + + # Done here: + return extracted_text + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import time + + my_scanner = DocumentScanner( + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], + # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], + ocr_languages = ["en"] + ) + + image = my_scanner.open_image(r"/home/developer/Downloads/talkaholics_card.jpg") + document_image = my_scanner.scan(image) + if document_image is not None: + cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image) + start_time = time.time() + document_text = my_scanner.extract_text( + document_image, + min_confidence = 0.5 + ) + print("OCR RESULT:") + print(document_text) + + print(f"FINISHED IN {time.time() - start_time} SECONDS!") + + else: print("No image") diff --git a/image/scanner/scanner_v2.py b/image/scanner/scanner_v2.py new file mode 100644 index 0000000..fc63627 --- /dev/null +++ b/image/scanner/scanner_v2.py @@ -0,0 +1,379 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 24th Sept., 2024 + + OBJECTIVE: + + To provide a way to easily scan documents like printouts and visiting cards. + + REFERENCES: + + 01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085 + 02. https://github.com/JaidedAI/ + 03. https://github.com/criistian14/flutter_document_scanner/blob/master/flutter_document_scanner_android/android/src/main/kotlin/com/christian/flutterDocumentScanner/OpenCVPlugin.kt + 04. https://www.geeksforgeeks.org/python-bilateral-filtering/ + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To work with image processing: +import cv2 +import imutils +import numpy as np + +# To run OCR: +import easyocr + +# To download images from the web: +import requests + +# My utils: +from utils_v2.string import json +from utils_v2.ai.object_detection.yolo import YoloDetect + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DocumentScanner: + + def __init__( + self, + layout_detection_yolo = None, + whitelisted_yolo_classes = None, + ocr_languages = None + ): + + self.__yolo = YoloDetect( + model_file = layout_detection_yolo, + debug = False + ) + self.__whitelisted_yolo_classes = whitelisted_yolo_classes or [] + self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"]) + pass + + @staticmethod + def open_image(image): + + """ + Opens an image from various sources. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :return: The image as a CV2 object. + """ + + # When the image is provided as a string, + # it could be either as a URL or a path to a local file: + if isinstance(image, str): + if image.startswith("https://") or image.startswith("http://"): + image = io.BytesIO(requests.get(image).content) + else: image = cv2.imread(image) + + # If the image is provided as a io.BytesIO buffer: + if isinstance(image, io.BytesIO): + image.seek(0) + image = np.asarray(bytearray(image.read()), dtype = np.uint8) + image = cv2.imdecode(image, cv2.IMREAD_COLOR) + + # Done here: + return image + + @staticmethod + def show(image, title = "Preview", wait = True): + + """ + Just a quick wrapper to show the image in a window. + :param image: The image that you want to show. + :param title: The title of the window. + :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many + windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the + very last one. + :return: None. + """ + + cv2.imshow(title, image) + if wait: + cv2.waitKey() + cv2.destroyAllWindows() + + @staticmethod + def get_edges(image): + + """ + Apply all the preprocessing filters on the image before sending it off for contour-finding. + :param image: The image to pre-process. + :return: The pre-processed image. + """ + + # Denoise the image: + processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + processed_image = cv2.bilateralFilter( + processed_image, + d = 9, + sigmaColor = 75, + sigmaSpace = 75 + ) + + # Detect edges in the image: + processed_image = cv2.Canny( + processed_image, + threshold1 = 75, + threshold2 = 200 + ) + + # Close small gaps in the edges: + kernel = np.ones((5, 5), np.uint8) + processed_image = cv2.dilate(processed_image, kernel, iterations = 1) + # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel) + # processed_image = cv2.erode(processed_image, kernel, iterations = 1) + + # Done here: + return processed_image + + def scan(self, image, min_area = 0.125): + + """ + Looks for a rectangle in an image and flattens it out. No colour filters are applied here. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_area: The minimum area occupied by the document in the image. + :return: The flattened image if any, else None. + """ + + # Open the image and read the data: + image = self.open_image(image) + if image is None: return None + height, width = image.shape[:2] + image_area = height * width + + # image = self.enhance(image) + processed_image = self.get_edges(image) + # self.show(processed_image) + + # Find the contours in the edges, + # and sort them in ascending order: + contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + contours = imutils.grab_contours(contours) + contours = sorted(contours, key = cv2.contourArea, reverse = True) + + # We try to get clean and passable contours. + # For our purposes, 'clean' is when it can be approximated to exactly 4 sides, + # and 'passable' is when the approximation comes close to it: + clean_rects = [] + passable_rects = [] + for contour in contours: + perimeter = cv2.arcLength(contour, closed = True) + approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True) + if len(approximation) == 4: clean_rects.append(approximation) + elif len(approximation) <= 32: + candidate_contour = cv2.minAreaRect(approximation) + candidate_contour = cv2.boxPoints(candidate_contour) + candidate_contour = np.intp(candidate_contour) + passable_rects.append(candidate_contour) + clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True) + passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True) + + # Now we decide between the best candidate: + document_outline = None + if len(clean_rects) > 0: + candidate_rect = np.array([p[0] for p in clean_rects[0]]) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + if document_outline is None and len(passable_rects) > 0: + candidate_rect = cv2.minAreaRect(passable_rects[0]) + candidate_rect = cv2.boxPoints(candidate_rect) + candidate_rect = np.intp(candidate_rect) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + + # If there is no rectangular contour, + # we exit with failure: + if document_outline is None: return None + + # Since we did get the best candidate for the document, + # we figure out which point n the contour is which corner of the document: + document_rectangle = np.zeros((4, 2), dtype = "float32") + sum_points = document_outline.sum(axis = 1) + document_rectangle[0] = document_outline[np.argmin(sum_points)] + document_rectangle[2] = document_outline[np.argmax(sum_points)] + diff_points = np.diff(document_outline, axis = 1) + document_rectangle[1] = document_outline[np.argmin(diff_points)] + document_rectangle[3] = document_outline[np.argmax(diff_points)] + (top_left, top_right, bottom_right, bottom_left) = document_rectangle + + # let's compute the dimensions of the document in the image: + left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2)) + right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2)) + top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2)) + bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2)) + max_height = max(int(left_height), int(right_height)) + max_width = max(int(top_width), int(bottom_width)) + + # We compute the destination of the transform: + destination = np.array( + object = [ + [0, 0], + [max_width - 1, 0], + [max_width - 1, max_height - 1], + [0, max_height - 1] + ], + dtype = "float32" + ) + + # We apply the transform that flattens out the document: + matrix = cv2.getPerspectiveTransform(document_rectangle, destination) + flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height)) + + # Done here: + return flattened_image + + def extract_text( + self, + image, + min_confidence = 0.5, + margin = 0.05 + ): + + """ + To run OCR on the input image. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_confidence: The minimum amount of confidence for detected text to be considered. + :param margin: The extra pixels to include when cropping into a section for OCR. + :return: The extracted text. + """ + + # Make a variable that will hold the final result: + extracted_text = {} + + # Open the image and read the data: + image = self.open_image(image) + height, width = image.shape[:2] + + # Run OCR on the entire canvas: + ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence] + extracted_text["fullDoc"] = " ".join(ocr_result) + + # Get the doc layout in the image: + document_sections = self.__yolo.predict(image) + document_sections = document_sections["boxes"] + + # For each section, we run the OCR process individually: + section_results = [] + for section in document_sections: + if section["class"] in self.__whitelisted_yolo_classes: + section_width = section["x2"] - section["x1"] + section_height = section["y2"] - section["y1"] + x1 = max(int(section["x1"] - (margin * section_width)), 0) + y1 = max(int(section["y1"] - (margin * section_height)), 0) + x2 = min(int(section["x2"] + (margin * section_width)), width) + y2 = min(int(section["y2"] + (margin * section_height)), height) + sub_image = image[y1:y2, x1:x2] + ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence] + section_results.append({ + "text": " ".join(ocr_result), + "class": section["class"], + "className": section["className"], + "x1": x1, + "y1": y1, + "x2": x2, + "y2": y2 + }) + extracted_text["bySection"] = section_results + + # Done here: + return extracted_text + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import time + + my_scanner = DocumentScanner( + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], + # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], + ocr_languages = ["en"] + ) + + image = my_scanner.open_image(r"/home/developer/Downloads/sushmita_card.jpg") + # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card.jpg") + # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card_2.jpg") + # image = my_scanner.open_image(r"/home/developer/Downloads/card_square.jpg") + document_image = my_scanner.scan(image) + if document_image is not None: + cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image) + start_time = time.time() + document_text = my_scanner.extract_text( + document_image, + min_confidence = 0.5 + ) + print("OCR RESULT:") + print(document_text) + + print(f"FINISHED IN {time.time() - start_time} SECONDS!") + + else: print("No image") diff --git a/logging/__init__.py b/logging/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/logging/__pycache__/__init__.cpython-310.pyc b/logging/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3141c9ca50ab77b99d26b459c3e8f00a6accac14 GIT binary patch literal 170 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vPKO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaMNIxe( nJv}ooT|YiPGcU6wK3=b&@)n0pZhlH>PO2Tqnqnp(!NLFlN=_;l literal 0 HcmV?d00001 diff --git a/logging/__pycache__/model.cpython-310.pyc b/logging/__pycache__/model.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd8e99904bf04055202a52fb9a508f3d104cf016 GIT binary patch literal 1713 zcmZ`(OK%%D5avE4tyZ!n+j3&34{Tqk1;m!@Bu*RzMHEL)oZ6Bg$u6*$1+%22wf9vn z<<`BXzoJC}8|bC~qQ8UJp86MZC_1DZ!I!MWZ{`~ghvaa`WMRQZ@YR3Y82?>E=pRXL zepZS7O&Ae&8<-5s(OvwC9r8%%OrQLfpvv0oz;FvBoA^GWZ@6fz4HnYixhuF zq)2n`4EHWwBlfw@dPPL^fok;x-5@cm?(O>pS+oI#SrgtD_> z*7>qEY~S)bU;)9G_nqx+K5@D}-@5PYG7N%i*ue=yaGI@ z`U@~t#Y7+WpgZ-+SF13_Gpk zHj}TBFyIb}is#`tPGELeVg@4%(#h)%SV{U=itYDKSQ#i=kdTugUo5sLp(0^P!m@-F z3Gl|jSb{m_ayz+Em|)Jk;I=ya9bNw9)y{9vD1KDj)QpIfCSKRE4^ez2-ty^Z2;J)Y?igMR@JLo z)vz=`^|L{TpP`sh)6^)LQTdE`UVU8@JUJPpB~RcYu|(vv#6PE8SZRnkJUtO;mjuT= z@R<%%F?p3lo?oW0oLQDszVfoKAmJMcUr4XgnDP&`#2jhKO;uej^p0A?gKq!WJM0{E zg(;2fE9p>#;__*hB}^0|>`&mOV_!puS~>enYV&-~G`W9aJ)2d=cW}K?_BnuwtS{x} XtHv56pj@)FMXjb)Yu1CjZvOW_RPeL- literal 0 HcmV?d00001 diff --git a/logging/context.py b/logging/context.py new file mode 100644 index 0000000..0591ae6 --- /dev/null +++ b/logging/context.py @@ -0,0 +1,469 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 2nd Aug., 2024 + + OBJECTIVE: + + To provide an easy way to log all system activities by way of managing the context of what is going on. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To manage context: +import contextvars +from contextlib import contextmanager, asynccontextmanager + +# To make decorators: +from functools import wraps + +# For system-level activities: +import os + +# My utils: +from utils_v2.string import json +from utils_v2.date_time import date_time +from utils_v2.security import sanitizers +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.log import APILogModel +from utils_v2.api.response import ResponseModel + +# The needed data models: +from utils_v2.logging.model import GeneralLogModel + +# To work with date and time: +import time +import datetime + +# For random strings: +import random + +# For debugging: +import traceback +import string + +# To work with datatypes: +from types import NoneType +import pandas as pd + +# To work with Pydantic objects: +from pydantic import BaseModel + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Chars to choose from for random strings: +ALPHANUMERIC_CHARS = string.ascii_letters + string.digits + +# To capture system information: +PROCESS_ID = os.getppid() +PARENT_PROCESS_ID = os.getppid() + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def describe_exception(exc): + + """ + Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire + traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it + to the dev team over some service like WhatsApp/Telegram. + :param exc: The exception that occurred. + :return: The dict that explains the exception. + """ + + exc_desc = { + "type": type(exc).__name__, + "msg": str(exc), + "tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)] + } + + return exc_desc + + +# ***************************************************************************************************************** +# ***** **** +# *** Classes *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongoLogger: + + def __init__(self, db_conn, collection = "logs"): + + """ + This class uses an instance of 'AsyncMongo' and makes it usable as a logger. + :param db_conn: The instance of 'AsyncMongo' to use. + :param collection: The collection to write the log into. + """ + + self.__db_conn = db_conn + self.__collection = collection + + async def log(self, log_json): + + """ + Log something to the database using the connection provided when the object was made. + :param log_json: The dict to log. + :return: True if logged successfully, else False. + """ + + asyncio.create_task(self.__db_conn.insert_one( + collection = self.__collection, + document = log_json, + raise_exception = False + )) + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncLoggerContext: + + # Create the context-aware variable(s): + logger = contextvars.ContextVar("logger", default = None) + log_chain = contextvars.ContextVar("log_chain", default = None) + + @classmethod + @asynccontextmanager + async def logging_context(cls, logger, log_chain = None): + + """ + This function makes the context manager that makes the value of the log chain available to everything that is + called within the scope of the context. + :param logger: The object which is to be used to write the log. It should have a 'log' method which should take + in a dict as its input. + :param log_chain: The value of the log chain to be made available within the scope. + :return: None. + """ + + # Set the context: + token_logger = cls.logger.set(logger) + token_log_chain = cls.log_chain.set(log_chain) + + # Make the objects available within the context: + try: yield + + # Release the objects when the context is over: + finally: + cls.logger.reset(token_logger) + cls.log_chain.reset(token_log_chain) + + @staticmethod + def generate_log_id(count = 8): + return "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(min(8, count))) + + @classmethod + def get_logger(cls): + return cls.logger.get() + + @classmethod + def get_log_chain(cls): + return cls.log_chain.get() + + @staticmethod + def summarize( + value, + str_limit = 100, + expand: bool | int = False, + sensitive_keys: list[str] = None + ): + + """ + To summarize an input value to capture the essence without hoarding to much data. + :param value: Anything that you want to summarize. + :param str_limit: The max. no. of chars of a string to retain. + :param expand: Set to True for full expansion, False for no expansion, and an integer for a specific level of + expansion. Applicable on iterables and dicts. The smaller this number, the more concise the summary will be, + and vice versa. + :param sensitive_keys: The list of keys (of a dict) to obscure when summarizing. + :return: The summarized version of the input. + """ + + # If the input is a Pydantic class: + if isinstance(value, BaseModel): value = value.model_dump() + + # Check the sensitive keys: + if sensitive_keys is None: sensitive_keys = [] + + # Handle datatypes that you don't want to modify: + if isinstance(value, (int, float, bool, NoneType)): pass + + # When the value is a list or similar iterable: + elif isinstance(value, (list, tuple, set)): + if expand: + if not isinstance(expand, bool): expand -= 1 + value = [AsyncLoggerContext.summarize( + v, + expand = expand, + sensitive_keys = sensitive_keys + ) for v in value] + else: value = f"array of {len(value)} item(s)" + + # If the value is a dict: + elif isinstance(value, dict): + if expand: + if not isinstance(expand, bool): expand -= 1 + value = { + k: AsyncLoggerContext.summarize( + v, + expand = expand, + sensitive_keys = sensitive_keys + ) if k not in sensitive_keys else "********" + for k, v in value.items() + } + else: value = f"object of {len(value.keys())} field(s) [{', '.join(value.keys())}]" + + # When a dataframe is passed: + elif isinstance(value, pd.DataFrame): + cols = value.columns.to_list() + value = f"table with {len(cols)} col(s) [{', '.join(cols)}] and {len(value)} row(s)" + str_limit = 999 + + # If the input is some form of non-standard object: + else: value = str(value) + + # Handle strings: + if isinstance(value, str): + if len(value) > str_limit: value = value[:str_limit] + "..." + + # Done here: + return value + + @classmethod + def log_it( + cls, + api_version: str = None, + project: str = None, + log_type: str = None, + operation: str = None, + log_input: bool | int = True, + log_output: bool | int = True, + sensitive_keys: list = None + ): + + """ + A decorator factor that can be used to log the results of functions automatically. + :param api_version: A string that indicates the version code of the function being decorated. + :param project: A hint about which project is being worked on. + :param log_type: A hint about which module is being worked on. + :param operation: A hint about which action in a particular module is being worked on. + :param log_input: Set to True to capture everything that went into the function, False to capture the least + info, and set it to an integer to capture a certain depth of the input (applicable on iterables and dicts. + :param log_output: The same as 'log_input', but applicable to the response from the function. + :param sensitive_keys: Keys of a dict whose values must be obscured even if that depth is being captured. + :return: A decorator with the configuration. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Make variables and extract available info.: + exception = None + response = None + request_ts = date_time.get_current_utc_date_time() + start_ts = time.perf_counter() + cpu_start_ts = time.process_time() + + # Execute the function that is being wrapped: + try: response = await func(*args, **kwargs) + except Exception as exc: exception = exc + + # Do the next steps only if within the logging context: + if cls.get_logger() is not None: + + # Create the log: + if not args: args = None + if not kwargs: kwargs = None + func_log = GeneralLogModel( + pid = PROCESS_ID, + ppid = PARENT_PROCESS_ID, + project = project, + log = log_type or func.__name__, + operation = operation or func.__name__, + apiVer = api_version, + logId = cls.generate_log_id(), + logChain = cls.get_log_chain(), + ts = request_ts, + tat = time.perf_counter() - start_ts, + cpuTime = time.process_time() - cpu_start_ts, + func = func.__name__, + args = cls.summarize(args, expand = log_input, sensitive_keys = sensitive_keys), + kwargs = cls.summarize(kwargs, expand = log_input, sensitive_keys = sensitive_keys), + exception = None if exception is None else describe_exception(exception), + response = cls.summarize(response, expand = log_output, sensitive_keys = sensitive_keys), + ).model_dump() + + # Write the log: + await cls.get_logger().log(func_log) + + # Done here: + if exception is not None: raise exception + return response + + return wrapper + + return decorator + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + from utils_v2.database.async_mongo_v2 import AsyncMongo + + @AsyncLoggerContext.log_it( + api_version = "0.0.1", + project = "testProj", + log_type = "work", + operation = "someWork", + log_input = True, + log_output = True, + sensitive_keys = None + ) + async def some_work(*args, **kwargs): + print("SOME WORK:", AsyncLoggerContext.get_log_chain()) + await asyncio.sleep(max(2.0 * random.random(), 1.0)) + total = sum(args) + return total + + @AsyncLoggerContext.log_it( + api_version = "0.0.1", + project = "testProj", + log_type = "work", + operation = "moreWork", + log_input = True, + log_output = True, + sensitive_keys = ["password", "sessionToken"] + ) + async def more_work(*args, **kwargs): + print("MORE WORK:", AsyncLoggerContext.get_log_chain()) + await asyncio.sleep(max(2.0 * random.random(), 1.0)) + return {"success": True, "sessionToken": "1234567890"} + + @AsyncLoggerContext.log_it( + api_version = "0.0.1", + project = "testProj", + log_type = "work", + operation = "moreWork", + log_input = True, + log_output = True, + sensitive_keys = None + ) + async def last_work(*args, **kwargs): + print("LAST WORK:", AsyncLoggerContext.get_log_chain()) + await asyncio.sleep(max(2.0 * random.random(), 1.0)) + + async def main(chain = None): + + # Connect to MongoDB: + mongo = AsyncMongo( + connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external", + database_name = "converse", + max_connections = 10, + debug = True + ) + + # Convert the connection to a logger instance that can be injected + # into the context as a dependency: + mongo_logger = AsyncMongoLogger( + db_conn = mongo, + collection = "logs" + ) + + # Initialize the context: + async with AsyncLoggerContext.logging_context( + logger = mongo_logger, + log_chain = chain + ): + + # Run some functions within the context: + await some_work(1, 2, 3, 4, 5) + await more_work(username = "john.doe@domain.com", password = "mySecretPass") + + # Run something outside the context: + await last_work() + + # async def multi_main(): + # tasks = [ + # main(chain = "kPRwXdItb1"), + # main(chain = "456") + # ] + # await asyncio.gather(*tasks) + + asyncio.run(main(chain = "00wGHRFYPY123")) diff --git a/logging/model.py b/logging/model.py new file mode 100644 index 0000000..355d07e --- /dev/null +++ b/logging/model.py @@ -0,0 +1,131 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 11th Oct., 2024 + + OBJECTIVE: + + To have a structure for the logs maintained for regular function calls. + This is different from the logs maintained for API calls. + + REFERENCES: + + 01. https://chatgpt.com/share/6708b6c9-6ba4-800f-9ea9-00ec35067512 + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# System-level activities: +import distro +import socket +import platform + +# For data-modelling: +from pydantic import BaseModel, Field +from typing import Any, Optional, List, Literal + +# To work with date and time: +import datetime + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Info for logging that will stay constant during runtime: +SERVER_HOSTNAME = str(socket.gethostname()) +PLATFORM_INFO = platform.uname() +HOST_OS = str(distro.name(True)) +HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class GeneralLogModel(BaseModel): + + # To identify the machine the code is running on. + # DO NOT MODIFY THESE: + hostname: str = SERVER_HOSTNAME + os: str = HOST_OS + cpu: str = HOST_CPU + # Can modify these: + pid: Optional[Any] = None + ppid: Optional[Any] = None + + # To identify the project and actions: + project: Optional[str] = None + log: str + operation: Optional[str] = None + apiVer: Optional[str] = None + logId: Optional[str] = None + logChain: Optional[str] = None + + # Timing metrics: + ts: datetime.datetime + tat: float + cpuTime: float + + # To understand the inputs: + func: str + args: Optional[Any] = None + kwargs: Optional[Any] = None + + # To understand the outputs: + exception: Optional[Any] = None + response: Optional[Any] = None + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/mail/__init__.py b/mail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mail/async_mail.py b/mail/async_mail.py new file mode 100644 index 0000000..839873c --- /dev/null +++ b/mail/async_mail.py @@ -0,0 +1,440 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 16th Jul, 2024 + + OBJECTIVE: + + To be able to send out mails from code. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For working with mails: +import aiosmtplib +from email.mime.multipart import MIMEMultipart +from email.mime.image import MIMEImage +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email import encoders + +# My utils: +from utils import rate_limit_utils + +# Common: +from shared.statuses import StatusCodes + +# For random strings: +import string +import random + +# For system-level activities: +import os + +# For working with files in RAM: +import io + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class MailMessage: + + def __init__(self, to_email, subject): + + """ + Create an instance of the message that you would like to send. + :param to_email: The EMail ID of th recipient. + :param subject: The subject of the mail. + """ + + self.message = MIMEMultipart() + self.message["To"] = to_email + self.message["Subject"] = subject + + def add_text(self, text): + + """ + Add plain-text to the mail body. + :param text: The text to add to the mail body. + :return: None. + """ + + self.message.attach(MIMEText(text, "plain")) + + def add_html(self, html_text): + + """ + Add HTML text to the mail body. + :param html_text: The HTML text to add to the mail body. + :return: None. + """ + + self.message.attach(MIMEText(html_text, "html")) + + def add_inline_image(self, image_file, content_id = None): + + """ + Add an inline image to the body of the mail. + NOTE: This is NOT the same as sending an image as an attachment. + :param image_file: The image data to attach to the mail body. + :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not + specified, I will generate a random string. You may write a custom value here if you know what you are + doing. For most use cases, please ignore this field. + :return: None. + """ + + # Read the image as bytes: + image_bytes = None + if type(image_file) is str: + with open(image_file, "rb") as opened_image_file: + image_bytes = opened_image_file.read() + if type(image_file) is io.BytesIO: + image_file.seek(0) + image_bytes = image_file.getvalue() + + # Declare the part to be attached to the multipart message: + if image_bytes is not None: + + # Create the HTML block if the image pointer is blank: + if content_id is None: + content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) + self.add_html(f""" + + +

    + + + """) + + # Then add the image: + image_part = MIMEImage(image_bytes) + image_part.add_header("Content-ID", f"<{content_id}>") + self.message.attach(image_part) + + def add_attachment(self, attachment_file, file_name = None): + + # Declare the part to be attached to the multipart message: + part = MIMEBase("application", "octet-stream") + + # If the attachment is a file stored in the local disk: + if type(attachment_file) is str: + file_name = file_name or os.path.split(attachment_file)[-1] + with open(attachment_file, "rb") as attachment: + part.set_payload(attachment.read()) + + # If the file is held in RAM: + if type(attachment_file) is io.BytesIO(): + attachment_file.seek(0) + part.set_payload(attachment_file.read()) + + # Encode and attach the file: + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + f"attachment; filename= {file_name}", + ) + self.message.attach(part) + + def get_message(self): + return self.message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncMailClient: + + # Constants: + SMTP_TLS_PORT = 587 + SMTP_SSL_PORT = 465 + + # variables: + __smtp = None + + def __init__( + self, + email, + password, + server, + port = 587, + rate_limiters = None, + wait_for_turn = True, + debug = True, + debug_prefix = "Mail (C) | " + ): + + """ + Set up the mail client. + :param email: The Email ID to use when sending out mails. + :param password: The password of the EMail ID that is being used. + :param server: The EMail server. + :param port: The port number to connect to the host. + :param rate_limiters: The rate limiters to use. Must have "get_turn" and "has_turn" methods. "get_turn" method + must wait for the turn, and "has_turn" method must only check if a turn is available. + :param wait_for_turn: To wait for turn if the rate limit has been exceeded, or to return with failure. + :param debug: Whether, or not, you want to print debugging messages. + :param debug_prefix: The prefix to identify the debugging messages. + """ + + # Initialize the debugger: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # Note down the credentials and other details: + self.__email = email + self.__password = password + self.__server = server + self.__port = port + self.__rate_limiters = rate_limiters if type(rate_limiters) is list else ([rate_limiters] if rate_limiters is not None else []) + self.__wait_for_turn = wait_for_turn + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + async def login(self): + + """ + To connect to the mail server and authenticate the user. + :return: True if authenticated, else False. + """ + + # Initialize the SMTP connection, + # and return with success if all goes well: + try: + self.__smtp = aiosmtplib.SMTP( + hostname = self.__server, + port = self.__port, + use_tls = False, + start_tls = False + ) + await self.__smtp.connect() + await self.__smtp.starttls() + await self.__smtp.login(self.__email, self.__password) + return True + + # Return with failure if something goes wrong: + except Exception as exception: + self.__printer(exception) + try: await self.__smtp.quit() + except Exception as exception: self.__printer(exception) + self.__smtp = None + return False + + async def ensure_connection(self): + + """ + Can be run before the sending operation to confirm that we are yet connected to the server. + If not connected, this code will reattempt to connect and log-in. + :return: True if connected, else False. + """ + + # If the login had failed initially, the object will be set to null. + # In such a case, we make an attempt to login: + if self.__smtp is None: + return await self.login() + + # If the login was successful, we check if the connection is active. + # If not, we try to re-login: + if self.__smtp.is_connected: + return True + else: + try: + await self.__smtp.connect() + await self.__smtp.starttls() + await self.__smtp.login(self.__email, self.__password) + return True + except Exception as exception: + self.__printer(exception) + try: await self.__smtp.quit() + except Exception as exception: self.__printer(exception) + self.__smtp = None + return False + + async def logout(self): + + """ + Closes the connection to the SMTP client. + :return: True by default. + """ + + if self.__smtp is not None: + try: await self.__smtp.quit() + except Exception as exception: self.__printer(exception) + self.__smtp = None + return True + + async def send(self, mail: MailMessage): + + """ + Send out the mail. + :param mail: The instance of 'MailMessage' with all the content populated. + :return: A dict with 'success' and 'message'. + """ + + # Return with failure if we aren't connected, + # and our attempt to (re)connect fails: + if not await self.ensure_connection(): + return { + "success": False, + "message": "login failed" + } + + # Comply with the rate-limit: + for rate_limiter in self.__rate_limiters: + if not self.__wait_for_turn: + if not await rate_limiter.has_turn(): return False + got_turn = await rate_limiter.get_turn() + if not got_turn: + return { + "success": False, + "message": "rate-limit wait timeout" + } + + # Try to send the message: + try: + mail.message["From"] = self.__email + await self.__smtp.send_message(mail.message) + return { + "success": True, + "message": "mail sent" + } + + # If something goes wrong: + except Exception as exception: + self.__printer(exception) + return { + "success": False, + "message": str(exception) + } + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + from utils_v2.string import json + from utils_v2.mail.mail_message import MailMessage + + async def test(): + + rate_lim = rate_limit_utils.TokenBucket( + rate_limit = 1, + seconds = 60.0, + ) + + mail_client = AsyncMailClient( + email = "sender@gmail.com", + password = "secret_password", + server = "smtp.gmail.com", + rate_limiters = rate_lim + ) + + my_mail = MailMessage( + to_email = "recipient@gmail.com", + subject = "Bhopli is the best!" + ) + my_mail.add_html( + """ + + + + + Sample HTML String + + + +

    Hello, Bhopli!

    +

    Bhopli is the best, most well-behaved cat in the known universe.

    + + + """ + ) + my_mail.add_text("This is how you should pet her 👇") + my_mail.add_inline_image(r"/path/to/image/cat_petting.png") + my_mail.add_attachment(r"/path/to/file/sample_label.pdf") + + await mail_client.login() + result = await mail_client.send(my_mail) + print("MAIL RESULT:", json.to_json_string(result)) + await mail_client.logout() + + asyncio.run(test()) diff --git a/mail/mail_message.py b/mail/mail_message.py new file mode 100644 index 0000000..5887aae --- /dev/null +++ b/mail/mail_message.py @@ -0,0 +1,268 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 16th Jul, 2024 + + OBJECTIVE: + + To be able to send out mails from code. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For working with mails: +import aiosmtplib +from email.mime.multipart import MIMEMultipart +from email.mime.image import MIMEImage +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email import encoders + +# My utils: +from utils import rate_limit_utils + +# Common: +from shared.statuses import StatusCodes + +# For random strings: +import string +import random + +# For system-level activities: +import os + +# For working with files in RAM: +import io + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class MailMessage: + + def __init__(self, to_email, subject): + + """ + Create an instance of the message that you would like to send. + :param to_email: The EMail ID of th recipient. + :param subject: The subject of the mail. + """ + + self.message = MIMEMultipart() + self.message["To"] = to_email + self.message["Subject"] = subject + + def add_text(self, text): + + """ + Add plain-text to the mail body. + :param text: The text to add to the mail body. + :return: None. + """ + + self.message.attach(MIMEText(text, "plain")) + + def add_html(self, html_text): + + """ + Add HTML text to the mail body. + :param html_text: The HTML text to add to the mail body. + :return: None. + """ + + self.message.attach(MIMEText(html_text, "html")) + + def add_inline_image(self, image_file, content_id = None): + + """ + Add an inline image to the body of the mail. + NOTE: This is NOT the same as sending an image as an attachment. + :param image_file: The image data to attach to the mail body. + :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not + specified, I will generate a random string. You may write a custom value here if you know what you are + doing. For most use cases, please ignore this field. + :return: None. + """ + + # Read the image as bytes: + image_bytes = None + if type(image_file) is str: + with open(image_file, "rb") as opened_image_file: + image_bytes = opened_image_file.read() + if type(image_file) is io.BytesIO: + image_file.seek(0) + image_bytes = image_file.getvalue() + + # Declare the part to be attached to the multipart message: + if image_bytes is not None: + + # Create the HTML block if the image pointer is blank: + if content_id is None: + content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) + self.add_html(f""" + + +

    + + + """) + + # Then add the image: + image_part = MIMEImage(image_bytes) + image_part.add_header("Content-ID", f"<{content_id}>") + self.message.attach(image_part) + + def add_attachment(self, attachment_file, file_name = None): + + # Declare the part to be attached to the multipart message: + part = MIMEBase("application", "octet-stream") + + # If the attachment is a file stored in the local disk: + if type(attachment_file) is str: + file_name = file_name or os.path.split(attachment_file)[-1] + with open(attachment_file, "rb") as attachment: + part.set_payload(attachment.read()) + + # If the file is held in RAM: + if type(attachment_file) is io.BytesIO(): + attachment_file.seek(0) + part.set_payload(attachment_file.read()) + + # Encode and attach the file: + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + f"attachment; filename= {file_name}", + ) + self.message.attach(part) + + def get_message(self): + return self.message + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + from utils import json_utils + from utils_v2.mail.async_mail import AsyncMailClient + + async def test(): + + rate_lim = rate_limit_utils.TokenBucket( + rate_limit = 1, + seconds = 60.0, + ) + + mail_client = AsyncMailClient( + email = "sender@gmail.com", + password = "secret_password", + server = "smtp.gmail.com", + rate_limiters = rate_lim + ) + + my_mail = MailMessage( + to_email = "recipient@gmail.com", + subject = "Bhopli is the best!" + ) + my_mail.add_html( + """ + + + + + Sample HTML String + + + +

    Hello, Bhopli!

    +

    Bhopli is the best, most well-behaved cat in the known universe.

    + + + """ + ) + my_mail.add_text("This is how you should pet her 👇") + my_mail.add_inline_image(r"/path/to/image/cat_petting.png") + my_mail.add_attachment(r"/path/to/file/sample_label.pdf") + + await mail_client.login() + result = await mail_client.send(my_mail) + print("MAIL RESULT:", json_utils.to_json_string(result)) + await mail_client.logout() + + asyncio.run(test()) diff --git a/network/__init__.py b/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/network/ip.py b/network/ip.py new file mode 100644 index 0000000..134078f --- /dev/null +++ b/network/ip.py @@ -0,0 +1,173 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 12th Jul., 2024 + + OBJECTIVE: + + To provide an easy way to get geolocation information of an IP address. + + REFERENCES: + + 1) https://medium.com/@tubelwj/how-to-retrieve-ip-geolocation-information-in-python-929e15041e3e + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For working with IP Addresses: +import ipaddress + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def ipv4_to_int(ip_string): + + """ + Converts an IP (v4) string to an integer value. + :param ip_string: The IP address (v4) that you want to convert to integer format. + :return: An integer representation of the IP (v4) address. + """ + + ip_numerical = int(ipaddress.IPv4Address(ip_string)) + return ip_numerical + + +# --------------------------------------------------------------------------------------------------------------------- + + +def int_to_ipv4(ip_numerical): + + """ + Interprets the IP (v4) value from the given integer value. + :param ip_numerical: The integer value that represents an IP (v4) address. + :return: + """ + + ip_string = str(ipaddress.IPv4Address(ip_numerical)) + return ip_string + + +# --------------------------------------------------------------------------------------------------------------------- + + +def ipv4_to_bin(ip_string): + + """ + Converts an input IP (v4) address to the binary string that represents the 32 bits. + :param ip_string: The IP (v4) string in a format like "192.168.0.1" + :return: The binary representation (as a string) of the input IP address. + """ + + ip_binary = bin(int(ipaddress.IPv4Address(ip_string)))[2:].zfill(32) + return ip_binary + + +# --------------------------------------------------------------------------------------------------------------------- + + +def bin_to_ipv4(ip_binary): + + """ + Interprets the IP (v4) value from the given binary string. + :param ip_binary: The string of 1s and 0s that represents the IP (v4) address. + :return: The IP (v4) address as a string. + """ + + ip_string = str(ipaddress.IPv4Address(int(ip_binary, 2))) + return ip_string + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_ipv4_range(ip_string, as_string = True): + + """ + Given a network description in the format "88.95.100.128/25", this function tells you the first and last IP + addresses of that network. Useful for determining if an IP address lies in a network. + :param ip_string: The input network description in the format "88.95.100.128/25" + :param as_string: To select between integer and string formats for the IP range output. + :return: The first and last IP addresses of the input network, and the count. + """ + + # Extract the components of the string: + ip_components = ip_string.split("/") + ip_addr = ipv4_to_int(ip_components[0]) + ip_bits = int(ip_components[1]) + + # Convert the mask number to binary representation: + ip_mask = (1 << ip_bits) - 1 + ip_mask = ip_mask << (32 - ip_bits) + inv_ip_mask = (~ip_mask) & 0xFFFF + + # Figure out the start and end IP addresses: + start_ip = ip_addr & ip_mask + end_ip = ip_addr | inv_ip_mask + count = end_ip - start_ip + 1 + + # If the IPs are needed as strings, we perform the conversion: + if as_string: + start_ip = int_to_ipv4(start_ip) + end_ip = int_to_ipv4(end_ip) + + # Done here: + return start_ip, end_ip, count + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + print(ipv4_to_int("255.255.255.255")) + print(ipv4_to_int("x.x.x.x")) diff --git a/network/ping.py b/network/ping.py new file mode 100644 index 0000000..39e42c7 --- /dev/null +++ b/network/ping.py @@ -0,0 +1,218 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 28th Jun, 2024 + + OBJECTIVE: + + To provide a way to ping a server and get the traceroute dump. + + REFERENCES: + + 1) https://www.geeksforgeeks.org/traceroute-implementation-on-python/ + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +# --- +import sys +sys.path.append(".") +sys.path.append("..") + +# Utils: +# --- +from utils import json_utils +from utils import time_utils +from utils import regex_utils + +# For networking: +# --- +import socket +from scapy.all import * + +# For running the script from the terminal: +# --- +import argparse + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_name_and_addr(destination): + + # If the user provided the IP address: + # --- + if ( + regex_utils.match(destination, regex_utils.REGEX_IPV4) or + regex_utils.match(destination, regex_utils.REGEX_IPV6) + ): + try: destination_name = socket.gethostbyaddr(destination)[0] + except Exception as exception: destination_name = "*" + destination_ip = destination + + # If the provided destination was the domain name: + # --- + else: + destination_name = destination + try: destination_ip = socket.gethostbyname(destination) + except Exception as exception: destination_ip = "*" + + # Done here: + # --- + return destination_name, destination_ip + + +# --------------------------------------------------------------------------------------------------------------------- + + +def tracert( + destination, + max_hops = 30, + timeout = 2.0, + port = 33434 +): + + # Initialize the variables: + # --- + destination_name, destination_ip = get_name_and_addr(destination) + full_trace = [] + ttl = 1 + + # Keep noting hops till the limit is reached: + # --- + while ttl <= max_hops: + + # Create a JSON for this stage: + # --- + this_hop = { + "destAddr": destination_ip, + "destName": destination_name, + "hopNo": ttl - 1, + "isDest": False, + "hopAddr": None, + "hopName": None, + "ts": None + } + + # Create the IP and UDP headers and combine them: + # --- + ip_packet = IP(dst = destination, ttl = ttl) + udp_packet = UDP(dport = port) + trace_packet = ip_packet / udp_packet + + # Send the packet and receive a reply and note down the timestamp: + # --- + reply = sr1(trace_packet, timeout = timeout, verbose = 0) + this_hop["ts"] = time_utils.get_current_utc_datetime(as_string = True) + + # No response: + # --- + if reply is None: this_hop["hopAddr"] = this_hop["hopName"] = "*" + + # If some response was received, we note the values and break out if this was the destination hop: + # --- + else: + this_hop["hopName"], this_hop["hopAddr"] = get_name_and_addr(f"{reply.src}") + if reply.type == 3: + this_hop["isDest"] = True + full_trace.append(this_hop) + break + + # Carry on to the next hop: + # --- + full_trace.append(this_hop) + ttl += 1 + + # Done here: + # --- + return full_trace + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description = "Traceroute Implementation in Python!") + + parser.add_argument( + "dest", + help = "Destination (Name or IP address)." + ) + parser.add_argument( + "-m", + "--max-hops", + type = int, + default = 30, + help = + "Maximum number of hops (default: 30)." + ) + parser.add_argument( + "-t", + "--timeout", + type = float, + default = 2.0, + help = "Timeout for each packet in seconds (default: 2.0)." + ) + parser.add_argument( + "-p", + "--port", + type = int, + default = 33434, + help = "Timeout for each packet in seconds (default: 33434)." + ) + + args = parser.parse_args() + + trace = tracert( + destination = args.dest, + max_hops = args.max_hops, + timeout = args.timeout, + port = args.port + ) + print("TRACE:") + print(json_utils.to_json_string(trace)) diff --git a/pdf/__init__.py b/pdf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pdf/pdf_maker.py b/pdf/pdf_maker.py new file mode 100644 index 0000000..fb97cdd --- /dev/null +++ b/pdf/pdf_maker.py @@ -0,0 +1,1642 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 13th Aug., 2024 + + OBJECTIVE: + + To generate custom PDFs. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For conversion to PDF: +from PyPDF2 import PdfMerger, PdfReader, PdfWriter, Transformation + +# For barcode and QR code generation: +import xml.etree.ElementTree as ET +from barcode import Code128 +from barcode.writer import ImageWriter, SVGWriter +import segno +from PIL import Image, ExifTags, ImageFilter + +# System-level activities: +import os +import io +import inspect +import contextlib + +# For calculations: +import math + +# For PDF generation: +import fitz +from reportlab.pdfgen import canvas +from reportlab.lib.units import mm, cm, inch +from reportlab.lib.styles import ParagraphStyle +from reportlab.platypus import Paragraph +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.lib.utils import ImageReader +from reportlab.graphics import renderSVG + +# My utils: +from utils_v2.string import json +from utils_v2.system import files +from utils_v2.string import regex + +# For debugging: +from icecream import IceCreamDebugger +import traceback + +# For random string generation: +import random +import string + +# To validate the font file: +from fontTools import ttLib + +# To make http calls: +import requests +import httpx + +# To work with base64 coding: +import base64 + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make async API calls: +max_connections = 5 +limits = httpx.Limits( + max_keepalive_connections = max_connections, + max_connections = max_connections, + keepalive_expiry = 3600 +) +http_client = httpx.AsyncClient( + limits = limits, + follow_redirects = False, + timeout = httpx.Timeout(None) +) + +# Headers for downloading files: +HEADERS_DOWNLOAD = { + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate" +} + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class PDFMaker: + + def __init__( + self, + width, + height, + unit = "mm", + dpi = 72, + debug = True, + debug_prefix = "PDF (M) | ", + raise_exception = True, + **kwargs + ): + + """ + IMPORTANT: To keep things intuitively oriented (left-to-right, and top-to-bottom), the internal bottom-up + coordinates have been flipped mathematically. All measurements start from the top-left corner. Moreover, I have + ensured that all numbers follow the same units. So, if you set your unit to "inch", everything will be measured + in the same unit. I hope this creates maximum consistency for you. + A NOTE ON IMAGES: All images will be measured in pixels when operating on them directly (like cropping and + resizing), but will be measured in the 'unit' of the canvas when placing it in the PDF. + A NOTE OF FONT SIZES: Font sizes (and their line spacings) are measured in 'points'. As per the standard, there + are 72 'points' in one inch. While everything else in this utility uses the unit of measurement defined in the + 'unit' param, fonts need to be specified in 'points' to maintain compatibility with HTML rendering engines to + work with well-formatted paragraphs. + WARNING: I STRONGLY RECOMMEND NOT USING DPI AS A SETTING. THE PDF FORMAT ITSELF DOESN'T HAVE ANY PROVISION FOR + PIXEL DENSITY. THIS SETTING HAS BEEN MADE AVAILABLE FOR PLACEBO EFFECT WHEN CLIENTS ASK FOR VERY HIGH RESOLUTION + FILES WITHOUT WANTING TO UNDERSTAND ANY TECHNICALITY. USE WITH CAUTION. + :param width: [int|float] The width of the page (in the specified unit). + :param height: [int|float] The height of the page (in the specified unit). + :param unit: [str] The unit ("mm", "cm", "inch") to use. + :param dpi: [int|float] The pixel density. AVOID USING THIS. + :param debug: [bool] Whether, or not, you want to print debugging messages. Can be changed on the fly. + :param debug_prefix: [str] The prefix to show in debugging messages. + :param raise_exception: If set to True, any failure will immediately stop the PDF building process. If set to + False, the PDF building will continue by ignoring failed steps. Set to True by default for an all-or-nothing + output behaviour. + :param kwargs: Just a placeholder, does nothing for now (20240904). + """ + + # For debugging: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # Note down the exception-handling choice: + self.__raise_exception = raise_exception + + # We pick the unit from the user's choice: + self.__unit = { + "mm": mm, + "cm": cm, + "inch": inch + }[unit] + + # To know the canvas size multipliers, we need a few parameters: + # 01. The DPI Factor is the multiplier to apply to achieve the target DPI considering a default DPI of 72, + # 02. The Unit Factor is the multiplier to apply to the unit to achieve a high-res result. + self.__dpi_factor = (dpi / 72.0) + self.__unit_factor = self.__unit * self.__dpi_factor + + # Now we create the canvas: + self.__pdf_in_ram = io.BytesIO() + self.__canvas_width = width * self.__unit_factor + self.__canvas_height = height * self.__unit_factor + self.__canvas = canvas.Canvas( + self.__pdf_in_ram, + pagesize = (self.__canvas_width, self.__canvas_height) + ) + + def enable_debug(self): + + """ + To enable the debugging text in the terminal. + :return: None. + """ + + self.__printer.enable() + + def disable_debug(self): + + """ + To disable the debugging text in the terminal. + :return: None. + """ + + self.__printer.disable() + + # ┏┓┓ ┓ ┳┳ •┓• • + # ┗┓┣┓┏┓┏┓┏┓┏┫ ┃┃╋┓┃┓╋┓┏┓┏ + # ┗┛┛┗┗┻┛ ┗ ┗┻ ┗┛┗┗┗┗┗┗┗ ┛ + + @staticmethod + async def download_from_url(url, follow_redirects = False): + + """ + Download a file from a URL. + :param url: The URL to download the file from. + :param follow_redirects: Whether, or not, to follow along with any redirects when downloading the file. + :return: The downloaded file or None. + """ + + file = io.BytesIO((await http_client.get( + url, + headers = HEADERS_DOWNLOAD, + follow_redirects = follow_redirects + )).content) + file.seek(0) + return file + + @staticmethod + def read_to_ram(file_path): + + """ + Reads a file into a BytesIO object in RAM. + NOTE: The method doesn't itself verify if the file is a PDF. + :param file_path: The path to the file on disk. + :return: The file in a BytesIO object. + """ + + with open(file_path, "rb") as file: + file_data = file.read() + file_in_ram = io.BytesIO(file_data) + file_in_ram.seek(0) + return file_in_ram + + def __parse_color(self, input_color): + + """ + The reportlab 'reportlab' needs the colour to be specified in an array of RGS values where each value is a + number in the range 0-1. + :param input_color: Either a hex string or an array of numbers. + :return: An array that can be used by 'reportlab'. + """ + + # Start by assuming failure: + processed_color = None + + # If the input color is a string, we convert from hex to an array of hex codes: + if isinstance(input_color, str): + input_color = input_color.lstrip("#") + color_tuple = ( + input_color[0:2].upper(), + input_color[2:4].upper(), + input_color[4:6].upper() + ) + input_color = color_tuple + + # If the input color is a list/tuple of hex-strings: + if isinstance(input_color, (list, tuple)): + try: + if all(isinstance(item, str) for item in input_color): + processed_color = [int(item, 16) / 255.0 for item in input_color] + elif all(isinstance(item, (float, int)) for item in input_color): + processed_color = [max(min(item, 255.0), 0.0) / 255.0 for item in input_color] + except: + if self.__raise_exception: raise + + # Done here: + return processed_color + + def __parse_x(self, *args): + + """ + To adjust the horizontal coordinates for the canvas. + :param args: As many horizontal coordinates as you would like to adjust. + :return: The adjusted coordinates. + """ + + if len(args) == 0: return None + parsed_x = [x * self.__unit_factor for x in args] + parsed_x = [x if x >= 0 else x + self.__canvas_width for x in parsed_x] + return tuple(parsed_x) if len(parsed_x) > 1 else parsed_x[0] + + def __parse_y(self, *args, flip = False): + + """ + To adjust the vertical coordinates for the canvas. + :param args: As many vertical coordinates as you would like to adjust. + :param flip: To convert from bottom-up system to top-down system. + :return: The adjusted coordinates. + """ + + if len(args) == 0: return None + parsed_y = [y * self.__unit_factor for y in args] + parsed_y = [y if y >= 0 else y + self.__canvas_height for y in parsed_y] + if flip: parsed_y = [self.__canvas_height - y for y in parsed_y] + return tuple(parsed_y) if len(parsed_y) > 1 else parsed_y[0] + + def __parse_font_size(self, *args): + + """ + Adjust the font size. + :param args: The font sizes to adjust + :return: The adjusted font-size. + """ + + if len(args) == 0: return None + parsed_font_sizes = args + return parsed_font_sizes if len(parsed_font_sizes) > 1 else parsed_font_sizes[0] + + @staticmethod + def _parse_coordinates_for_rotation( + original_x, + original_y, + angle + ): + + """ + Adjusts the x and y coordinates to compensate for the effect of a rotation operation. + :param original_x: The x coordinate before applying rotation. + :param original_y: The y coordinate before applying rotation. + :param angle: The angle (in degrees) by which the rotation will be applied. + :return: The adjusted x and y coordinates that compensate for the effect of the canvas's rotation. + """ + + # Get the angle in radians: + angle_radians = math.radians(angle) + angle_cos = math.cos(angle_radians) + angle_sin = math.sin(angle_radians) + + # Compute the inverse rotation: + adjusted_x = original_x * angle_cos + original_y * angle_sin + adjusted_y = -original_x * angle_sin + original_y * angle_cos + + # Done here: + return adjusted_x, adjusted_y + + # ┳ ┏┓ • + # ┃┏┓┏╋┏┓┏┓┏┏┓ ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏ + # ┻┛┗┛┗┗┻┛┗┗┗ ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛ + # ┛ + + @property + def canvas(self): + + """ + Use this to fetch just the canvas with all the inputs applied to it. + :return: The canvas (page) that is being used currently in the instance. + """ + + return self.__canvas + + @property + def unit_factor(self): + + """ + The scaling factor after taking into consideration the physical unit of measurement being use (e.g.: "mm"). + :return: The unit-factor. + """ + + return self.__unit_factor + + @property + def dpi_factor(self): + + """ + The scaling factor derived from comparing the default DPI of 72 and a custom DPI specified when creating the + instance. Since the PDF format doesn't have a native DPI configuration, this scaling factor will help in + adjusting the sizes of whatever you wish to add to the canvas. + :return: The DPI scaling factor. + """ + + return self.__dpi_factor + + @property + def shape(self): + + """ + an array of the dimensions (width, height) of the canvas. + :return: The width and height of the canvas. + """ + + return self.__canvas_width, self.__canvas_height + + # ┏┓ ┏┓ ┏┓•┓ + # ┃ ┏┓┏┓┓┏┏┓┏ ┣╋ ┣ ┓┃┏┓ + # ┗┛┗┻┛┗┗┛┗┻┛ ┗┻ ┻ ┗┗┗ + + def save(self, path: str = None): + + """ + Save the Canvas as a PDF either to RAM or a file on disk. + :param path: The path you want to save the PDF to on disk. If not specified, the PDF will be saved to RAM and + returned as a BytesIO object. + :return: A BytesIO object, or True/False if a path is supplied and the file gets saved. + """ + + # Save the changes to the file in RAM. + self.__canvas.save() + self.__pdf_in_ram.seek(0) + + # If the user has not specified a path, return the buffer: + if path is None: return self.__pdf_in_ram + + # In case a path was specified, + # We try to save the file in the specified path: + try: + with open(path, "wb") as file: file.write(self.__pdf_in_ram.getvalue()) + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def next_page(self): + + """ + To start working on the next page. + The library that is being used doesn't support moving back and forth between pages in any random order, it, + instead, needs to go in a sequential fashion from one page to the next. + :return: None. + """ + + self.__canvas.showPage() + + # ┏┓ + # ┣ ┏┓┏┓╋┏ + # ┻ ┗┛┛┗┗┛ + + @staticmethod + async def register_font_from_url(name, url, follow_redirects = False): + + """ + Download a font from a URL and then register it for use. + :param name: The name you would later refer to the font by. + :param url: The URL to download the font from. + :param follow_redirects: Whether, or not, to follow along with any redirects when downloading the file. + :return: True if registered, else False. + """ + + try: return PDFMaker.register_font( + name, + await PDFMaker.download_from_url(url, follow_redirects = follow_redirects) + ) + except Exception as exception: + return False + + @staticmethod + def register_fonts_from_directory(directory): + + """ + Register all the fonts from a directory to be used later in the PDF. + :param directory: The directory that has the TTF files to register. + :return: A dict describing the fonts that were registered. + """ + + # List out all the files in the folder: + all_files = files.list_files(directory, full_path = True) + + # Keep only TTF files and make a name-to-path map: + font_map = { + os.path.split(file)[-1].split(".")[0].lower(): file + for file in all_files if file.lower().rstrip().endswith(".ttf") + } + + # Register the custom fonts: + registered_fonts = [] + for font_name, font_path in font_map.items(): + if PDFMaker.register_font(font_name, font_path): + registered_fonts.append({"name": font_name, "file": font_path}) + + # Done here: + return registered_fonts + + @staticmethod + def register_font(name, file): + + """ + Register one TTF font from either a BytesIO object or a path on the disk. + :param name: The name of the font as you would like to use when invoking it. + :param file: The font file (TTF format). + :return: True if registered, else False. + """ + + try: + + if isinstance(file, io.BytesIO): file.seek(0) + pdfmetrics.registerFont(TTFont(name, file)) + return True + + except Exception as exception: + return False + + @staticmethod + def get_font_height(font, size, count_ascent = True, count_descent = True): + + """ + Compute the height of the font for the given size. + :param font: [str] The name of the font (as registered). + :param size: [int|float] The size of the font (in points). + :param count_ascent: [bool] Whether, or not, you would like to consider the ascent of the font. + :param count_descent: [bool] Whether, or not, you would like to consider the descent of the font. + :return: The height of the font in points. + """ + + # Get font's information and compute the height: + ascent, descent = pdfmetrics.getAscentDescent(font, size) + if not count_ascent: ascent = 0.0 + if not count_descent: descent = 0.0 + height = ascent - descent + return height + + @staticmethod + def font_is_registered(font): + + """ + Checks if a particular font is available for use. + :param font: [str] The name of the font to check. + :return: True if registered, else False + """ + + return font in pdfmetrics.getRegisteredFontNames() + + @staticmethod + def get_first_registered_font(): + + """ + Returns the name of the first registered font. Useful for cases like those when your font fails to load and you + need a default to fall back on. + :return: The name of the first font that has been registered. + """ + + return pdfmetrics.getRegisteredFontNames()[0] + + @staticmethod + def list_registered_fonts(): + + """ + Returns the list of names of all the registered fonts. + :return: The list of names of the fonts that have been registered. + """ + + return pdfmetrics.getRegisteredFontNames() + + @staticmethod + def is_valid_ttf(file): + + """ + Checks if a font file is valid and usable or not. + :param file: [str|io.BytesIO] The file to check. + :return: True if valid, else False. + """ + + if isinstance(file, io.BytesIO): file.seek(0) + try: font = ttLib.TTFont(file)["head"].fontRevision + except Exception as exception: return False + return True + + # ┳ + # ┃┏┳┓┏┓┏┓┏┓┏ + # ┻┛┗┗┗┻┗┫┗ ┛ + # ┛ + + @staticmethod + def make_barcode_image( + data, + bar_width = 2, + bar_height = 125, + border = 10, + bar_color = "#000000", + background_color = "#FFFFFF", + dpi = 300, + format = "png", + as_pil = True + ): + + """ + Makes a barcode in Code128 format. + :param data: [str] The data to encode in the barcode. + :param bar_width: [int] The base bar-module width in pixels. + :param bar_height: [int] The bar-module height in pixels. + :param border: [int] The size of the border around all sides of the barcode in pixels. + :param bar_color: [str] The colour of the bars as a hex string. + :param background_color: [str] The colour of the background as a hex string. + :param dpi: [int] The pixel density to note in the image. Leave it to default for most use cases. + :param format: [str] The file format to save the image in. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :return: Either a PIL object or a PNG file in RAM. + """ + + # Create the barcode in RAM: + barcode_in_ram = io.BytesIO() + barcode_options = { + "module_width": (bar_width / dpi) * 25.4, + "module_height": (bar_height / dpi) * 25.4, + "quiet_zone": 0, + "foreground": bar_color, + "background": background_color, + "format": format, + "dpi": dpi + } + Code128(str(data), writer = ImageWriter()).write(barcode_in_ram, options = barcode_options) + barcode_in_ram.seek(0) + + # Remove the rasterized text at the bottom, and create a border of the desired size: + barcode_image = Image.open(barcode_in_ram, formats = [format]) + width, height = barcode_image.size + barcode_image = barcode_image.crop(( + 0, + int(dpi * 0.0393701), + width, + int(dpi * 0.0393701) + bar_height + )) + width, height = barcode_image.size + background_image = Image.new( + mode = "RGB", + size = (width + border + border, height + border + border), + color = background_color + ) + background_image.paste(barcode_image, (border, border)) + barcode_image = background_image + + # If the user has asked for a PIL object: + if as_pil: return barcode_image + + # Else, ave the PNG in ram: + barcode_in_ram = io.BytesIO() + barcode_image.save(barcode_in_ram, format = "png") + barcode_in_ram.seek(0) + + # Done: + return barcode_in_ram + + @staticmethod + def make_qr_image( + data, + scale = 5, + border = 2, + foreground_color = "#000000", + background_color = "#FFFFFF", + border_color = "#FFFFFF", + background_art = None, + format = "png", + as_pil = True + ): + + """ + Create a QR code. + :param data: [str] The data that you want to encode in the QR code. + :param scale: [int] The size (in pixels) for each tiny block in the QR code. + :param border: [int] The size (in multiples of scale) of the quiet zone. + :param foreground_color: [str] The color of the foreground, a.k.a. the dark color. + :param background_color: [str] The color of the background, a.k.a. the light color. + :param border_color: [str] The color of the quiet zone. + :param background_art: [io.BytesIO|PIL.Image] Any image that you would like to feed into the background instead + of a plain QR code. This can be an animated GIF image as well. + :param format: [str] The type of output file that you want. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :return: Either a PIL object or a PNG file in RAM. + """ + + qr_code = segno.make_qr(str(data)) + params = { + "scale": scale, + "border": border, + "dark": foreground_color, + "light": background_color, + "quiet_zone": border_color, + "kind": format + } + + qr_image = io.BytesIO() + + if background_art is not None: + if isinstance(background_art, Image.Image): + buffer = io.BytesIO() + background_art.save(buffer, format = format) + buffer.seek(0) + background_art = buffer + params["background"] = background_art + params["target"] = qr_image + qr_code.to_artistic(**params) + + else: + params["out"] = qr_image + qr_code.save(**params) + + qr_image.seek(0) + if as_pil: qr_image = Image.open(qr_image, formats = [format]) + return qr_image + + async def image_from_url(self, url, follow_redirects = False, as_pil = True, format = "png"): + + """ + Asynchronously downloads an image and returns it as a PIL object. + Use this instead of just passing the URL to 'draw_image' for better efficiency. + :param url: [str] The URL to download the image from. + :param follow_redirects: [bool] Whether, or not, to follow redirect URLs when downloading the file. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :param format: [str] The type of output file that you want. Not applicable for PIL objects. + :return: The image as a PIL object or as a file in RAM. + """ + + try: + + pil_image = Image.open(await PDFMaker.download_from_url(url, follow_redirects = follow_redirects)) + if as_pil: return pil_image + image_in_memory = io.BytesIO() + pil_image.save(image_in_memory, format = format) + image_in_memory.seek(0) + return image_in_memory + + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + + def image_from_base64(self, data, as_pil = True): + + """ + Converts a base64 string to an image. + :param data: [str] The base-64 representation of the image data. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :return: The image as a PIL object or as a file in RAM. + """ + + try: + + image = io.BytesIO(base64.b64decode(data)) + image.seek(0) + if not as_pil: return image + image = Image.open(image) + return image + + except Exception as exception: + if self.__raise_exception: raise + + @staticmethod + def crop_image_to_aspect_ratio(image, target_aspect_ratio, anchor = "c"): + + """ + Shaves off pixels from an image to achieve the target aspect ratio. + :param image: [PIL.Image] The image to work on, held in a PIL object. + :param target_aspect_ratio: [int|float] The aspect ratio you want to achieve. + :param anchor: [str] Which part of the image you want to retain. Imagine it like a compass - "nw", "n", "ne", + "e", "se", "s", "sw", "w", and "c". + :return: The adjusted image. + """ + + # Get the existing figures: + width, height = image.size + original_aspect_ratio = width / height + + # Compute the new dimensions: + if original_aspect_ratio > target_aspect_ratio: + scaling_factor = target_aspect_ratio / original_aspect_ratio + new_width, new_height = int(width * scaling_factor), height + else: + scaling_factor = original_aspect_ratio / target_aspect_ratio + new_width, new_height = width, int(height * scaling_factor) + + # Now we figure out the start and end points of cropping: + match anchor.strip().lower(): + case "nw": + start_x = 0 + start_y = 0 + case "n": + start_x = int(abs(new_width - width) / 2.0) + start_y = 0 + case "ne": + start_x = int(abs(new_width - width)) + start_y = 0 + case "e": + start_x = int(abs(new_width - width)) + start_y = int(abs(new_height - height) / 2.0) + case "se": + start_x = int(abs(new_width - width)) + start_y = int(abs(new_height - height)) + case "s": + start_x = int(abs(new_width - width) / 2.0) + start_y = int(abs(new_height - height)) + case "sw": + start_x = 0 + start_y = int(abs(new_height - height)) + case "w": + start_x = 0 + start_y = int(abs(new_height - height) / 2.0) + case _: + start_x = int(abs(new_width - width) / 2.0) + start_y = int(abs(new_height - height) / 2.0) + + end_x = start_x + new_width + end_y = start_y + new_height + + # And we finally crop and return the image: + return image.crop((start_x, start_y, end_x, end_y)) + + @staticmethod + def crop_image(image, start_x, start_y, end_x, end_y): + + """ + Crops an image. + :param image: [PIL.Image] The image to work on, held in a PIL object. + :param start_x: [int|float] The left coordinate (in the same unit as the dimension of the canvas). + :param start_y: [int|float] The top coordinate (in the same unit as the dimension of the canvas). + :param end_x: [int|float] The right coordinate (in the same unit as the dimension of the canvas). + :param end_y: [int|float] The bottom coordinate (in the same unit as the dimension of the canvas). + :return: The cropped image. + """ + + return image.crop((start_x, start_y, end_x, end_y)) + + @staticmethod + def resize_image_to_aspect_ratio(image, target_aspect_ratio): + + """ + Resizes the image (by stretching and squishing) till it fits the target aspect ratio. + :param image: [PIL.Image] The image to adjust. + :param target_aspect_ratio: [int|float] The aspect ratio that you want to achieve. + :return: The adjusted image. + """ + + # Get the existing figures: + width, height = image.size + original_aspect_ratio = width / height + + # Compute the new dimensions: + if original_aspect_ratio > target_aspect_ratio: + new_width, new_height = width, int(width / target_aspect_ratio) + else: + new_width, new_height = int(height * target_aspect_ratio), height + + # Resize and return the image: + return image.resize((new_width, new_height)) + + @staticmethod + def resize_image(image, width, height): + + """ + Resizes an image to the specified width and height. + :param image: [PIL.Image] The image as a PIL object. + :param width: [int|float] The new width. + :param height: [int|float] The new height. + :return: The resized image. + """ + + return image.resize((width, height)) + + @staticmethod + def fit_image_to_aspect_ratio( + image: Image, + target_aspect_ratio: float, + blur_strength = 0.01, + alpha: float = 0.8 + ): + + """ + Fits an image to any aspect ratio by adding a blurred background to it. + :param image: [PIL.Image] The image to adjust. + :param target_aspect_ratio: [int|float] The aspect ratio that you want to achieve. + :param blur_strength: How intense the blur must be. + :param alpha: The opacity of the blurred background. + :return: The adjusted image. + """ + + # Get the existing figures: + width, height = image.size + original_aspect_ratio = width / height + + # Compute the new dimensions: + if original_aspect_ratio > target_aspect_ratio: + new_width, new_height = width, int(width / target_aspect_ratio) + scaling_factor = new_height / height + paste_x = 0 + paste_y = int((new_height - height) / 2) + else: + new_width, new_height = int(height * target_aspect_ratio), height + scaling_factor = new_width / width + paste_x = int((new_width - width) / 2) + paste_y = 0 + + # Prepare the background image: + # bg_image = image.copy().resize((new_width, new_height)) + bg_image = image.copy().resize((int(width * scaling_factor), int(height * scaling_factor))) + bg_image = bg_image.filter(ImageFilter.GaussianBlur(radius = int((new_width + new_height) / 2) * blur_strength)) + bg_image = PDFMaker.crop_image_to_aspect_ratio(bg_image, target_aspect_ratio) + + # darken the background a little: + bg_image = Image.blend( + bg_image, + Image.new("RGB", bg_image.size, (0, 0, 0)), + alpha = 1.0 - alpha + ) + + # Paste the original image on top of the background: + bg_image.paste(image, (paste_x, paste_y)) + + # Resize and return the image: + return bg_image + + @staticmethod + def adjust_image_orientation_from_exif(image): + + """ + Rotates the image as per the instructions in the EXIF tags. + :param image: The PIL object that holds the image data. + :return: The adjusted image. + """ + + # Guard clause to return if the attribute is not even found: + if not hasattr(image, "getexif"): return image + + # Else we start extracting EXIF data: + exif = image.getexif() + if exif is None: return image + + # We look for the tag that indicates orientation: + # ORIENTATION CODES: + # 1: Normal (Landscape) + # 2: Flipped horizontally + # 3: Upside down (Reverse Landscape) + # 4: Flipped vertically + # 5: Rotated 90° clockwise and flipped horizontally + # 6: Rotated 90° clockwise (Portrait) + # 7: Rotated 90° counter-clockwise and flipped horizontally + # 8: Rotated 90° counter-clockwise (Reverse Portrait) + orientation = exif.get(274, 0) + if orientation == 1: pass + elif orientation == 2: image = image.transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 3: image = image.rotate(180, expand = True) + elif orientation == 4: image = image.rotate(180, expand = True).transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 5: image = image.rotate(270, expand = True).transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 6: image = image.rotate(270, expand = True) + elif orientation == 7: image = image.rotate(90, expand = True).transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 8: image = image.rotate(90, expand = True) + + # Done here: + return image + + # ┏┓┳┓┏ + # ┃┃┃┃╋ + # ┣┛┻┛┛ + + def draw_line( + self, + start_x, + start_y, + end_x, + end_y, + thickness = 1.0, + color = (0, 0, 0), + alpha = 1.0 + ): + + """ + Draw a line in the PDF page. + :param start_x: [int|float] The left coordinate. + :param start_y: [int|float] The top coordinate. + :param end_x: [int|float] The right coordinate. + :param end_y: [int|float] The bottom coordinate. + :param thickness: [int|float] The thickness of the line (in the same unit as the canvas's dimensions). + :param color: [str] The RGB color in an array. + :param alpha: [int|float] The opacity of the line. 1 is 100% and 0 is 0% + :return: True or False based on the success of the operation. + """ + + # Handle color: + color = self.__parse_color(color) + + # Adjust the dimensions: + thickness *= self.__unit_factor + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = True) + + try: + + self.__canvas.setLineWidth(thickness) + self.__canvas.setStrokeColorRGB(color[0], color[1], color[2], alpha) + self.__canvas.line( + start_x, + start_y, + end_x, + end_y + ) + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def draw_circle( + self, + x, + y, + radius, + fill_color = (255, 255, 255), + fill_alpha = 0.0, + stroke_color = (0, 0, 0), + stroke_alpha = 1.0, + thickness = 1.0 + ): + + """ + Draw a circle in the PDF page. + :param x: [int|float] The horizontal coordinate of the center of the circle. + :param y: [int|float] The vertical coordinate of the center of the circle. + :param radius: [int|float] The radius of the circle. + :param fill_color: [str] The color-array (0-255) or hex string to define the inner fill color of the rectangle. + :param fill_alpha: [int|float] The opacity of the inner fill color. + :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. + :param stroke_alpha: [int|float] The opacity of the outline color. + :param thickness: [int|float] The thickness of the line (in the same unit as the canvas's dimensions). + :return: True or False based on the success of the operation. + """ + + try: + + # Handle color: + fill_color = self.__parse_color(fill_color) + stroke_color = self.__parse_color(stroke_color) + + # Adjust the dimensions: + x = self.__parse_x(x) + y = self.__parse_y(y, flip = True) + thickness *= self.__unit_factor + radius *= self.__unit_factor + + # Draw the circle here: + self.__canvas.setLineWidth(thickness) + self.__canvas.setFillColorRGB(fill_color[0], fill_color[1], fill_color[2]) + self.__canvas.setFillAlpha(fill_alpha) + self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) + self.__canvas.setStrokeAlpha(stroke_alpha) + self.__canvas.circle(x, y, radius, stroke = 1, fill = 1) + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def draw_rect( + self, + start_x, + start_y, + end_x, + end_y, + radius = 0.0, + fill_color = (255, 255, 255), + fill_alpha = 0.0, + stroke_color = (0, 0, 0), + stroke_alpha = 1.0, + thickness = 1.0, + ): + + """ + Draws a rectangle on the page. + :param start_x: [int|float] The left coordinate. + :param start_y: [int|float] The top coordinate. + :param end_x: [int|float] The right coordinate. + :param end_y: [int|float] The bottom coordinate. + :param radius: [int|float] The corner radius. + :param fill_color: [str] The color-array (0-255) or hex string to define the inner fill color of the rectangle. + :param fill_alpha: [int|float] The opacity of the inner fill color. + :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. + :param stroke_alpha: [int|float] The opacity of the outline color. + :param thickness: [int|float] The line thickness of the outline (in the same unit as the canvas's dimensions). + :return: True or False based on the success of the operation. + """ + + try: + + # Handle color: + fill_color = self.__parse_color(fill_color) + stroke_color = self.__parse_color(stroke_color) + + # Adjust the dimensions: + thickness *= self.__unit_factor + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = False) + + # Compute the width and height desired by the user: + width = end_x - start_x + height = end_y - start_y + + # Draw the rectangle here: + self.__canvas.setLineWidth(thickness) + self.__canvas.setFillColorRGB(fill_color[0], fill_color[1], fill_color[2]) + self.__canvas.setFillAlpha(fill_alpha) + self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) + self.__canvas.setStrokeAlpha(stroke_alpha) + self.__canvas.roundRect( + start_x, + self.__canvas_height - start_y - height, + width, + height, + radius * self.__unit_factor, + stroke = 1, + fill = 1 + ) + + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def make_paragraph_style( + self, + font_name = "Helvetica", + font_size = 12, + font_color = (0, 0, 0), + line_spacing = 12, + border_size = 0.0, + border_color = (255, 255, 255), + align = "left", + style_name = "CustomStyle" + ): + + """ + Creates a text style for using in a paragraph. + :param font_name: [str] The name of the font. Has to be registered. + :param font_size: [int|float] The size that you want the font to be printed in. + :param font_color: [str] The color-array (0-255) or hex string that you want the font to be printed in. + :param line_spacing: [int|float] The line spacing to apply. + :param border_size: [int|float] The thickness of the border. + :param border_color: [str] The color-array (0-255) or hex string of the color you wan the border to be in. + :param align: [str] Text alignment ("left", "center", or "right"). + :param style_name: [str] A name that you would like to give this style. + :return: The paragraph style. + """ + + # Make adjustments: + font_color = self.__parse_color(font_color) + font_size = self.__parse_font_size(font_size) + border_color = self.__parse_color(border_color) + border_size *= self.__unit_factor + + return ParagraphStyle( + name = style_name, + fontName = font_name, + fontSize = font_size, + textColor = (font_color[0], font_color[1], font_color[2]), + alignment = { + "left": 0, + "center": 1, + "right": 2 + }[align], + leading = line_spacing * font_size, + spaceBefore = 0, + spaceAfter = 0, + borderWidth = border_size, + borderColor = border_color + ) + + def write_paragraph( + self, + text, + start_x, + start_y, + end_x, + end_y, + alpha = 1.0, + style = None + ): + + """ + To write text into a bounding box in the PDF. + This can be either plaintext or an HTML string. + :param text: [str] The text to type in the bounding box. + :param start_x: [int|float] The left coordinate (in the same unit as used while setting up the canvas). + :param start_y: [int|float] The top coordinate (in the same unit as used while setting up the canvas). + :param end_x: [int|float] The right coordinate (in the same unit as used while setting up the canvas). + :param end_y: [int|float] The bottom coordinate (in the same unit as used while setting up the canvas). + :param alpha: [int|float] The opacity of the text. 1 is 100% and 0 is 0% + :param style: The paragraph-style generated by using "make_paragraph_style" method. Avoid using this if you are + passing in an HTML string. + :return: True or False based on the success of the operation. You will get a False even if the text flows out of + the bounding box (which will cause it to not render). + """ + + try: + + # Adjust the dimensions: + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = False) + + # Compute the width and height desired by the user: + width = end_x - start_x + height = end_y - start_y + + # Make the paragraph: + paragraph = Paragraph(text, style = style) + + # Wrap the paragraph to compute the needed dimensions: + needed_width, needed_height = paragraph.wrap(width, height) + + # Adjust the y coordinates (for top-down system): + start_y = self.__canvas_height - start_y - needed_height + + # Write the text to the canvas if the text fits. + # Return true in case of success, else false: + if width >= needed_width and height >= needed_height: + self.__canvas.setFillAlpha(alpha) + paragraph.drawOn(self.__canvas, start_x, start_y, _sW = 0) + return True + else: return False + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def write_string( + self, + text, + x, + y, + font = "Helvetica", + size = 12, + color = (0, 0, 0), + alpha = 1.0, + align = "left", + angle = 0 + ): + + """ + Writes a string onto the canvas. + :param text: [str] The text to be written. + :param x: [int|float] The horizontal coordinate (of the baseline). + :param y: [int|float] The vertical coordinate (of the baseline). + :param font: [str] The name of the font to be used. + :param size: [int|float] The size of the font to be used (in points). + :param color: [str] The color-array (0-255) or hex string that you want the font to be printed in. + :param alpha: [int|float] The opacity of the text. 1 is 100% and 0 is 0% + :param align: [str] The alignment to use ("left", "center", "right"). + :param angle: The angle at which you want the text to be printed in. + :return: True or False based on the success of the operation. + """ + + try: + + # Handle the color + color = self.__parse_color(color) + + # Adjust the dimensions: + x = self.__parse_x(x) + y = self.__parse_y(y, flip = True) + size = self.__parse_font_size(size) + + # Adjust the canvas's values: + self.__canvas.setFont(font, size) + self.__canvas.setFillColorRGB(color[0], color[1], color[2]) + self.__canvas.setFillAlpha(alpha) + + # Handle angular adjustments: + self.__canvas.rotate(angle) + x, y = self._parse_coordinates_for_rotation( + original_x = x, + original_y = y, + angle = angle + ) + + # Adjust the font coordinates as per the angle: + text_width = self.__canvas.stringWidth(text, fontName = font, fontSize = size) + if align == "center": x -= text_width / 2 + elif align == "right": x -= text_width + + # Draw the string: + self.__canvas.drawString(x, y, text) + + # Reset angular adjustments: + self.__canvas.rotate(-angle) + + # Return with success if nothing broke till here: + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def draw_image( + self, + image, + start_x, + start_y, + end_x, + end_y, + crop = False, + crop_anchor = "c", + alpha = 1.0, + stroke_color = (0, 0, 0), + stroke_alpha = 0.0, + thickness = 1.0, + identifier: str = None, + format = "jpeg" + ): + + """ + Draw an image onto the PDF page in the specified bounding box. The image can be drawn in either "fit" mode where + the image is scaled down to fit in the bounding box, or in "fill" mode where the image is cropped to fill up the + space in the bounding box by matching their aspec ratios. + :param image: The image data as a PIL object or a valid URL or a file path. NOTE: If given as a URL, the image + will be downloaded synchronously. + :param start_x: [int|float] The left coordinate. + :param start_y: [int|float] The top coordinate. + :param end_x: [int|float] The right coordinate. + :param end_y: [int|float] The bottom coordinate. + :param crop: [bool] Set to True for "fill" mode, False for "fit" mode. + :param crop_anchor: [str] If cropping is to be performed, what should the anchor be for cropping. Refer to the + description in 'crop_image_to_aspect_ratio'. + :param alpha: [int|float] The opacity of the image. 1 is 100% and 0 is 0% + :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. + :param stroke_alpha: [int|float] The opacity of the outline color. + :param thickness: [int|float] The line thickness of the outline (in the same unit as the canvas's dimensions). + :param identifier: [str] A unique name for the image. + :param format: The file format of the image (when it will be embedded in the PDF). + :return: True or False based on the success of the operation. + """ + + try: + + # Cleaning: + format = format.strip().lower().split(".")[-1] + + # Handle color: + stroke_color = self.__parse_color(stroke_color) + + # The identifier can allow us to use the same image multiple times in a PDF file. + # If the identifier is not provided, we give the file a random identifier. + if identifier is None: identifier = "".join([random.choice(string.ascii_lowercase) for _ in range(8)]) + + # Adjust the dimensions: + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = True) + + # Compute the width and height desired by the user: + width = end_x - start_x + height = end_y - start_y + + # In case the image is passed as a URL, we try to download it and open it as a PIL object: + if isinstance(image, str): + if image.startswith("https://") or image.startswith("http://"): + image = Image.open(io.BytesIO(requests.get(image).content)) + else: image = Image.open(image) + + # The image can have orientation data in its EXIF tags. + # If so, we adjust for it so that reportlab can work with it properly: + image = self.adjust_image_orientation_from_exif(image) + + # If the image needs to be cropped, we compute the aspect ratio and get the image adjusted: + if crop: image = self.crop_image_to_aspect_ratio( + image, + abs(end_x - start_x) / abs(end_y - start_y), + anchor = crop_anchor + ) + + # Else we add a blurred background to the image: + else: image = self.fit_image_to_aspect_ratio( + image, + abs(end_x - start_x) / abs(end_y - start_y), + blur_strength = 0.1 + ) + + # We set the image opacity and outline settings here: + self.__canvas.setFillAlpha(alpha) + self.__canvas.setLineWidth(thickness * self.__unit_factor) + self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) + self.__canvas.setStrokeAlpha(stroke_alpha) + + # Draw the image onto the canvas: + if ( + image.mode == "P" or + format in ["jpeg", "jpg"] + ): + image = image.convert("RGB") + image_buffer = io.BytesIO() + image.save(image_buffer, format = format) + image_buffer.seek(0) + self.__canvas.drawImage( + ImageReader(image_buffer, ident = identifier), + start_x, + start_y, + width = width, + height = height, + preserveAspectRatio = True, + mask = "auto", + anchor = "c", + showBoundary = True + ) + + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + # def draw_svg( + # self, + # svg, + # start_x, + # start_y, + # end_x, + # end_y + # ): + # + # renderSVG.draw(svg, sta) + + # ┏┓┳┓┏ ┏┳┓ ┓ + # ┃┃┃┃╋ ┃ ┏┓┏┓┃┏ + # ┣┛┻┛┛ ┻ ┗┛┗┛┗┛ + + @staticmethod + def make_thumbnails( + pdf_file, + pages = None, + scale = 1.0, + as_pil = True, + format = "png" + ): + + """ + Makes thumbnails of pages of the PDF and returns them + :param pdf_file: The PDF file whose pages must be converted to thumbnails. + :param pages: The list of page nos. to make thumbnails of. If not specified, all pages will be converted. + :param scale: The scaling multiplier, applied to both axes, to change the output size. + :param as_pil: If True, the output will be an array of PIL objects, else the output will be an array of + file-like BytesIO objects. + :param format: The file format of the output image. + :return: The array of thumbnails, either as PIL objects, or as BytesIO objects. + """ + + # Open the PDF, and ensure that we have the pages numbers: + if isinstance(pdf_file, io.BytesIO): + pdf_file.seek(0) + pdf_document = fitz.open(stream = pdf_file) + pdf_file.seek(0) + else: pdf_document = fitz.open(pdf_file) + if pages is None: pages = list(range(len(pdf_document))) + + # Iterate over the pages and make the thumbnails: + thumbnails = [] + for page_no in pages: + img = pdf_document.load_page(page_no).get_pixmap(matrix = fitz.Matrix(scale, scale)) + img = io.BytesIO(img.tobytes(format)) + if as_pil: img = Image.open(img, formats = [format]) + thumbnails.append(img) + + # Done here: + return thumbnails + + @staticmethod + def join_from_ram(pdf_files, raise_exception = True): + + """ + Joins a set of PDF files held in RAM (as BytesIO objects). + :param pdf_files: The list of PDF files (as BytesIO objects). + :param raise_exception: Set to True for an all-or-nothing process. + :return: The joined PDF file (as a BytesIo object). + """ + + try: + + # Create a file in RAM that will hold the merged PDF: + joined_pdf_in_ram = io.BytesIO() + + # Merge the files into one: + merger = PdfMerger() + for pdf_file in pdf_files: merger.append(pdf_file) + merger.write(joined_pdf_in_ram) + merger.close() + + # Done here: + joined_pdf_in_ram.seek(0) + return joined_pdf_in_ram + + # In case something goes wrong: + except Exception as exception: + if raise_exception: raise + return None + + @staticmethod + def grid_from_ram( + pdf_files, + input_width, + input_height, + output_width, + output_height, + cutting_width, + cutting_height, + unit = "mm", + dpi = 72 + ): + + """ + Lays out a set of input PDFs into a larger pdf in a grid. + NOTE: The 'unit' and 'dpi' choice will be used for both (input and output). + WARNING: I STRONGLY RECOMMEND NOT USING DPI AS A SETTING. THE PDF FORMAT ITSELF DOESN'T HAVE ANY PROVISION FOR + PIXEL DENSITY. THIS SETTING HAS BEEN MADE AVAILABLE FOR PLACEBO EFFECT WHEN CLIENTS ASK FOR VERY HIGH RESOLUTION + FILES WITHOUT WANTING TO UNDERSTAND ANY TECHNICALITY. USE WITH CAUTION. + :param pdf_files: The list of PDF files (as BytesIO objects) to lay onto the larger canvas. + :param input_width: The width of the input PDFs. + :param input_height: The height of the input PDFs + :param output_width: The width of the large PDF on which the smaller PDFs will be laid out. + :param output_height: The height of the large PDF on which the smaller PDFs will be laid out. + :param cutting_width: The horizontal buffer to leave between the grid. + :param cutting_height: The vertical buffer to leave between the grid. + :param unit: The unit to use (common to the input and output PDFs). + :param dpi: The pixel density to consider (common to the input and output PDFs). + :return: The PDF grid. + """ + + # Create the base PDF: + base_canvas = PDFMaker( + width = output_width, + height = output_height, + unit = unit, + dpi = dpi + ) + base_canvas.draw_line(0, 1, 1, 0, alpha = 0.0) + base_pdf = base_canvas.save() + + # Adjust all units to match the pixel density adjustments: + input_width *= base_canvas.unit_factor + input_height *= base_canvas.unit_factor + output_width *= base_canvas.unit_factor + output_height *= base_canvas.unit_factor + cutting_width *= base_canvas.unit_factor + cutting_height *= base_canvas.unit_factor + + # Calculate how many input pdfs will fit and the page count needed: + per_row = math.floor((output_width - cutting_width) / (input_width + cutting_width)) + per_column = math.floor((output_height - cutting_height) / (input_height + cutting_height)) + per_page = per_row * per_column + output_page_count = math.ceil(len(pdf_files) / per_page) + + # Calculate the starting point on the page from where the pasting will start: + page_start_x = (output_width - (per_row * (input_width + cutting_width)) + cutting_width) / 2.0 + page_start_y = (output_height - (per_column * (input_height + cutting_height)) + cutting_height) / 2.0 + + # We make a writer object to write pages to the output file, + # and we make a list that will hold the individual output pages: + grid_pdf = PdfWriter() + + # We create each needed page: + for page_number in range(output_page_count): + + # Open the base PDF and create a page here: + grid_page = PdfReader(base_pdf).pages[0] + + # Shortlist the input pdfs to be pasted on this page: + starting_offset = page_number * per_page + ending_offset = starting_offset + per_page + ending_offset = min(ending_offset, len(pdf_files)) + pdf_files_for_page = pdf_files[starting_offset:ending_offset] + + # We paste all the input PDFs one-by-one: + for index, input_pdf in enumerate(pdf_files_for_page): + + # Compute the coordinates to paste on: + row_number = math.floor(index / per_row) + column_number = index if index < per_row else index % per_row + paste_x = page_start_x + (column_number * input_width) + (column_number * cutting_width) + paste_y = page_start_y + (row_number * input_height) + (row_number * cutting_height) + + # The default system for the reportlab is bottom-up, + # we convert to top-down for intuitive operations: + paste_y = base_canvas.shape[1] - paste_y - input_height + + # Open th smaller PDF and paste it on those coordinates: + input_page = PdfReader(input_pdf).pages[0] + input_page.add_transformation(Transformation().translate( + tx = paste_x, + ty = paste_y + ), expand = True) + grid_page.merge_page(input_page) + + # Add the grid page to the grid PDF: + grid_pdf.add_page(grid_page) + + # Join and return the result: + grid_pdf_in_ram = io.BytesIO() + grid_pdf.write(grid_pdf_in_ram) + return grid_pdf_in_ram + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + async def main(): + + my_pdf = PDFMaker(width = 210, height = 297) + # my_image = await my_pdf.image_from_url(r"https://images.pexels.com/photos/28973930/pexels-photo-28973930/free-photo-of-historic-saigon-central-post-office-architecture.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1") + # my_image = await my_pdf.image_from_url(r"") + # my_image = await my_pdf.image_from_url(r"") + my_image = await my_pdf.image_from_url(r"https://images.pexels.com/photos/18317748/pexels-photo-18317748/free-photo-of-peoples-committee-of-ho-chi-minh.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1") + # my_image.show() + blurred_image = PDFMaker.fit_image_to_aspect_ratio(my_image, 1.49) + blurred_image.show() + + asyncio.run(main()) diff --git a/queue/__init__.py b/queue/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/queue/async_kafka.py b/queue/async_kafka.py new file mode 100644 index 0000000..d6c0a82 --- /dev/null +++ b/queue/async_kafka.py @@ -0,0 +1,554 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 13th Jul, 2024 + + OBJECTIVE: + + To be able to work with keys + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To use Kafka: +from aiokafka import AIOKafkaProducer +from aiokafka import AIOKafkaConsumer + +# For working with JSON strings: +from utils_v2.string import json +from utils_v2.serialization.json_serializer import JSONSerializer + +# For debugging: +from icecream import IceCreamDebugger + +# For SSL security: +import ssl + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_ssl_context( + ca_file, + cert_file, + key_file +): + + """ + Generate the SSL context to use with the Kafka instances. + :param ca_file: The Certificate Authority file as a path to a local file. + :param cert_file: The Certificate file as a path to a local file. + :param key_file: The Key file as a path to a local file. + :return: The SSL context instance as a path to a local file. + """ + + ssl_context = ssl.create_default_context() + ssl_context.load_verify_locations(ca_file) + ssl_context.load_cert_chain(certfile = cert_file, keyfile = key_file) + return ssl_context + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class ProducerKafka: + + def __init__( + self, + topic, + serializer = None, + debug = True, + debug_prefix = "Kafka (P) | ", + **kwargs + ): + + """ + Create a Kafka Producer. + :param topic: The topic to produce on. + :param serializer: The serializer to use. + :param debug: Whether, or not, you want to print the debug strings. + :param debug_prefix: The prefix to use while debugging. + :param kwargs: Any configuration parameters for the Kafka instances. + """ + + # Initialize the debugger: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # initialize the Kafka producer: + self.__topic = topic + self.__kwargs = kwargs + self.__producer = None + self.__connected = False + self.__serializer = serializer or JSONSerializer() + + # For establishing connection: + self.__exclusive_semaphore = asyncio.Semaphore(1) + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + async def connect(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + async with self.__exclusive_semaphore: + if not self.__connected: + try: + self.__producer = AIOKafkaProducer(**self.__kwargs) + await self.__producer.start() + self.__connected = True + except Exception as exception: self.__printer(exception) + return self.__connected + + async def ensure_connection(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + if not self.__connected: await self.connect() + return self.__connected + + async def close(self): + + """ + Terminates the connection. + :return: None. + """ + + if self.__connected: + try: + await self.__producer.stop() + self.__printer("Producer closed!") + self.__connected = False + except Exception as exception: self.__printer(exception) + + async def produce(self, value, key = None, topic = None, encoding = "utf-8"): + + """ + Sends one message to the Kafka server on the topic that has been set for this instance. + :param value: The message to send. + :param key: The key to use when you want the messages to follow an order. + :param topic: A custom topic for this message, else the topic defined during the creation of this instance will + be used by default. + :param encoding: The encoding format. + :return: True or False based on the success of the operation. + """ + + # Ensure connectivity to the server. + # If not connected, return with failure immediately. + if not await self.ensure_connection(): return False + + try: + + # Send the message: + await self.__producer.send_and_wait( + topic = topic or self.__topic, + value = self.__serializer.serialize(data = value, encoding = encoding), + key = key + ) + + # Return with success if no exception occurred: + return True + + # Return with failure if something went wrong: + except Exception as exception: + self.__printer(exception, self.__topic, type(value), value) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +class ConsumerKafka: + + def __init__( + self, + topic, + serializer = None, + debug = True, + debug_prefix = "Kafka (C) | ", + **kwargs + ): + + """ + Create a Kafka Consumer. + :param topic: The topic to consumer on. + :param serializer: The serializer to use. + :param debug: Whether, or not, you want to print the debug strings. + :param debug_prefix: The prefix to use while debugging. + :param kwargs: Any configuration parameters for the Kafka instances. + """ + + # Initialize the debugger: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # initialize the Kafka producer: + self.__topic = topic + self.__kwargs = kwargs + self.__consumer = None + self.__connected = False + self.__serializer = serializer or JSONSerializer() + + # For establishing connection: + self.__exclusive_semaphore = asyncio.Semaphore(1) + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + async def connect(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + async with self.__exclusive_semaphore: + if not self.__connected: + try: + self.__consumer = AIOKafkaConsumer(self.__topic, **self.__kwargs) + await self.__consumer.start() + self.__connected = True + except Exception as exception: self.__printer(exception) + return self.__connected + + async def ensure_connection(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + if not self.__connected: await self.connect() + return self.__connected + + async def close(self): + + """ + Terminates the connection. + :return: None. + """ + + if self.__connected: + try: + await self.__consumer.stop() + self.__printer("Consumer closed!") + self.__connected = False + except Exception as exception: self.__printer(exception) + + async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"): + + """ + Get messages from the Kafka server. + :param count: The number of messages to get from the Kafka server. + :param timeout: The time in seconds to wait for retrieval. + :param encoding: The encoding to use. + :return: The messages that were received. If no messages are available, an empty list will be returned. + """ + + # Ensure connectivity to the server. + # If not connected, return with failure immediately. + if not await self.ensure_connection(): return [] + + # Make a variable that will hold the final results: + messages = [] + + try: + + # Read some messages: + results = await self.__consumer.getmany( + max_records = max(1, count), + timeout_ms = int(timeout * 1_000) + ) + + # Format the received messages: + if results: + for topic_partition, records in results.items(): + for record in records: + record_dict = record.__dict__ + record_dict["value"] = self.__serializer.deserialize( + data = record_dict["value"], + encoding = encoding + ) + messages.append(record_dict) + + # Debugging print if something went wrong: + except Exception as exception: self.__printer(exception) + + # Done here: + return messages + + +# --------------------------------------------------------------------------------------------------------------------- + + +class BidirectionalKafka: + + # The 'roles' that the instance can take. + # The master talks on the channel (topic) that the slave listens on and vice versa. + # Master-Slave is only for deciding who talks on which channel and who listens on which. + # In a two-party system, one must be the master, the other must be the slave. + # There are no extra privileges that the master enjoys. The naming convention was borrowed from common protocols + # used in electronics (like I2C). + ROLE_MASTER = 1 + ROLE_SLAVE = 0 + + def __init__( + self, + role, + topic, + ack_topic: str = None, + group: str = None, + serializer = None, + debug = True, + debug_prefix = "Kafka (B) | ", + **kwargs + ): + + """ + Creates a walkie-talkie type setup to use Kafka in a bidirectional manner. Fo more information on all the + individual methods, please read through the doc-strings of the component classes 'ProducerKafka', and + 'ConsumerKafka'. + :param role: Select from "ROLE_MASTER" and "ROLE_SLAVE". Between the two parties that are talking, one will be + the master and the other will be the slave. The channel that the master uses to speak will the one the slave + uses to listen, and vice versa. + :param topic: The topic to communicate on. Will be the same between the master and the slave. + :param ack_topic: Explicitly provide this for the second channel, or it will be created from the name of the + topic itself. Will be the same between the master and the slave. + :param group: The group to assign the instance to. + :param debug: Whether, or not, you want to print the debug strings. + :param debug_prefix: The prefix to use while debugging. + :param kwargs: Any configuration parameters for the Kafka instances. + """ + + # Not down the basic variables: + self.__role = role + self.__topic = topic + self.__ack_topic = ack_topic or topic + "Ack" + self.__group = group + + # In case the current instance is the master, + # it will talk on "topic", and listen on "ack_topic": + if self.__role == self.ROLE_MASTER: + self.__producer_kwargs = kwargs.copy() + self.__producer = ProducerKafka( + topic = self.__topic, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (P) | ", + **self.__producer_kwargs + ) + self.__consumer_kwargs = kwargs.copy() + self.__consumer_kwargs["group_id"] = self.__group + self.__consumer = ConsumerKafka( + topic = self.__ack_topic, + group = group, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (C) | ", + **self.__consumer_kwargs + ) + + # On the other hand, if the current instance is a slave, + # It will listen on "topic", and talk on "ack_topic": + else: + self.__producer_kwargs = kwargs.copy() + self.__producer = ProducerKafka( + topic = self.__ack_topic, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (P) | ", + **self.__producer_kwargs + ) + self.__consumer_kwargs = kwargs.copy() + self.__consumer_kwargs["group_id"] = self.__group + self.__consumer = ConsumerKafka( + topic = self.__topic, + group = group, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (C) | ", + **self.__consumer_kwargs + ) + + def enable_debug(self): + self.__producer.enable_debug() + self.__consumer.enable_debug() + + def disable_debug(self): + self.__producer.disable_debug() + self.__consumer.disable_debug() + + async def ensure_connection(self): + await self.__producer.ensure_connection() + await self.__consumer.ensure_connection() + + async def close(self): + await self.__producer.close() + await self.__consumer.close() + + async def produce(self, message, encoding = "utf-8"): + return await self.__producer.produce(message, encoding = encoding) + + async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"): + return await self.__consumer.consume(count = count, timeout = timeout, encoding = encoding) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + import time + from data_models.kafka_message import KafkaMessage + + ssl_ctx = get_ssl_context( + ca_file = r"/home/developer/PycharmProjects/utils/cred/kafka/cert_authority.pem", + cert_file = r"/home/developer/PycharmProjects/utils/cred/kafka/fullchain.pem", + key_file = r"/home/developer/PycharmProjects/utils/cred/kafka/privkey.pem" + ) + + async def consumer_test(): + + consumer = ConsumerKafka( + topic = "kft_file_upload", + # group_id = "assessImg", + group_id = "updateMedia", + bootstrap_servers = "wtt.ditscentre.in:9092", + security_protocol = "SSL", + ssl_context = ssl_ctx + ) + await consumer.connect() + await asyncio.sleep(1.5) + print("READY!") + + while True: + messages = await consumer.consume(count = 1) + if len(messages) > 0: print("MESSAGE:", json.to_string(messages[0], default=str)) + await asyncio.sleep(1.0) + + async def producer_test(): + + producer = ProducerKafka( + topic = "kft_file_upload", + bootstrap_servers = "del.ditscentre.in:9092", + security_protocol = "SSL", + ssl_context = ssl_ctx + ) + await producer.connect() + print("READY!") + + while True: + my_msg = KafkaMessage( + data = { + "accepted": False, + "reason": "low resolution" + }, + media = { + "name": "pikachu_poster.jpg", + "ext": "jpg", + "url": "https://nexcom.ditscentre.in/utils/files/small/download/66ded1c1c1c05139a618b5ff", + "attr": { + "user": "SarangKabir", + "project": "ACE-PGP", + "id": 173, + "campaignActivityId": "25", + "idCampaign": 49, + "phoneNo": "7977821877" + } + }, + appId = "aceWockhardt", + proc = { + "name": "_assessImg", + "attr": { + "blurThreshold": 0.25, + "clarityThreshold": 0.65, + "nsfwThreshold": 0.25, + "minWidth": 512, + "minHeight": 512 + } + }, + ack = None + ) + success = await producer.produce(my_msg.model_dump()) + print("produced...") + time.sleep(1.0) + break + + await producer.close() + + + asyncio.run(consumer_test()) diff --git a/rate_limit/__init__.py b/rate_limit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rate_limit/async_token_bucket.py b/rate_limit/async_token_bucket.py new file mode 100644 index 0000000..4fb85e7 --- /dev/null +++ b/rate_limit/async_token_bucket.py @@ -0,0 +1,212 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Wednesday, 15th May, 2024 + + OBJECTIVE: + + To provide a rate-limiting mechanism using the simple token bucket algorithm. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For date and time keeping: +import datetime +import time + +# For asynchronous activities: +import asyncio + +# For mathematical operations: +import math + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class TokenBucket: + + """ + MODES: "add" - Add 'rate_limit' no. of tokens to the bucket every interval. + "reset" - Reset the token count to 'rate_limit' every interval + """ + + MODE_RESET = 0 + MODE_ADD = 1 + + ONE_YEAR = 3_15_36_000.0 + ONE_MONTH = 26_78_400.0 + ONE_WEEK = 6_04_800.0 + ONE_DAY = 86_400.0 + ONE_HOUR = 3_600.0 + ONE_MINUTE = 60.0 + ONE_SECOND = 1.0 + + def __init__(self, rate_limit, seconds = 1.0, mode = "reset", sleep = 0.1, in_sequence = False): + + """ + Initialize the rate controller. + :param rate_limit: The no. of operations allowed per unit of time. + :param seconds: The time period in seconds in which the tokens get reset or added. + :param mode: To select what happens when the period is over. 'reset' mode means that the remaining tokens from + the previous period are discarded and the counter is set back to the rate limit, and 'add' mode means that + new tokens are added on top of the exiting ones that were unused in the previous period. + :param sleep: The delay to add before checking back to see if tokens are available. Keep it longer for longer + periods (which is adjusted by the 'seconds' parameter). + :param in_sequence: Whether, or not, you want to maintain the sequence in which the requests were made + (experimental). + """ + + if rate_limit is not None: + rate_limit = int(rate_limit) + self.__semaphore = asyncio.Semaphore(1 if in_sequence else rate_limit) + mode = mode.lower() + if mode not in [self.MODE_RESET, self.MODE_ADD]: mode = self.MODE_RESET + self.__mode = mode + self.__rate_limit = rate_limit + self.__token_count = rate_limit + self.__last_token_generation_time = time.time() + self.__sleep = sleep + self.__seconds = seconds + + async def has_turn(self): + + """ + Just to check if a token is available. + Should be used only if you don't want to wait for turn in case it is unavailable. + :return: True if the token is available, False if not. + """ + + # If the user doesn't want any rate-limits, we return immediately. + # Otherwise, we check if the user has any tokens available: + if self.__rate_limit is None: return True + if self.__token_count > 0: return True + else: return False + + async def get_turn(self, timeout = None): + + """ + To wait till either you get your turn or the wait gets timed-out. + :param timeout: The time (in seconds) to wait to get a turn before exiting with failure. + :return: True if a turn was received, else False if timed-out. + """ + + # If the user doesn't want any rate-limits, we return immediately: + if self.__rate_limit is None: return + + # Start by making variables: + got_turn = False + wait_start = time.time() + + # Otherwise, we wait for the semaphore: + async with self.__semaphore: + + # Wait till you get your turn or the attempt gets timed-out: + while True: + + # If the timeout has been exceeded: + if ( + timeout is not None and + time.time() - wait_start >= timeout + ): break + + # If it is time to generate new tokens: + time_delta = time.time() - self.__last_token_generation_time + if time_delta >= self.__seconds: + if self.__mode == self.MODE_RESET: self.__token_count = self.__rate_limit + else: self.__token_count += int(self.__rate_limit * math.floor(time_delta / self.__seconds)) + self.__last_token_generation_time = time.time() + + # If a token is available (or not): + if self.__token_count > 0: + self.__token_count -= 1 + got_turn = True + break + else: await asyncio.sleep(self.__sleep) + + # Done here: + return got_turn + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import random + + async def single_test(rate_controller, count): + has_turn = await rate_controller.has_turn() + got_turn = await rate_controller.get_turn() + print(f"TURN: {count:.<5} {'Y' if has_turn else '-'} / {'Y' if got_turn else '-'} ({datetime.datetime.now()})") + if got_turn: await asyncio.sleep(random.random()) + + async def multi_test(max_count): + rate_controller = TokenBucket( + rate_limit = 10, + seconds = 1, + in_sequence = False, + sleep = 0.025, + mode = "reset", + timeout = 3 + ) + tasks = [single_test(rate_controller, count + 1) for count in range(max_count)] + await asyncio.gather(*tasks) + + + start_time = time.time() + asyncio.run(multi_test(100)) + print(f"FINISHED IN {time.time() - start_time} SECONDS!") diff --git a/security/__init__.py b/security/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/security/__pycache__/__init__.cpython-310.pyc b/security/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a34e9d2443e31b8dc74f39bf41ca9d834df7710 GIT binary patch literal 171 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vHKO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaMNWVBW oxwI&=q*6aVJ~J<~BtBlRpz;=nO>TZlX-=vg$ev;*Ai=@_0Do{QsQ>@~ literal 0 HcmV?d00001 diff --git a/security/__pycache__/sanitizers.cpython-310.pyc b/security/__pycache__/sanitizers.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62e16b635009285f0881ce0c10f03286afefee88 GIT binary patch literal 1933 zcmZuxO>Y}T7~a`kuQzU*ln@~}P=*WCC|*YqAQ7U7kOt9GIiU_9xelwz&e+~$zdAFU z)|P!ru03$&)E@H#_!0btIdI{`Ux)kUXVTlZYX&^^-|zMR zST>Bm_2Bwr!QdNcVgMaAItE45j8F$LbY#A?sPzgVV;b8!hWScl$CXZ{h|S39IK`f7 zr%D~>_Dou(?wQ@0qjR)&W^`)o7QJN{CUM%slt_Yuh>;{r`nZ=Tfeh0`;9kmcm}I#WI0}y$&IW11aFWM| zjEfb_ViHFBUj|-z;NzX|b{_Avws+dsK(?A&Q}DaLv>xql-G5{9A0XanDddez&#NO< z;jGV|tNG$=Uq}wH0$^+Q=(qxH8=80@I%yoCvGEdV8D5#xqBgC(u-Zo58U0mIysf`Z z;5@@}z;Kxo%M_C+N}u6S>R6B-i-!1+1!=6=>4gzXNX)z>P0$#^(lK zgujcu<^ZT@Qg*_kG-JHEHw*@Z$9p_IVu2J*l7&9$_kEfMdCU^o%q5hBf3nsTEXa8% zhfM)hCBqR^jF%17oF>)RtUiDPgm$Lw){qH5=U@H~3~vanMV<+r!9WXqW=ufri7Ab- zAMj}>z!%zYWcN+=EM70e#I*EJ{B#FqBhK1C8qvuBE;_1G{s55 zvR50+g*T|Z^iUgE6>u+3J){Yy)(dL_bi(J`t8IXClm;Xcn_jv823}L| zgEqt42ZRUC1cB7(6#ZsC1-Xhd;RczT<*b{^^j4L7N$Ts+Y8#p-aD2AOH9URHvN8H_ zmf#B1)yle~Y9Sa}NRk9h**b-?wL+>AN)`+7-4OipE+ng0)IxcY%^)nnYqc~}apoR{ zjukq24PlYwiH4>{a%=Wn&00XvU)bml`UKflu|m1N5BG86`}KKM^>t3)SC$w87I6PS zIV8(iLe=7>MZNMCEH41394LoFd8aXKl)SLxEV=fq?_UE~Xk5taZp10gBetn)s<@AB X!~J+3!OwCn7cJa%m)y@B)Bf*&y(9n{ literal 0 HcmV?d00001 diff --git a/security/hash.py b/security/hash.py new file mode 100644 index 0000000..b3fc582 --- /dev/null +++ b/security/hash.py @@ -0,0 +1,245 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 26th Aug., 2024 + + OBJECTIVE: + + To provide an easy way to hash inputs. + + REFERENCES: + + 1) Book: Full Stack Python Security - Dennis Byrne + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# For hashing: +import hashlib +import hmac +import secrets +from bcrypt import hashpw, gensalt + +# To work with buffers: +import io + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class Hasher: + + def __init__( + self, + algorithm = hashlib.sha256, + key = None + ): + + """ + hashes data or a message or a file. Uses HMAC if 'key' is specified, else performs simple hashing. + :param algorithm: The algorithm to use. SHA256 by default. + :param key: Specify this as either a string or as bytes to use HMAC. Leave as null for simple hashing. + """ + + # Initialize the hasher: + self.__hasher = None, + self.__algorithm = algorithm + self.__hmac_key = key + if self.__hmac_key is not None: + self.__hmac_key = self.__hmac_key.encode("utf-8") if isinstance(self.__hmac_key, str) else self.__hmac_key + self.reset() + + @staticmethod + def generate_key(byte_count = 32, url_safe = False): + + """ + A mechanism to generate key/salt values. + NOTE: For a proper salt for passwords, I recommend using "generate_salt" method. It's far better. + :param byte_count: The number of bytes to have in the key. The hex output (as string) will have 2x the + characters. + :param url_safe: Set to True if you need the generated output to be a part of a URL. + :return: The generate key/salt. + """ + + return secrets.token_urlsafe(byte_count) if url_safe else secrets.token_hex(byte_count) + + @staticmethod + def generate_salt(): + + """ + Generate a salt to use while hashing things like passwords. + :return: The salt as bytes. + """ + + return gensalt() + + @staticmethod + def hash_password(password, salt): + + """ + Hashes a password with the given salt. + :param password: The password to hash, either as a string or as bytes. + :param salt: The salt to hash the password with, either as a string or as bytes. + :return: The hashed string. + """ + + return hashpw( + password = password.encode("utf-8") if isinstance(password, str) else password, + salt = salt.encode("utf-8") if isinstance(salt, str) else salt + ) + + @staticmethod + def compare_hashes(hash_0, hash_1): + + """ + Compares two hashes to see if they match. + Comparison is done in constant time to avoid timing-based side-channel attacks. + :param hash_0: One of the hashes to compare. + :param hash_1: The other hash to compare. + :return: True if they match, else False. + """ + + return hmac.compare_digest(hash_0, hash_1) + + def reset(self): + + """ + Resets the hasher by removing all the data that was fed into it. + :return: None. + """ + + if self.__hmac_key is not None: + self.__hasher = hmac.new( + key = self.__hmac_key, + digestmod = self.__algorithm + ) + else: self.__hasher = self.__algorithm() + + def update(self, data): + + """ + Adds data to the hash to update it. + :param data: The data to be hashed. + :return: None. + """ + + data = data.encode("utf-8") if isinstance(data, str) else data + self.__hasher.update(data) + + def digest(self): + + """ + Returns the hexadecimal representation of the hash as a string. + :return: The hexadecimal representation of the hash as a string + """ + + return self.__hasher.digest() + + def hexdigest(self): + + """ + Returns the hexadecimal representation of the hash as a string. + :return: The hexadecimal representation of the hash as a string + """ + + return self.__hasher.hexdigest() + + def hash_message(self, message, as_hex = True): + + """ + Hashes one message and returns the result, and then resets the instance. + :param message: The data you want to hash. + :param as_hex: Invokes 'hexdigest' if True, else 'digest'. + :return: The hash of the message in either hexadecimal string form or binary form. + """ + + self.update(message) + hash_result = self.hexdigest() if as_hex else self.digest() + self.reset() + return hash_result + + def hash_file(self, file, chunk_size = 4096, as_hex = True): + + """ + Hashes one file and returns the result, and then resets the instance. + :param file: The file you want to hash either as a path or as some buffer (like io.BytesIO). + :param chunk_size: The size of data (in bytes) that you would like to pick at one time. + :param as_hex: Invokes 'hexdigest' if True, else 'digest'. + :return: The hash of the file in either hexadecimal string form or binary form. + """ + + # In case the file was given as a io.BytesIO buffer: + if isinstance(file, io.BytesIO): + file.seek(0) + while True: + chunk = file.read(chunk_size) + if not chunk: break + self.update(chunk) + + # In case the file was given as a path: + else: + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + self.update(chunk) + + # Now we capture the results, reset the instance, and return the result: + hash_result = self.hexdigest() if as_hex else self.digest() + self.reset() + return hash_result + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + print( + Hasher.hash_password( + password = "mic test, mic test, 123", + salt = Hasher.generate_salt() + ) + ) diff --git a/security/otp.py b/security/otp.py new file mode 100644 index 0000000..49edcd2 --- /dev/null +++ b/security/otp.py @@ -0,0 +1,156 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 10th Sept., 2024 + + OBJECTIVE: + + To provide a way to generate and verify OTPs. + + REFERENCES: + + 01. https://pyauth.github.io/pyotp/# + 02. https://en.wikipedia.org/wiki/Google_Authenticator + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with OTPs: +import pyotp +import base64 +import hashlib + +# To work with date and time: +import time +import datetime + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class HashedOTP: + + def __init__( + self, + secret + ): + + """ + Used to generate and verify HMAC-based OTPs. + :param secret: The key to use to generate and verify OTPs. + """ + + # Note down the input variables: + self.__secret = secret + self.__otp_client = pyotp.HOTP(secret) + + @staticmethod + def generate_secret(message = None): + + """ + Generate a secret key to then use to generate and verify the OTPs. + You may override the random generator by giving a "message" of any length. + :param message: A custom value to convert into a key. Avoid using this for better security, but this can be used + to generate keys based on user identifiers. THERE IS NO RANDOMNESS IF YOU USE THIS FEATURE. IT IS FOR + CONVENIENCE ONLY. NOT RECOMMENDED. + :return: The key (as a string) that can be used to generate and verify the OTPs. + """ + + # If the user wants to generate a key from a custom input: + if message: + + # Ensure we have a bytes object: + if not isinstance(message, (str, bytes)): message = str(message) + if isinstance(message, str): message = message.encode("utf-8") + + # Hash the bytes object: + sha256_hash = hashlib.sha256() + sha256_hash.update(message) + hashed_key = sha256_hash.digest() + + # Convert to base-32: + return base64.b32encode(hashed_key).decode("utf-8") + + # If the user wants a totally random key: + else: return pyotp.random_base32() + + def generate_otp(self, count: int): + + """ + Generates the OTP at a particular step. + :param count: The step at which the OTP needs to be generated. + :return: The OTP string (6 digits). + """ + + return str(self.__otp_client.at(count)) + + def verify_otp(self, otp, count: int): + + """ + Verifies the claimed OTP. + :param otp: The OTP as claimed by the end user. + :param count: The step at which the OTP needs to be verified. + :return: True if the OTP is valid, else False. + """ + + return self.__otp_client.verify(otp, counter = count) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/security/sanitizers.py b/security/sanitizers.py new file mode 100644 index 0000000..26dfe5b --- /dev/null +++ b/security/sanitizers.py @@ -0,0 +1,135 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 25th Jul., 2024 + + OBJECTIVE: + + To provide a set of data cleaning functions for inputs like phone numbers, emails, etc. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# My utils: +from utils_v2.string import regex + +# For random strings and tokens: +import string +import random + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def file_name(input_string: str): + + """ + Cleans up the string to allow it to safely become a filename. + :param input_string: The string that you want to make safe for using as a filename. + :return: The string that can safely be used as a filename. + """ + + return regex.replace( + text = input_string.replace("\n", " "), + pattern = r"[^a-zA-Z0-9 \-_\.]", + substitute_text = "" + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def for_mongo(input_document): + + """ + Sanitizes and disarms any JSON-like input that could be used for NoSQL-injection attacks. + :param input_document: The list or dict to be sanitized. + :return: The sanitized list or dict. + """ + + # A special function that disarms any input string by dealing with special characters + # that Mongo may consider to be instructions: + def disarm(input_string): + input_string = regex.replace( + text = input_string, + pattern = r"[^a-zA-Z0-9,_\-\.\\\/:;'\(\) ]", + substitute_text = "" + ) + return input_string + + # Initially we assign the value of the input to the output: + sanitized_document = input_document + + # Handle the case where the input is an array: + if isinstance(input_document, list): + sanitized_document = [for_mongo(document) for document in input_document] + + # Handle the case when the input is a document: + elif isinstance(input_document, dict): + sanitized_document = {} + for k, v in input_document.items(): + sanitized_document[disarm(k)] = v if type(v) not in [list, dict] else for_mongo(v) + + # Done here: + return sanitized_document + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/serialization/__init__.py b/serialization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/serialization/json_serializer.py b/serialization/json_serializer.py new file mode 100644 index 0000000..87595e0 --- /dev/null +++ b/serialization/json_serializer.py @@ -0,0 +1,129 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Sunday 1st Sept. 2024. + + OBJECTIVE: + + To provide a way to convert any input data to serialized bytes, and back. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with files: +from utils_v2.string import json + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class JSONSerializer: + + def __init__(self): + + """ + Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and + python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour. + """ + + pass + + @staticmethod + def serialize(data, encoding = "utf-8"): + + """ + Serializes the data that is given to it. + The input has to be JSON-compatible. + :param data: The data to serialize. + :param encoding: The encoding to use. + :return: The bytes representing the data. + """ + + # If the data is not already a JSON string, parse it. Then return it as bytes: + data = data if isinstance(data, str) else json.to_string(data, no_space = True) + return data.encode(encoding) + + @staticmethod + def deserialize(data, encoding = "utf-8"): + + """ + Deserializes the bytes that are given to it. + The input has to be JSON-compatible. + :param data: The bytes to deserialize. + :param encoding: The encoding to use. + :return: The data from the bytes that described it. + """ + + data = data.decode(encoding) + return json.from_string(data) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/serialization/pickle_serializer.py b/serialization/pickle_serializer.py new file mode 100644 index 0000000..8dd2b65 --- /dev/null +++ b/serialization/pickle_serializer.py @@ -0,0 +1,122 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday 26th Oct. 2024. + + OBJECTIVE: + + To provide a way to convert any input data to serialized bytes, and back. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with pickling: +import pickle + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class PickleSerializer: + + def __init__(self): + + """ + Use this serializer when working with custom python objects. This is meant to be fully flexible, but efficiency + is not guaranteed. + """ + + pass + + @staticmethod + def serialize(data): + + """ + Serializes the data that is given to it. + :param data: The data to serialize. + :return: The bytes representing the data. + """ + + return pickle.dumps(data) + + @staticmethod + def deserialize(data): + + """ + Deserializes the bytes that are given to it. + :param data: The bytes to deserialize. + :return: The data from the bytes that described it. + """ + + return pickle.loads(data) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/serialization/universal_serializer.py b/serialization/universal_serializer.py new file mode 100644 index 0000000..23818c8 --- /dev/null +++ b/serialization/universal_serializer.py @@ -0,0 +1,265 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Sunday 1st Sept. 2024. + + OBJECTIVE: + + To provide a way to convert any input data to serialized bytes, and back. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with files: +from utils_v2.string import json + +# To work with tabulated data: +import pandas as pd + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class UniversalSerializer: + + def __init__(self): + + """ + Use this when you are working with varied datatypes. You can add custom data-converters also using the + 'add_converters' method. Otherwise, most default pythonic datatypes are supported out of the box. Note that this + is NOT recommended because of how large the serialized messages become. Try using 'JSONSerializer' when you know + you will be working specifically with JSOn-compatible inputs. + """ + + # These are the converters to use when serializing data: + self.__forward_converters = { + "set": lambda x: list(x), + "tuple": lambda x: list(x), + "complex": lambda x: {"r": x.real, "i": x.imag}, + "DataFrame": lambda x: x.to_dict() + } + + # These are the converters to use when deserializing data: + self.__reverse_converters = { + "set": lambda x: set(x), + "tuple": lambda x: tuple(x), + "complex": lambda x: complex(x["r"], x["i"]), + "DataFrame": lambda x: pd.DataFrame.from_dict(x) + } + + def add_converters( + self, + type_name, + forward_converter_func, + reverse_converter_func + ): + + """ + Add custom datatype converters. + RULES: + 01. Each of the converter functions must take in exactly on argument and return one output of native python + type. This is very important. + 02. Each forward and reverse converters must give symmetric results. + :param type_name: The name of the datatype. HINT: type(obj).__name__ + :param forward_converter_func: The function to handle conversion to bytes. Use when serializing. + :param reverse_converter_func: The function to handle conversion from bytes. Used when deserializing. + :return: None. + """ + + self.__forward_converters[type_name] = lambda x: forward_converter_func(x) + self.__reverse_converters[type_name] = lambda x: reverse_converter_func(x) + + def __describe(self, data): + + """ + Notes down the input datatypes of everything. + Does everything upto conversion to byes. + :param data: The data to process. + :return: The description of the datatypes and values of what was given. + """ + + # Note down the type of data that was sent as the input: + data_type = type(data).__name__ + + # Handle iterables: + if isinstance(data, list): data = [self.__describe(item) for item in data] + elif isinstance(data, set): data = [self.__describe(item) for item in data] + elif isinstance(data, tuple): data = [self.__describe(item) for item in data] + elif isinstance(data, dict): data = [ + { + "k": self.__describe(k), + "v": self.__describe(v) + } for k, v in data.items() + ] + + # Convert here, and return: + conv = self.__forward_converters.get(data_type) + if conv is not None: data = conv(data) + return {"d": data, "t": data_type} + + def serialize(self, data, encoding = "utf-8"): + + """ + Serializes the data that is given to it. + :param data: The data to serialize. + :param encoding: The encoding to use. + :return: The bytes representing the data. + """ + + data = self.__describe(data) + data = json.to_string(data, no_space = True) + return data.encode(encoding) + + def __interpret(self, data): + + """ + Interprets the types of data that were serialized originally. + :param data: The data in the serialized form. + :return: Data where the appropriate datatypes have been applied. + """ + + # Handle iterables: + if data["t"] == "list": data = [self.__interpret(item) for item in data["d"]] + elif data["t"] == "set": data = set([self.__interpret(item) for item in data["d"]]) + elif data["t"] == "tuple": data = tuple([self.__interpret(item) for item in data["d"]]) + elif data["t"] == "dict": data = { + self.__interpret(item["k"]): self.__interpret(item["v"]) + for item in data["d"] + } + + # Handle custom types: + else: + conv = self.__reverse_converters.get(data["t"]) + data = data["d"] + if conv is not None: data = conv(data) + + # Done here + return data + + def deserialize(self, data, encoding = "utf-8"): + + """ + Deserializes the bytes that are given to it. + :param data: The bytes to deserialize. + :param encoding: The encoding to use. + :return: The data from the bytes that described it. + """ + + data = data.decode(encoding) + data = json.from_string(data) + return self.__interpret(data) + + +# --------------------------------------------------------------------------------------------------------------------- + + +class JSONSerializer: + + def __init__(self): + + """ + Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and + python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour. + """ + + pass + + @staticmethod + def serialize(data, encoding = "utf-8"): + + """ + Serializes the data that is given to it. + The input has to be JSON-compatible. + :param data: The data to serialize. + :param encoding: The encoding to use. + :return: The bytes representing the data. + """ + + # If the data is not already a JSON string, parse it. Then return it as bytes: + data = data if isinstance(data, str) else json.to_string(data, no_space = True) + return data.encode(encoding) + + @staticmethod + def deserialize(data, encoding = "utf-8"): + + """ + Deserializes the bytes that are given to it. + The input has to be JSON-compatible. + :param data: The bytes to deserialize. + :param encoding: The encoding to use. + :return: The data from the bytes that described it. + """ + + data = data.decode(encoding) + return json.from_string(data) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/sms/__init__.py b/sms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sms/async_nimbus.py b/sms/async_nimbus.py new file mode 100644 index 0000000..1d4b926 --- /dev/null +++ b/sms/async_nimbus.py @@ -0,0 +1,252 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 9th Sept., 2024 + + OBJECTIVE: + + To be able to send SMSs from Nimbus's API and manage the templates and other things from one place. + + REFERENCES: + + 01. https://github.com/innovativevijay/SmsHitApiSample + 02. https://nimbusit.net/appforms/apimanual.php + + DOWNLOADS: + + N/A + + WEB-PORTAL: + + 01. http://nimbusit.net/ + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To make API Calls: +import httpx + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncNimbusSMS: + + MESSAGE_TYPE_REGULAR = 0 + MESSAGE_TYPE_UNICODE = 1 + + def __init__( + self, + entity_id, + sender_id, + user_id, + api_key, + debug = True, + debug_prefix = "Nimbus SMS | " + ): + + """ + Sets up an instance of the SMS sender through Nimbus IT. + :param entity_id: The entity id as registered with DLT. + :param sender_id: The 6-char code like "HDFCBK", "NSESMS", "ZRODHA" that you see in your SMS inbox. + :param user_id: The 6-digit id that Nimbus has assigned to you. + :param api_key: The API key generated through Nimbus's portal. + :param debug: Whether, or not, you would like to show debugging messages (can be changed on the fly). + :param debug_prefix: The prefix text to show with the debug string. + """ + + # Create the debugging tools: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # Create an HTTP client to work with: + limits = httpx.Limits( + max_connections = 5, + max_keepalive_connections = 5, + keepalive_expiry = 3600 + ) + self.__http_client = httpx.AsyncClient(limits = limits, timeout = 120) + + # Capture the input config: + self.__entity_id = entity_id + self.__sender_id = sender_id + self.__user_id = user_id, + self.__api_key = api_key + + async def get_balance(self): + + """ + Checks the balance in the Nimbus wallet. + :return: The balance (float) if the request was successful, else None. + """ + + balance = None + + try: + + # Call the API: + response = await self.__http_client.get( + url = r"http://nimbusit.net/api/balance", + params = {"user": self.__user_id, "authkey": self.__api_key} + ) + + # The response of a successful API call looks like "BALANCE:599". We need just the number: + if response.status_code in [200]: balance = float(response.content.decode().split(":")[-1].strip()) + + except Exception as exception: + self.__printer(exception) + + return balance + + async def send_sms( + self, + template_id, + recipient_number, + message, + message_type = MESSAGE_TYPE_REGULAR + ): + + """ + Sends one SMS through Nimbus IT's system. The text of the message must match the template that had been + submitted. A mismatch may cause the message to fail at best, and raise troubles in the real-world with + government bodies at worst. Be careful. + :param template_id: The id of the SMS template as registered on Nimbus's portal. + :param recipient_number: The phone number of the recipient. You can send an array of numbers, too, BUT IT IS + STRONGLY RECOMMENDED TO NOT DO THAT TO AVOID BEING BLOCKED BY DLT. + :param message: The message to send to the recipient. Should match the template that is being sent. + :param message_type: Choose between 'AsyncNimbusSMS.MESSAGE_TYPE_REGULAR' (default) and + 'AsyncNimbusSMS.MESSAGE_TYPE_UNICODE' based on the type of characters being sent. Both are class variables. + :return: The dict of all the details of the message that was sent including whether, or not, it was successfully + sent. Other details depend on the service provider (Nimbus IT in this case). + """ + + # Construct the basic structure of the response of this method: + summary = { + "success": False, + "info": None, + "sender": self.__sender_id, + "recipient": recipient_number, + "message": message, + "length": len(message), + "template_id": template_id, + "raw": None + } + + try: + + # Pre-process the recipient's number: + if not isinstance(recipient_number, (list, set, tuple)): recipient_number = [recipient_number] + + # Call the API: + response = await self.__http_client.get( + url = r"http://nimbusit.net/api/pushsms", + params = { + "user": self.__user_id, + "authkey": self.__api_key, + "sender": self.__sender_id, + "mobile": ",".join([str(num) for num in recipient_number]), + "text": message, + "entityid": self.__entity_id, + "templateid": template_id, + "type": message_type + } + ) + + # For a successful API call: + if response.status_code == 200: + response_json = response.json() + summary["success"] = True if response_json.get("STATUS", "ERROR").lower() in ["ok"] else False + summary["info"] = response_json.get("RESPONSE", {}).get("INFO") + summary["raw"] = { + "http_code": response.status_code, + "response": response_json, + } + + # For any other code that indicates some form of failure: + else: summary["raw"] = { + "http_code": response.status_code, + "response": response.content.decode() + } + + except Exception as exception: + self.__printer(exception) + + return summary + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + async def main(): + + sender = AsyncNimbusSMS( + entity_id = "", + sender_id = "", + user_id = "", + api_key = "" + ) + + response = await sender.send_sms( + template_id = "", + recipient_number = "", + message = "" + ) + print("SMS API RESPONSE:", response) + + my_balance = await sender.get_balance() + print("REMAINING BALANCE:", my_balance) + + + asyncio.run(main()) diff --git a/string/__init__.py b/string/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/string/__pycache__/__init__.cpython-310.pyc b/string/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8836d67c78a63461d9920b832af9129f52b5476f GIT binary patch literal 169 zcmd1j<>g`k0?k<(=^*+sh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6v9KO;XkRX-)Q zEHx*;Ahk$8pfWilu_!m7C_gJTxujS>u^=-(F+Dv#B|o_|H#M(BzqBMXr#QaMNWZwG mC^IizKR!M)FS8^*Uaz3?7Kcr4eoARhsvXFdVkRKL!T@O<7Yfm}%RHrTa-i##Ej@#UelHhPS^Z4G(``-KBh+DUs z4qV}%-aq^JM~?GX{3u@)`1mC}>i5tv=fq(y^Ah*Op0COfvfxZ=+V(&&wk**ucx`1(u5q7p%5A4@;tVa54)pYNp4?qs`(xn zQk_di7dzy^M|w&=!`|*scXy3;RQUOf8Ntysm&yq4?q-a1^RqSzeqgm9ee$@sKRW!P zY+@wHOp42xaY8e~salXZT@WqET*wPDkHOx(_PG+-Jpwi;NHH>r6Rz4;>7aMe8}$17 zyZTspC~Qk>uWk=|cJu;f| z*EKO?V)cXMfL>;(I^AuAhtg^!wxY9c68@Y&cHDz@zEjMYCoGJqYvc+{DSly?}v zKn=nOa|v|fRt8Huz1x+}#o(6o?hkL>44i(OMW9ae>R#tKzco`O^=q2Pp%2Od1&Y2Lv1|T`jQ=ku!VDvOjVhscZh%R0VfH~A{?X}61?x26z{}ll6=x|7mzUU1G zhYx#0(u3ZY{zL7~EU!znoy@eM(+Nk%)_Wy4Co?H8eUzgtlfmdj+0t5J% zJp2$k+F3dpD&JS0++I3R|6Htj(DR==*KStT)tj27MO|OIU%A)b5~^0{UH!#)cmN=7 z$ProAP`>rCA~A#%&TjnoolJ=;Y@*F51@4Nz7-bDnLwVNvQp05$D*|W{k89G;6x_%x2_tu!ts?RE}^90T^ zL&r7umkz4~#sU2sa&DQ=aJ_q^2X&q!0&9#`SO7wZ&W?rWG@ndBYK0BF43a{om|K** z5hq(Nzs1p;_*wHiFyz>l0pP|}kd%`76{`C`d;c%8x6FCxfF@9^VvHseiE|(ub0F9u z9A_}#6ixYSv1ZSyI60TG)^L6>p;O{^|20;YZ$rLaRdYSL|B>N`!kxph(of-NI64R$ zhG*qFASB<#<~?YNDr)CNjc0g*l;6jR3Qjc5gdvNtE`NaIZ>R*u<{OJZ9f|-Z;veBp zbr%}%y*23r?SIn>wt+;TC{(uHn%i>akI`rlMJbK5C<=YK1q1#5L3pdEM|jqYqN1V} zs_@Z^MUBp8JY#kzreNJ!9|PMUrFm+$VR+7N9Jy*^B?E^?QK@hl;xpAN_&GQ%Pxvp8 oV$}{bz7zbU1#1i{L8Vm*?s{+8oix7*!u772FM(j17pB4LpL zvj8oV0()_0a_IQbLoYoglasHV-uf5x-d=F+DV=FAy=B^I-&+78smM;!cBbu&$-(ly z{lb3y-tWC#n#YbsCHSbnnyh?~kfcAN()$?z`J;3=KTMD1kI|9*2tA%ZPDk^j z^hEvyJ()jAPvuY1)A`d>rSH%)pUL^RKpLZGL3$gcbM!n&?|}3!y#Ue~kSL1U{*UxqE9HNl(FypH!PxRjZM8k~K|(GX6yO3YSy4bWgo()1X>w zn#|p>t0dLnaS&(b=DoUWAAXcx%B_Cf+aPC>1~;GTlo5@TP2(vubY|FOS8EX)${61^ zd7W&qO8OZwLDg*CG%QlncNp2$4XPUz($H+1amdvSO04FVW$Sj+#uB^A>T&Pw8|jtw zMmn>U&i0IVRwY&2Zdl1gVt02pzI)XwS532K#miW(KCr{Y2=?#)ei+L_0bi$9J} zZECT0QT4=BEFXvk8c3J!yf^;-;}51L)zX!{PGPf9O#bNB?dka9&kBV?B34x6dpD*# z#X3y?8WxUyKFdo$BlI|Xa1Fo_C?>f&ZBUa*Ru z+Lh-!+7dvcur;}ugqri67^HK4>Okg4n&GUlE$4y89oTgEZ8;w`4R~r!s#$Tetl_Mc zZ6{+sb(UFKEm&8ehuP`5&h(-u&vzU@r;z9sy7~A{Y~j5-s0}Z-Y{#Gur;c64j=kYP z+SGVUUBq#L!hH=!2wJ%c1u#$$^%{u&ctK^L!KgvPwlH>(nwAYR%mHMZ)wdW26{Z8K z=_xQTP=X?uVOLYTGgJ#(?dzK}v74~-olV^3y@iga&-?mpXLB}oQxwg1@>u4}?K7`x z(2jduz+QI>-$QPHj`g+Jb}F`l2$}CB|2rA&)83TkJ2NPVb-C7Q$Iui!y&yn52bjIq zx#MRSgI}?UFp|(fSl;XuJo20n7 zxl>%Md?P>pMjp?tlohmb|AYPlpCx##`;bt%qc|b!h@;rj6M0PnEzC!S7DA1NX!tV; z%EFH9NRC347vW{;x!j4+2$nbzPje!`$^+W(kd={xH<;ZNBnRYY*~HxTsEz~ASwM7} zP3oFmt|I4=iVj5Tv!>6EL^ni{ZZw)UvDq^_F1k%NG_KV}Mv~;9Ef!(r-Ky^Q7W7~+ zM&VT^*#ho0f$IBu@N0RUc!V7^geEo9+<`{CuSr?6Sjl3BrQ0yzBzaV2FnT^sOim2b zo+d4`39bPA0ZwY4{{10f>)3wlzK*GBm$X_fNm8|%xl2GKS2>thaC->g(Km!pgRl&>DJ4t5izVEH~@m zcoX0^YgXy$TmrfTWF!r$WxW)`V1l7f~ykr>FW02_Zh~b=+OQ~$y9avt=rS9K%2XUYP1a$xw4Iu6g`upaN^uy2{^wGnQL&xr*hZF0- zn_UI_dxWpXfhfZx@`!vkJR&P$eikzNI3fZW`y4?)fz5^ObwW;<$}bdP2*n!4C=A#j zJO)@$740_%?H)~FTjyZAAH&LMwq1O?QAAluY&(Y%YZQ{u z(U9zY2mUrRI3)9PkXc5I1)n~jGzYohlOj$Sd{V^sNfAc@pM)CvB>Y8h6arL6Xy}Ck z2n=geVB)^#Y1FhbAWSn_J>c^litra*|KkI(&jIvtGuD5Vr|Aao804eB@wvXPQP$VX z>%0y6!d_ps-z)3)Sl#^6fkBDs?CRh)I1_Uh#^p1CXLY-bALa%7x#WMzr6ID#UhV2| zI7717pFbX{;$*dpYTWB!defq=$@PkEz!A{11)^K9G#HQsYbpTFM*wVK0Vm*xpTy)8 zB;cQ#Tk1d;^zQNfS2MBnT|*J1d(a97{ayGubY`I&P>vzpo(%H~pnQvJ(G)No`|Axz zzAGH-3B*D#fV*iJ4#a2x5$&shf|shSyknu+zlvkca=8hSMJn^tNB7{E83bEh+d!J| zPrCgeJPdig;PpU9iB{2c1IQmo5JVik0o>~h;{PuE%JHUP*`CRIbjz%1!fF;HD-dq@ zjL=11Zspt98?czh%hi{Mb1u6018Cy}^1gz{!iyr$v7ZhrvdDiW*7!xJ44C=D>o6nW z&}9*JAVT+8rxz$9ATJ~O(BO~+iD}eY1PD$Tu&r@%5&%VD!3*v^6YZnRHz+$K$p6ow zj3ZR|B~1DqxG4B7T>tj%U7;%9QSDeUmnaU7BtKh}uF^PuZ zCJ^K=K*O6#aiUbTUj`7(Hfp+!ULEKP4~0IpZ^5~W0?vA=24uh-U~r3J-073j8;HSi zFCgec&yoKk#9TP}A9SJ-3%8471n7nvW~0}k9`K)TRQOkMHE~B3aZVeKglJ@<%jF5 ziK{a==N4`(T$`Q0elsyUpPap(ytfB*M`V|S$E8xD%~z$7LC_axmHQ)@_ALNSgz&%nyz#)#+!EU^Jz&yoC&y9+!mC_T- zG!EFPJ7}9C|IcWrXzA7HDKkm(MUN_Bu&sicP63vi`Fn_xb80fQip zkq0e|I7McRTwlPfY!fbl;D46-7W}uOm3P|X-`Q-2RN5zI;tOO4t{K5Rbg$v}0In*F zd8;7^%PzTtCA?%RmEbVYvRnni)v7z7H5$yIVkO;?jr860C#Cd*)aw1x>T;m3r?S~c zYa7dM)GN=V9;Ab^hgmQp)K7b5OKX{2YAILBJbdtBdLw9o|53R|y&8A3xz&tcK& znV`<<`p5G<^0j?=CA$<<2ZMG`cr72KmP)DRWiZODd&S)ir6TL8#z}o;6BnF01|B0Y&X&XZnt}2 zBccwxh;oN+)nU(T>^69L$;v>YNJC^;9+ro~%29yNtG= threshold: + choice = target_df.at[0, "choice"] + all_matches[target] = choice + all_matches_df = all_matches_df[all_matches_df["choice"] != choice] + else: + all_matches[target] = None + something_is_null = True + + # print(all_matches) + if something_is_null and not allow_null: return None + else: return all_matches + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import async_json_utils + + awb_numbers = [ + "SF1111BIC", + "SF2222BIC", + "SF3333BIC", + "SF4444BIC", + ] + + chat_text = "SF1112BIC" + + # print(chat_text == names[0]) + best_match = get_best_match(chat_text, awb_numbers, threshold = 0.60, partial = False) + print(f"Best match for '{chat_text}' is '{best_match}'") diff --git a/string/help.py b/string/help.py new file mode 100644 index 0000000..85dcc97 --- /dev/null +++ b/string/help.py @@ -0,0 +1,177 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 24th Aug., 2024 + + OBJECTIVE: + + To provide an overview of any function or class in a string. + The generated overview can then either be shown on the terminal, or transmitted over some other medium for + collaborative work. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# System-level activities: +import io +import inspect + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_help_for_class(cls, skip_methods = None): + + """ + Returns the help documentation to use this class. + :param cls: The class whose help string is desired. + :param skip_methods: A list of methods to NOT include in the help text. + :return: This help documentation. + """ + + separator = "\n\n" + ("=" * 120) + "\n\n" + if skip_methods is None: skip_methods = [] + elif not isinstance(skip_methods, list): skip_methods = [skip_methods] + + # Get class name and docstring: + class_name = cls.__name__ + docstring = inspect.getdoc(cls) or "" + help_string = "HELP FOR:\n\n" + help_string += class_name + "\n\n" + help_string += "This document has upto 120 chars per line.\n" + help_string += "Best viewed with monospaced font :)" + help_string += docstring + separator + + # Get all methods and their docstrings. + # Then note the documentation of the methods while ignoring the blacklisted ones: + members = inspect.getmembers(cls, predicate = inspect.isfunction) + func_help = [] + for name, method in members: + + # Ignore if asked, or extract the details: + if name in skip_methods or name.startswith(f"_{class_name}__"): continue + else: func_help.append(get_help_for_function(method)) + + # Put all the things together: + func_help = separator.join(func_help) + help_string += func_help + + # Done here: + return help_string + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_help_for_function(func): + + """ + Get the help string for one function. + It could be a standalone function, or a method of a class. + :param func: The function (or method) whose help string is needed. + :return: The help string of the function. + """ + + # Get the name and documentation: + func_name = func.__name__ + async_indicator = " (async)" if inspect.iscoroutinefunction(func) else "" + func_doc = inspect.getdoc(func) or "" + + # Create the decorated header: + func_decorator = "-" * (len(func_name) + 2) + func_head = "." + func_decorator + f".\n| {func_name} |{async_indicator}\n`" + func_decorator + "`\n\n" + + # Add the 'args' and 'kwargs': + func_args = [] + for name, param in inspect.signature(func).parameters.items(): + default = param.default + if isinstance(default, str): default = f"\"{default}\"" + if default == inspect.Parameter.empty: func_args.append(f"{name}") + else: func_args.append(f"{name}: {type(default).__name__} = {default}") + if len(func_args) > 0: func_args = f"{func_name} (\n\t" + "\n\t".join(func_args) + "\n):\n\n" + else: func_args = f"{func_name} ():\n\n" + + # Get the params and return value part from the doc: + params_start = func_doc.find(":param") + return_start = func_doc.find(":return") + func_params = "\n" + func_doc[params_start:return_start] if params_start >= 0 else "" + func_return = "\n" + func_doc[return_start:] if return_start >= 0 else "" + + # Isolate the documentation part: + if params_start >= 0: func_doc = func_doc[:params_start] + elif return_start >= 0: func_doc = func_doc[:return_start] + + # Done here: + return func_head + func_args + func_doc + func_params + func_return + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_help(entity, skip_methods = None): + + """ + Get the help documentation for anything from its docstring. + :param entity: The entity you want to get help for. + :param skip_methods: A list of methods to ignore if inspecting a class. Not valid for standalone functions. + :return: The help string. + """ + + if inspect.isclass(entity): return get_help_for_class(entity, skip_methods = skip_methods) + else: return get_help_for_function(entity) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/string/json.py b/string/json.py new file mode 100644 index 0000000..67d39f7 --- /dev/null +++ b/string/json.py @@ -0,0 +1,202 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Create: Saturday, 18th May, 2022 + Update: Thursday, 22nd Aug. 2024 + + OBJECTIVE: + + To provide an easy way to work with '.json' data and files. + + REFERENCES: + + 1) https://www.w3schools.com/python/python_json.asp + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To work with the JSON standard: +import json + +# To work with files: +from utils_v2.system import files + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def from_string(json_data): + + """ + Decodes a JSON string to a pythonic variable like a dict. + :param json_data: The JSON string to decode. + :return: The decoded pythonic variable. + """ + + python_data = json.loads(json_data) + return python_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_string( + python_data, + indent = 4, + default = None, + separators = None, + no_space = False +): + + """ + Converts the given pythonic data to a JSON string. + :param python_data: The input data like a dict. + :param indent: The tab-width for pretty presentation. + :param default: The function to use on something that cannot be directly parsed into a JSON string. + :param separators: Custom separators to use. + :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not + good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT + 'default'. + :return: The JSON string representation of the input pythonic data. + """ + + if no_space: + json_data = json.dumps( + python_data, + default = default, + separators = (',', ':') + ) + + else: + json_data = json.dumps( + python_data, + indent = indent, + default = default, + separators = separators + ) + + return json_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +def from_file(file): + + """ + Reads a JSON file and returns it as a pythonic variable like a dict. + :param file: The path to the file on the disk or a file held in RAM as a BytesIO object. + :return: The decoded pythonic variable. + """ + + if isinstance(file, io.BytesIO): + file.seek(0) + json_data = file.getvalue() + else: json_data = files.read_file(file) + python_data = from_string(json_data) + return python_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_file( + file, + python_data, + indent = 4, + default = None, + separators = None, + no_space = False +): + + """ + + :param file: Either a path to a file on disk, or a buffer in RAM in the form of a BytesIO object. + :param python_data: The pythonic data to be converted to the JSON string. + :param indent: The tab-width for pretty presentation. + :param default: The function to use on something that cannot be directly parsed into a JSON string. + :param separators: Custom separators to use. + :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not + good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT + 'default'. + :return: True/False if a path was given, else the same BytesIO object with the written JSON data. + """ + + json_data = to_string( + python_data, + indent = indent, + default = default, + separators = separators, + no_space = no_space + ) + + if isinstance(file, io.BytesIO): + file.write(json_data.encode("utf-8")) + file.seek(0) + return file + + else: + try: + files.write_file(file, json_data, mode = "w") + return True + except: return False + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/string/regex.py b/string/regex.py new file mode 100644 index 0000000..469a942 --- /dev/null +++ b/string/regex.py @@ -0,0 +1,288 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Sunday, 28th Apr., 2024 + + OBJECTIVE: + + To provide a convenient way to perform RegEx operations like finding patterns and substituting them. + + REFERENCES: + + 1) https://www.w3schools.com/python/python_regex.asp + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with RegEx: +import re + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Common RegEx patterns: +REGEX_EMAIL_ID = r"[\d\w_.+]*@[\d\w_]*.[\d\w]{2,}" +REGEX_PASSWORD = r"^(?=.*[a-z])(?=.*[A-Z])(?=.*[\d])(?=.*[!@#$%^&*()_+{}\[\]:;<>,.?~\\\/-]).{8,}$" +REGEX_NAME = r"^[\d\w .\-]{1,30}$" +REGEX_USERNAME = r"^[\d\w_]{8,25}$" +REGEX_CONTACT_NUMBER = r"\+?\d{0,3}\s*\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}" +REGEX_DATE = r"\b(?:\d{4}-\d{2}-\d{2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},?\s+\d{4}|\d{1,2}\/\d{1,2}\/\d{4}|\d{1,2}-\d{1,2}-\d{2}|\d{1,2}(?:st|nd|rd|th)\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?),?\s+\d{4})\b" +REGEX_GSTIN = r"[0-9]{2}[A-Z]{5}[0-9]{4}[A-Z]{1}[1-9A-Z]{1}Z[0-9A-Z]{1}" +REGEX_PAN = r"[A-Z]{5}[0-9]{4}[A-Z]{1}" +REGEX_IPV4 = (r"[0-9]{1,3}\." * 3) + r"[0-9]{1,3}" +REGEX_IPV6 = (r"[0-9a-fA-F]{1,4}:" * 7) + r"[0-9a-fA-F]{1,4}" +REGEX_IFSC = r"[A-Z]{4}0[A-Z0-9]{6}" +REGEX_UPI = r"[a-zA-Z0-9\.\-]{2,256}@[a-zA-Z][a-zA-Z]{2,64}" +REGEX_MAC_ADDRESS = r"([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})|([0-9a-fA-F]{4}\\.[0-9a-fA-F]{4}\\.[0-9a-fA-F]{4})" +REGEX_METRIC_WEIGHT = r"[\d\.]+[ ]?[k]?g" + + +# RegEx chars (append them to the patterns if needed): +REGEX_START = "^" +REGEX_END = "$" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def find(text, pattern, case_sensitive = True, dot_all = False): + + """ + Returns a list of substrings that match the given RegEx pattern in the input text. + :param text: The text in which the pattern needs to be found. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: An array (list) of substring that match the pattern. Can be an empty list as well. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and clean the results: + matches = [match if type(match) is str else match[1] for match in re.findall(pattern, text, flags = flags)] + matches = [match for match in matches if len(match) > 0] + + # Return the results: + return matches + + +# --------------------------------------------------------------------------------------------------------------------- + + +def find_first(text, pattern, case_sensitive = True, dot_all = False): + + """ + Returns the first substring that matches the given RegEx pattern in the input text. + :param text: The text in which the pattern needs to be found. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: The first match as a string, or None if no match was found.. + """ + + matches = find( + text = text, + pattern = pattern, + case_sensitive = case_sensitive, + dot_all = dot_all + ) + + if not matches: return None + else: return matches[0] + + +# --------------------------------------------------------------------------------------------------------------------- + + +def replace(text, pattern, substitute_text, case_sensitive = True, dot_all = False): + + """ + Replaces any substring in the text that matches the RegEx pattern. + :param text: The text in which the substitutions need to be made. + :param pattern: The RegEx pattern that needs to be substituted. + :param substitute_text: The text that will replace the matches that were found. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: The text with the substitutions. If no matches are found, the original string is returned. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + return re.sub(pattern, substitute_text, text, flags = flags) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def search(text, pattern, case_sensitive = True, dot_all = False): + + """ + Checks if the given RegEx pattern occurs ANYWHERE in the text that was provided. + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: True if the pattern matches, else False. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + if re.search(pattern, text, flags = flags): return True + else: return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def match(text, pattern, case_sensitive = True, dot_all = False): + + """ + Checks if the given text matches the RegEx pattern that was provided. The check is made only at the start of the + input string. + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: True if the pattern matches, else False. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + if re.match(pattern, text, flags = flags): return True + else: return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def split(text, pattern, case_sensitive = True, dot_all = False): + + """ + Splits an input string based on the pattern that is being matched. + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: True if the pattern matches, else False. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + substrings = re.split(pattern, text, flags = flags) + if len(substrings) > 0 and substrings[0] == "": substrings.pop(0) + return substrings + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_json(text, pattern, case_sensitive = True, dot_all = False): + + """ + Gives out a dict from the extracted features in a string. It is based on the concept of Named Groups. + Consider the following example (assuming the search is case-insensitive): + TEXT: "UPI/309258561479/14:17:35/UPI/omsainurses@okhdfc" + PATTERN: "upi/.*/(?P