commit 8494a2c0c077cd73fd582b332550d2bcbc4d0a3d Author: khushal Date: Tue Oct 1 10:33:59 2024 +0530 Squashed 'utils_v2/' content from commit ef9630d git-subtree-dir: utils_v2 git-subtree-split: ef9630d728847764d4832ce8f3f956571901383a diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ccfbee4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/.venv/ +/.idea/ +/__pycache__/ +*.pem \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..76abec7 --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,38 @@ + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/material_theme_project_new.xml b/.idea/material_theme_project_new.xml new file mode 100644 index 0000000..2236d19 --- /dev/null +++ b/.idea/material_theme_project_new.xml @@ -0,0 +1,12 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..9c49622 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..e89cf70 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/utils_v2.iml b/.idea/utils_v2.iml new file mode 100644 index 0000000..2c80e12 --- /dev/null +++ b/.idea/utils_v2.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/__init__.py b/ai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/image_classification/__init__.py b/ai/image_classification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/image_classification/async_blur.py b/ai/image_classification/async_blur.py new file mode 100644 index 0000000..c921a4e --- /dev/null +++ b/ai/image_classification/async_blur.py @@ -0,0 +1,202 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 2nd Sept., 2024 + + OBJECTIVE: + + To provide an easy way to assess images for blurriness. + Tried and implemented using HuggingFace models. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# For using the AI model: +import torch +from transformers import pipeline + +# To download images from URLs: +import requests + +# To read images: +from PIL import Image + +# Common: +from shared import variables + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AssessImageBlur: + + def __init__( + self, + model, + blur_label = "Blur", + clarity_label = "Normal", + blur_threshold = 0.25, + clarity_threshold = 0.65 + ): + + """ + This class assesses the input image and tells if it is blurry or clear. + :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is + stored. + :param blur_threshold: The max allowed blurriness (0 to 1 range). + :param clarity_threshold: The minimum needed clarity (0 to 1 range). + """ + + # Note down the config: + self.__model = model + self.__device = "gpu" if torch.cuda.is_available() else "cpu" + self.__blur_label = blur_label + self.__clarity_label = clarity_label + self.__blur_threshold = blur_threshold + self.__clarity_threshold = clarity_threshold + + # Initialize the classifier: + self.__classifier = pipeline( + task = "image-classification", + model = self.__model, + device = self.__device + ) + + @staticmethod + async def read_image(source): + + """ + Reads an image in whichever format it is provided and returns it as a PIL object. + :param source: The image as either a path or a URL or a io.BytesIO object. + :return: The image opened as a PIL object. + """ + + # If the input image is already a PIL image: + if isinstance(source, Image.Image): return source + + # If a buffer is provided: + elif isinstance(source, io.BytesIO): + source.seek(0) + return Image.open(source) + + # If a string is provided (local path or URL): + elif isinstance(source, str): + if source.startswith("http://") or source.startswith("https://"): + response = await variables.http_client.get(source) + return Image.open(io.BytesIO(response.content)) + else: return Image.open(source) + + async def classify(self, image): + + """ + To get the prediction of the model from the given input image. + :param image: The image (as a PIL object or file path or a URL). + :return: The dictionary of classes with their respective predictions. + """ + + image = await self.read_image(image) + predictions = self.__classifier(image) + classes = {p["label"]: p["score"] for p in predictions} + return classes + + async def is_ok( + self, + image, + blur_threshold = None, + clarity_threshold = None + ): + + """ + Checks if the image given to it can be used, or should be rejected. + :param image: The image (as a PIL object or file path or a URL). + :param blur_threshold: A custom threshold to test against. If not provided, the default will be taken that was + provided when the instance was created. + :param clarity_threshold: A custom threshold to test against. If not provided, the default will be taken that + was provided when the instance was created. + :return: True if the image is okay, else False. + """ + + image = await self.read_image(image) + blur_threshold = blur_threshold or self.__blur_threshold + clarity_threshold = clarity_threshold or self.__clarity_threshold + classes = await self.classify(image) + if ( + classes[self.__blur_label] <= blur_threshold and + classes[self.__clarity_label] >= clarity_threshold + ): return True + return False + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + + async def main(): + image = r"/home/developer/Downloads/low-res-check.png" + assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/image_classification/BlurOrBokeh") + usable = await assessor.is_ok(image) + classes = await assessor.classify(image) + print("IS OKAY:", usable) + print("CLASSES:", classes) + + asyncio.run(main()) diff --git a/ai/image_classification/async_nsfw.py b/ai/image_classification/async_nsfw.py new file mode 100644 index 0000000..c59189b --- /dev/null +++ b/ai/image_classification/async_nsfw.py @@ -0,0 +1,188 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 2nd Sept., 2024 + + OBJECTIVE: + + To provide an easy way to assess images for adult content. + Tried and implemented using HuggingFace models. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# For using the AI model: +import torch +from transformers import pipeline + +# To download images from URLs: +import requests + +# To read images: +from PIL import Image + +# Common: +from shared import variables + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AssessImageNSFW: + + def __init__( + self, + model, + nsfw_label = "nsfw", + nsfw_threshold = 0.25, + ): + + """ + This class assesses the input image and tells if it is blurry or clear. + :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is + stored. + :param nsfw_threshold: The max allowed blurriness (0 to 1 range). + """ + + # Note down the config: + self.__model = model + self.__device = "gpu" if torch.cuda.is_available() else "cpu" + self.__nsfw_label = nsfw_label + self.__nsfw_threshold = nsfw_threshold + + # Initialize the classifier: + self.__classifier = pipeline( + task = "image-classification", + model = self.__model, + device = self.__device + ) + + @staticmethod + async def read_image(source): + + """ + Reads an image in whichever format it is provided and returns it as a PIL object. + :param source: The image as either a path or a URL or a io.BytesIO object. + :return: The image opened as a PIL object. + """ + + # If the input image is already a PIL image: + if isinstance(source, Image.Image): return source + + # If a buffer is provided: + elif isinstance(source, io.BytesIO): + source.seek(0) + return Image.open(source) + + # If a string is provided (local path or URL): + elif isinstance(source, str): + if source.startswith("http://") or source.startswith("https://"): + response = await variables.http_client.get(source) + return Image.open(io.BytesIO(response.content)) + else: return Image.open(source) + + async def classify(self, image): + + """ + To get the prediction of the model from the given input image. + :param image: The image (as a PIL object or file path or a URL). + :return: The dictionary of classes with their respective predictions. + """ + + image = await self.read_image(image) + predictions = self.__classifier(image) + classes = {p["label"]: p["score"] for p in predictions} + return classes + + async def is_ok( + self, + image, + nsfw_threshold = None + ): + + """ + Checks if the image given to it can be used, or should be rejected. + :param image: The image (as a PIL object or file path or a URL). + :param nsfw_threshold: A custom threshold to test against. If not provided, the default will be taken that was + provided when the instance was created. + :return: True if the image is okay, else False. + """ + + image = await self.read_image(image) + nsfw_threshold = nsfw_threshold or self.__nsfw_threshold + classes = await self.classify(image) + return True if classes[self.__nsfw_label] <= nsfw_threshold else False + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + + async def main(): + + assessor = AssessImageNSFW(model = r"/path/to/model") + usable = await assessor.is_ok(r"/path/to/image/img.jpg") + classes = await assessor.classify(r"https://...") + + + asyncio.run(main()) diff --git a/ai/image_segmentation/__init__.py b/ai/image_segmentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/image_segmentation/async_dichotomous_segmentation.py b/ai/image_segmentation/async_dichotomous_segmentation.py new file mode 100644 index 0000000..04ec460 --- /dev/null +++ b/ai/image_segmentation/async_dichotomous_segmentation.py @@ -0,0 +1,223 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 14th Sept., 2024 + + OBJECTIVE: + + To provide an easy way to get masks from dichotomous image segmentation. + + This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait". + You may experiment with other models too, but make sure that model is made for "dichotomous" behaviour. This + means that the model should have only two classes like "foreground", and "background". The specified model was + trained for implementing portrait mode style blurring of backgrounds. + + The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not + support drop-in replacement for other models, please be aware about this. + + REFERENCES: + + 01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait + 02. https://github.com/ZhengPeng7/BiRefNet + + DOWNLOADS: + + 01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait/tree/main + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For system-level activities: +import io + +# To work with PIL images: +from PIL import Image + +# To make asynchronous HTTP calls: +from shared.variables import http_client + +# For using the AI model: +import torch +from torchvision import transforms +from transformers import AutoModelForImageSegmentation + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DichotomousSegmenter: + + def __init__( + self, + model = r"zhengpeng7/BiRefNet-portrait" + ): + + """ + This class uses Dichotomous Image Segmentation to produce a mask of what is in the foreground (or what is of + interest in a given image). + :param model: The path of the model. + """ + + # Check if GPU is available for faster predictions: + self.__cuda_is_available = True if torch.cuda.is_available() else False + + # Initialize the model: + self.__model = AutoModelForImageSegmentation.from_pretrained( + pretrained_model_name_or_path = model, + trust_remote_code = True + ) + torch.set_float32_matmul_precision(["high", "highest"][0]) + self.__model.to("cuda" if self.__cuda_is_available else "cpu") + + @staticmethod + async def read_image(source): + + """ + Reads an image in whichever format it is provided and returns it as a PIL object. + :param source: The image as either a path or a URL or a io.BytesIO object. + :return: The image opened as a PIL object. + """ + + # If the input image is already a PIL image: + if isinstance(source, Image.Image): return source + + # If a buffer is provided: + elif isinstance(source, io.BytesIO): + source.seek(0) + return Image.open(source) + + # If a string is provided (local path or URL): + elif isinstance(source, str): + if source.startswith("http://") or source.startswith("https://"): + response = await http_client.get(source) + return Image.open(io.BytesIO(response.content)) + else: return Image.open(source) + + async def get_mask(self, image): + + """ + Process the image to figure out the mask. In some cases (like the image of a sunset) you will have no white + pixels in the mask. White pixels represent the areas that the AI considered to be the foreground (area of + interest) and the black parts are the background. + :param image: The image data as either a PIL object, or a path to a file on the local disk, or a URL. + :return: The mask as a PIL object. + """ + + # Prepare the image transformer: + image_size = (1024, 1024) + transform_image = transforms.Compose([ + transforms.Resize(image_size), + transforms.ToTensor(), + transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ]) + + # Read and transform the image: + image = await self.read_image(image) + input_image = transform_image(image).unsqueeze(0).to("cuda" if self.__cuda_is_available else "cpu") + + # Make the prediction: + with torch.no_grad(): + if self.__cuda_is_available: mask = self.__model(input_image)[-1].sigmoid().cuda() + else: mask = self.__model(input_image)[-1].sigmoid().cpu() + + # Create a PIL object of the mask data: + mask = mask[0].squeeze() + mask = transforms.ToPILImage()(mask) + mask = mask.resize(image.size) + + # Done here: + return mask + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + import numpy as np + + async def main(): + + model = DichotomousSegmenter() + + images = [ + # r"/home/developer/Downloads/kate.jpg", + # r"/home/developer/Downloads/IMG-20240730-WA0001.jpg", + # r"/home/developer/Downloads/pexels-pixabay-57416.jpg", + # r"/home/developer/Downloads/summer_clothes.jpg", + # r"/home/developer/Downloads/low_res_cat - upscaled.png", + # r"/home/developer/Downloads/blurry_traffic.jpg", + r"/home/developer/Downloads/card_0.jpg", + r"/home/developer/Downloads/card_1.jpg", + r"/home/developer/Downloads/card_2.jpg", + r"/home/developer/Downloads/card_3.jpg", + r"/home/developer/Downloads/sushmita_card.jpg", + r"/home/developer/Downloads/niranjan_card.jpg", + r"/home/developer/Downloads/niranjan_card_2.jpg", + ] + + for image in images: + print(image.split("/")[-1]) + image = await DichotomousSegmenter.read_image(image) + mask = await model.get_mask(image) + image.show() + mask.show() + print("MASK!") + array = np.array(mask) + print("Dimensions of the array:", array.shape) + masked_count = np.sum(array > 125) + print(masked_count, type(masked_count), np.size, type(np.size)) + percentage = masked_count / np.prod(array.shape) + print("Area:", round(percentage * 100, 4)) + # image.putalpha(mask) + # image.show() + # mask.show() + + + asyncio.run(main()) diff --git a/ai/object_detection/__init__.py b/ai/object_detection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/object_detection/yolo.py b/ai/object_detection/yolo.py new file mode 100644 index 0000000..91747f2 --- /dev/null +++ b/ai/object_detection/yolo.py @@ -0,0 +1,176 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 4th Jul, 2024 + + OBJECTIVE: + + To provide a class to detect objects in images using YOLO models. + + REFERENCES: + + 1) Code Examples: https://docs.ultralytics.com/usage/python/ + 2) Models: https://docs.ultralytics.com/models/yolov8/#supported-tasks-and-modes + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import os + +# To use YOLO architecture: +from ultralytics import YOLO + +# To work with images: +from PIL import Image + +# utils: +from utils_v2.string import json + +# Common: +from shared import constants + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class YoloDetect: + + def __init__( + self, + model_file, + debug = True, + debug_prefix = "YOLO | " + ): + + # Load the model: + self._model = YOLO(model = model_file) + + # Initialize the dbugging tool: + self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self._printer.disable() + + def disable_debug(self): + self._printer.disable() + + def enable_debug(self): + self._printer.enable() + + @staticmethod + def _process_result(raw_result): + + # Extract the various types of results available from the inference: + speed = {k: (v / 100.0) for k, v in raw_result.speed.items()} + class_mapping = raw_result.names + probabilities = raw_result.probs + boxes = raw_result.boxes + masks = raw_result.masks + + # Construct a default response: + response_json = { + "speed": speed, + "classes": class_mapping, + "boxes": None + } + + # Object-Detection results: + formatted_boxes = [] + for box in boxes: + detected_class = int(box.cls.numpy()[0]) + x1, y1, x2, y2 = box.xyxy[0] + formatted_boxes.append( + { + "class": detected_class, + "className": class_mapping[detected_class], + "confidence": float(box.conf.numpy()[0]), + "x1": int(x1.numpy()), + "y1": int(y1.numpy()), + "x2": int(x2.numpy()), + "y2": int(y2.numpy()) + } + ) + response_json["boxes"] = formatted_boxes + + # Done here: + return response_json + + def predict(self, image, show = False, verbose = False): + + # Process the input image with the given task: + results = self._model( + source = image, + show = show, + verbose = verbose + ) + + # Based on the task, interpret the results: + results_json = self._process_result(results[0]) + + # Done here: + return results_json + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "models", "yolov8x.pt") + model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt" + # sample_image_path = r"/home/developer/Downloads/2_cats.jpg" + sample_image_path = r"/home/developer/Downloads/flattened_image.jpg" + my_yolo = YoloDetect(model_file = model_file_path) + results = my_yolo.predict(image = Image.open(sample_image_path)) + print("FINAL RESULTS:", json.to_string(results)) + diff --git a/ai/tts/__init__.py b/ai/tts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai/tts/easy_tts.py b/ai/tts/easy_tts.py new file mode 100644 index 0000000..fc4bb9b --- /dev/null +++ b/ai/tts/easy_tts.py @@ -0,0 +1,175 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 26th Sept., 2024 + + OBJECTIVE: + + To provide a fast way to make audio from TTS engines. + + REFERENCES: + + 01. Usage: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#python-api + 02. Installation: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#linux-and-macos-install + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To use the AI: +from melo.api import TTS +import numpy as np +import soundfile + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class EasyTTS: + + __sampling_rate = 44_100 + + def __init__( + self, + language = "EN", + speed = 1.0 + ): + + self.__audio = np.zeros(1) + self.__language = language + self.__speed = speed + self.__model = TTS(language = language, device = "auto") + self.__speakers = self.__model.hps.data.spk2id + + def list_speakers(self): + return list(self.__speakers.keys()) + + def speak( + self, + text, + speaker + ): + + this_audio = self.__model.tts_to_file( + text, + self.__speakers[speaker], + speed = self.__speed, + quiet = True + ) + + self.__audio = np.concatenate((self.__audio, this_audio)) + + def pause(self, seconds): + + self.__audio = np.concatenate(( + self.__audio, + np.zeros(int(self.__sampling_rate * seconds)) + )) + + def to_wav(self, path = None): + + if path is None: + audio_buffer = io.BytesIO() + soundfile.write(audio_buffer, self.__audio, self.__sampling_rate) + return audio_buffer + + else: soundfile.write(path, self.__audio, self.__sampling_rate) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + speaker = "EN-BR" + tts_maker = EasyTTS(language = "EN", speed = 0.9) + + tts_maker.speak( + text = """ + Imagine delighting your doctors with a personalized calendar, + crafted from their own cherished memories. + """, + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = """ + Every day, as they turn the page, + they’ll not only relive those special moments but also remember you, + the one who made it happen. + """, + speaker = speaker + ) + tts_maker.pause(seconds = 0.75) + tts_maker.speak( + text = "STEP 1:", + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = "Start by identifying the doctors you’d like to engage with, and add them to our app.", + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = "No rush, you can add their photographs later as well.", + speaker = speaker + ) + tts_maker.pause(seconds = 0.3) + tts_maker.speak( + text = "With this, your engagement funnel is created.", + speaker = speaker + ) + + tts_maker.to_wav(r"/home/developer/Downloads/voiceover.wav") diff --git a/ai/tts/test.wav b/ai/tts/test.wav new file mode 100644 index 0000000..d38cfc0 Binary files /dev/null and b/ai/tts/test.wav differ diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/__pycache__/__init__.cpython-310.pyc b/api/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..2ee960f Binary files /dev/null and b/api/__pycache__/__init__.cpython-310.pyc differ diff --git a/api/__pycache__/async_quart.cpython-310.pyc b/api/__pycache__/async_quart.cpython-310.pyc new file mode 100644 index 0000000..e948687 Binary files /dev/null and b/api/__pycache__/async_quart.cpython-310.pyc differ diff --git a/api/__pycache__/codes.cpython-310.pyc b/api/__pycache__/codes.cpython-310.pyc new file mode 100644 index 0000000..b7c160c Binary files /dev/null and b/api/__pycache__/codes.cpython-310.pyc differ diff --git a/api/__pycache__/metrics_prometheus.cpython-310.pyc b/api/__pycache__/metrics_prometheus.cpython-310.pyc new file mode 100644 index 0000000..2970917 Binary files /dev/null and b/api/__pycache__/metrics_prometheus.cpython-310.pyc differ diff --git a/api/__pycache__/response.cpython-310.pyc b/api/__pycache__/response.cpython-310.pyc new file mode 100644 index 0000000..bd3d7e5 Binary files /dev/null and b/api/__pycache__/response.cpython-310.pyc differ diff --git a/api/async_quart.py b/api/async_quart.py new file mode 100644 index 0000000..0c29f69 --- /dev/null +++ b/api/async_quart.py @@ -0,0 +1,1007 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 2nd Aug., 2024 + + OBJECTIVE: + + To provide an easy way to perform repetitive tasks in quart. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + 01. PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING ANY OF THESE + DECORATORS. + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To work with Quart: +from quart import request, current_app, g + +# To make decorators: +from functools import wraps + +# My utils: +from utils_v2.string import json +from utils_v2.date_time import date_time +from utils_v2.security import sanitizers +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.response import ResponseModel +from utils_v2.api.metrics_prometheus import ( + TOTAL_REQUEST_COUNT, + LIVE_REQUEST_COUNT, + REQUEST_LATENCY, + MetricsAPI +) + +# To work with date and time: +import time +import datetime + +# System-level activities: +import io +import distro +import socket +import platform + +# For Pydantic data-models: +import pydantic + +# For hashing and shortening the hash: +import hashlib +import base64 + +# To make things human-readable: +import humanize + +# For debugging: +import traceback +import random +import string + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Info for logging that will stay constant during runtime: +SERVER_HOSTNAME = socket.gethostname() +PLATFORM_INFO = platform.uname() +HOST_OS = distro.name(True) + +# Chars to choose from for random strings: +ALPHANUMERIC_CHARS = string.ascii_letters + string.digits + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class AuthDetailsIncompleteException(Exception): + def __str__(self): + return "incomplete auth details" + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +async def data_from_request(inbound_request): + + """ + Adaptively extract the params from incoming request in whichever way it was provided. + :param inbound_request: The request that came in. + :return: A dict (could be empty) of the data/params that came in with the request. + """ + + inbound_data = {} + + # Extract data from the params in the URL: + from_args = inbound_request.args.to_dict() + if isinstance(from_args, dict): + for k, v in from_args.items(): + inbound_data[k] = v + + # Extract data from the raw JSON data: + from_json = await request.get_json() + if isinstance(from_json, dict): + for k, v in from_json.items(): + inbound_data[k] = v + + # Extract inputs from the form body: + from_form = await inbound_request.form + from_form = from_form.to_dict() + if isinstance(from_form, dict): + for k, v in from_form.items(): + inbound_data[k] = v + + return inbound_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def file_from_request(inbound_request, file_key): + + """ + Extracts ONE file from the incoming request's form-data. + WARNING: NOT RECOMMENDED FOR LARGE FILES. STRICTLY USE FOR SMALL FILES THAT WON'T CRASH THE SCRIPT. + :param inbound_request: The incoming request. + :param file_key: The key of the file that you want to extract. + :return: A tuple of the file's name and data. + """ + + file_name = None + file_data = None + files = await inbound_request.files + + if file_key in files: + file_name = files[file_key].filename + file_data = io.BytesIO(files[file_key].read()) + + return file_name, file_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def files_from_request(inbound_request: request): + + """ + Extracts ALL files from the incoming request's form-data. + WARNING: NOT RECOMMENDED FOR LARGE FILES. STRICTLY USE FOR SMALL FILES THAT WON'T CRASH THE SCRIPT. + :param inbound_request: The incoming request. + :return: A dict describing the file's name, data, and size. + """ + + files = await inbound_request.files + + inbound_files = {} + for file_key in files: + file_data = io.BytesIO(files[file_key].read()) + file_size = file_data.seek(0, 2) + file_data.seek(0) + inbound_files[file_key] = { + "name": files[file_key].filename, + "data": file_data, + "size": file_size, + "type": files[file_key].content_type + } + + return inbound_files + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def headers_from_request( + inbound_request, + mandatory_keys: list = None +): + + """ + Extract custom headers and some extra info. from the incoming request. + Raises an exception if any mandatory key is missing. + IMPORTANT: CUSTOMIZE THIS FOR THE NEEDS OF YOUR PROJECT. + :param inbound_request: The request that came in. + :param mandatory_keys: The keys that you need to have in the auth. + :return: The extracted auth details. + """ + + # Start by extracting whatever complies with the format of "X-{Header-Name}": + head_json = {k: v for k, v in request.headers.items() if k.startswith("X-")} + + # Now note down things that you want to keep from incoming requests: + head_json["Remote-IP"] = inbound_request.remote_addr + head_json["Host"] = inbound_request.headers.get("Host") + head_json["Origin"] = inbound_request.headers.get("Origin") + head_json["User-Agent"] = inbound_request.headers.get("User-Agent") + + # Raise an exception if any of the mandatory auth details were missing: + if mandatory_keys is not None: + available_keys = head_json.keys() + for mandatory_key in mandatory_keys: + if mandatory_key not in available_keys: raise AuthDetailsIncompleteException + + # Done here: + return head_json + + +# --------------------------------------------------------------------------------------------------------------------- + + +def cause_exception(): + + """ + Call this from any function when you want to raise an exception. + Example use case would be when receiving data from an API call and that field is not supposed to be null. + :return: None. + """ + + return 100/0 + + +# --------------------------------------------------------------------------------------------------------------------- + + +def describe_exception(exc): + + """ + Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire + traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it + to the dev team over some service like WhatsApp/Telegram. + :param exc: The exception that occurred. + :return: The dict that explains the exception. + """ + + exc_desc = { + "type": type(exc).__name__, + "msg": str(exc), + "tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)] + } + + return exc_desc + + +# --------------------------------------------------------------------------------------------------------------------- + + +def messages_from_pydantic_exception(exception, as_str = True, sep = ", "): + + """ + Creates a list of readable error messages from Pydantic's validation failure. + :param exception: Pydantic's ValidationError + :param as_str: Set to True to receive all messages as one string, False to receive an array of strings. + :param sep: The separator to use when joining multiple messages as one string. + :return: A list of messages of all the things that went wrong. + """ + + # Make a variable to hold all individual messages: + messages = [] + + # Interpret all the problems: + for error in exception.errors(): + loc = " --> ".join([str(item) for item in error["loc"]]) + if error["type"] == "missing": messages.append(f"missing input: {loc}") + elif error["type"] == "model_type": messages.append(f"invalid input: {loc}") + elif error["type"] == "bool_parsing": messages.append(f"invalid bool: {loc}") + elif error["type"] == "string_type": messages.append(f"invalid string: {loc}") + elif error["type"] == "float_parsing": messages.append(f"invalid float: {loc}") + elif error["type"] == "int_parsing": messages.append(f"invalid integer: {loc}") + elif error["type"] == "extra_forbidden": messages.append(f"extra input: {loc}") + elif error["type"] == "value_error": messages.append(f"validation failed: {loc}") + else: messages.append(f"invalid datatype: {loc}") + + # Return a response as per the preference of the user: + if as_str: return sep.join(messages) + else: return messages + +# --------------------------------------------------------------------------------------------------------------------- + + +def set_api_version(api_version): + + """ + Use this decorator to automatically note down the API version no. and propagate it throughout the downstream + decorators. Use this as the entry point if possible. + :param api_version: The version code to assign to the API. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # set the version information in the variable. + # This makes it available to the downstream decorators too! + kwargs["api_version"] = api_version + + # we are ready to call the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + if isinstance(response, ResponseModel): response.api_version = api_version + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def read_input( + sanitize_headers = True, + sanitize_data = True +): + + """ + Use this decorator to read the inputs from the incoming request and sanitize them. Sanitization makes the inputs + safe against certain threats like injections attacks. If you expect to take in inputs that you want to use to run + database commands, you could disable them manually. + PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING THIS DECORATOR. + :param sanitize_headers: Whether, or not, you would like to sanitize the params coming in through the headers. + :param sanitize_data: Whether, or not, you would like to sanitize the params coming in through the body or query. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Get the headers: + kwargs["inbound_headers"] = await headers_from_request(request) + if sanitize_headers: kwargs["inbound_headers"] = sanitizers.for_mongo(kwargs["inbound_headers"]) + + # Get the data: + kwargs["inbound_data"] = await data_from_request(request) + if sanitize_data: kwargs["inbound_data"] = sanitizers.for_mongo(kwargs["inbound_data"]) + + # Get small files from the request: + kwargs["inbound_files"] = await files_from_request(request) + + # We also make a provision for capturing an identifier + # for the logs that we make through a sister decorator: + kwargs["log_id"] = "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(8)) + + # Now that we have unpacked the incoming data, + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def validate_input( + mandatory_header_keys = None, + mandatory_data_keys = None, + mandatory_file_keys = None, + header_validator = None, + data_validator = None +): + + """ + USE THIS ONLY AFTER YOU HAVE USED 'read_input'. This decorator will help you run validation on the inputs that were + extracted from the request. The mandatory keys will be checked first and the validations will be run after that. If + your validator already checks for keys, you may skip mentioning mandatory keys. DO NOTE THAT YOUR VALIDATOR + FUNCTIONS MUST RAISE AN EXCEPTION FOR THIS DECORATOR TO WORK. + :param mandatory_header_keys: The keys in 'inbound_headers' that are absolutely necessary. + :param mandatory_data_keys: The keys in 'inbound_data' that are absolutely necessary. + :param mandatory_file_keys: The keys in 'inbound_files' that are absolutely necessary. + :param header_validator: The function to use to validate the 'inbound_headers'. + :param data_validator: The function to use to validate 'inbound_data'. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # We first validate the mandatory header keys. + # Having a null value in this case is NOT allowed: + if mandatory_header_keys is not None: + for mandatory_key in mandatory_header_keys: + if kwargs["inbound_headers"].get(mandatory_key) is None: + return ResponseModel( + status_code = StatusCodes.HEADERS_INCOMPLETE, + message = f"missing: '{mandatory_key}'" + ) + + # Return with failure if any of the mandatory JSON details are missing. + # Having a null value is allowed, but is should be sent by the user on intention. + if mandatory_data_keys is not None: + for mandatory_key in mandatory_data_keys: + try: kwargs["inbound_data"][mandatory_key] + except: return ResponseModel( + status_code = StatusCodes.DATA_INCOMPLETE, + message = f"missing: '{mandatory_key}'" + ) + + # Return with failure if any of the mandatory file-keys details are missing: + if mandatory_file_keys is not None: + provided_file_keys = kwargs["inbound_files"].keys() + for mandatory_key in mandatory_file_keys: + if mandatory_key not in provided_file_keys: + return ResponseModel( + status_code = StatusCodes.FILE_MISSING, + message = f"missing: '{mandatory_key}'" + ) + + # Next we validate the headers: + if header_validator is not None: + + # Try validate the data: + try: kwargs["inbound_headers"] = header_validator(kwargs["inbound_headers"]) + + # In case some needed field is missing: + except KeyError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = "missing: " + str(exception), + http_code = HttpCodes.BAD_REQUEST, + ) + + # In case some pydantic data model fails validation: + except pydantic.ValidationError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = messages_from_pydantic_exception(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # In case some other exception was raised: + except Exception as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = str(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # Finally, we validate the incoming data: + if data_validator is not None: + + # Try validate the data: + try: kwargs["inbound_data"] = data_validator(kwargs["inbound_data"]) + + # In case some needed field is missing: + except KeyError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = "missing: " + str(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # In case some pydantic data model fails validation: + except pydantic.ValidationError as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = messages_from_pydantic_exception(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # In case some other exception was raised: + except Exception as exception: + return ResponseModel( + status_code = StatusCodes.DATA_VALIDATION_FAILURE, + message = str(exception), + http_code = HttpCodes.BAD_REQUEST + ) + + # Now that we have unpacked the incoming data, + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def log_request_to_mongo( + attr_name, + collection: str = "logs", + log_type: str = None, + operation: str = None, + log_input: bool = True, + log_output: bool = True, + sensitive_keys: list = None +): + + """ + USE THIS ONLY AFTER YOU HAVE USED 'read_input'. This decorator will log the whole process of the API call to + MongoDB. The variable that holds the instance of 'AsyncMongo' needs to be accessible in the scope of 'current_app'. + PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING THIS DECORATOR. + :param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the + scope of 'current_app'. + :param collection: The name of the collection to write the log into. + :param log_type: A hint to identify what the log was for. + :param operation: A hint to identify what was action was being performed. + :param log_input: Whether, or not, you would like to log the input that came in. + :param log_output: Whether, or not, you would like to log the output of the API call. + :param sensitive_keys: The list of keys to not log. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Make variables and extract available info.: + exception = None + response = None + request_ts = date_time.get_current_utc_date_time() + start_ts = time.perf_counter() + cpu_start_ts = time.process_time() + + # Execute the function that is being wrapped: + try: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + except Exception as exc: exception = exc + + # Ensure that the response is not null: + response = response if response is not None else ResponseModel( + status_code = StatusCodes.UNKNOWN_ERROR, + message = "null response for request" + ) + + # Add params to the response. + # THIS IS ONLY APPLICABLE WHEN THE TYPE OF THE RESPONSE IS 'ResponseModel': + if isinstance(response, ResponseModel): + response.api_version = kwargs.get("api_version") + response.log_id = kwargs.get("log_id") + + # Extract the response to log: + response_to_log = "not logged" + http_code_to_log = 200 + if log_output: + if isinstance(response, ResponseModel): response_to_log, http_code_to_log = response.for_quart() + elif isinstance(response, tuple): response_to_log, http_code_to_log = response + else: response_to_log, http_code_to_log = str(response), 200 + + # Try to get the information about the request: + request_method = None + request_url = None + request_route = None + try: + request_method = f"{request.method}" + request_url = f"{request.url}" + request_route = str(request.url_rule.rule) + except: pass + + # Redact the sensitive keys: + if sensitive_keys: + for k in sensitive_keys: + for var in ["inbound_headers", "inbound_data"]: + try: kwargs[var][k] = len(str(kwargs[var][k])) * "*" + except: pass + + # Construct the log: + # for k in sensitive_keys: kwargs.get("inbound_headers", {}).pop(k, None) + # for k in sensitive_keys: kwargs.get("inbound_data", {}).pop(k, None) + log_json = { + "hostname": SERVER_HOSTNAME, + "os": f"{HOST_OS}", + "cpu": f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})", + "logId": kwargs.get("log_id"), + "logChain": kwargs.get("inbound_headers", {}).get("X-Log-Chain"), + "log": log_type, + "operation": operation, + "apiVer": kwargs.get("api_version"), + "method": request_method, + "url": request_url, + "route": request_route, + "ts": request_ts, + "tat": time.perf_counter() - start_ts, + "cpuTime": time.process_time() - cpu_start_ts, + "headers": kwargs.get("inbound_headers"), + "data": kwargs.get("inbound_data") if log_input else "not logged", + "files": { + k: { + "name": v["name"], + "size": v["size"] + } for k, v in kwargs.get("inbound_files", {}).items() + }, + "exception": None if exception is None else describe_exception(exception), + "response": response_to_log, + "httpCode": http_code_to_log + } + + # Write the log: + app_attr = getattr(current_app, attr_name) + inserted_id = await app_attr.insert_one( + collection = collection, + document = log_json + ) + + # Return the response from the wrapped function. + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def should_not_be_under_maintenance(attr_name): + + """ + Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create + a boolean variable within the scope of the 'current_app' for this to work. + :param attr_name: The name of the boolean variable that will hold the information about the app being under + maintenance. If its value is True at the time of checking, the incoming request will be rejected. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Get the attribute and check if it indicates that the app is under maintenance, + # call the wrapped function if not under maintenance: + app_attr = getattr(current_app, attr_name) + if app_attr: response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() + else: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def only_whitelisted_ips(attr_name): + + """ + Use this decorator to reject any requests coming from unauthorized IPs. The list of IP addresses to allow must be + in a list that is accessible in the context of 'current_app'. + :param attr_name: The name of the boolean variable that will hold the information about the app being under + maintenance. If its value is True at the time of checking, the incoming request will be rejected. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Get the attribute and check if the request's IP is in the permitted list: + app_attr = getattr(current_app, attr_name) + if request.remote_addr not in app_attr: + return ResponseModel( + status_code = StatusCodes.AUTHORIZATION_FAILED, + message = "bad ip", + http_code = HttpCodes.UNAUTHORIZED + ).for_quart() + + # Now that we have checked that the IP is permitted, + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def limit_rate( + attr_name, + rate_limit: int = 5, + seconds: float = 1.0, + message = None, + header_keys: list = None, + data_keys: list = None, + allow_if_exception = False, + count_for_http_codes = None +): + + """ + Use this decorator to apply rate-limiting to incoming requests. + SUGGESTION: WHEN STACKING UP MANY RATE LIMITS, PUT THE SMALLEST TIME PERIOD ON TOP AND LARGEST TIME PERIOD AT THE + BOTTOM. THIS ENSURES PROPER FUNCTIONALITY. + WARNING: TIMING STARTS WHEN THE FIRST PERMITTED REQUEST GOES THROUGH. THIS MEANS THAT, IF YOU HAVE A PER-DAY LIMIT, + AND YOU START MAKING REQUESTS AT 11:00 PM AND EXHAUST YOUR LIMIT AT 11:59 PM, YOUR LIMIT WILL BE REPLENISHED AT + 11:00 PM OF THE NEXT DAY, NOT AT 12:00 AM. + :param attr_name: The name of the variable that holds the instance of 'AsyncRedisCache'. Should be available in the + context of 'current_app'. + :param rate_limit: The number of requests per unit time. + :param seconds: The time period in which the rate limit is to be applied. + :param message: The custom message to respond with. + :param header_keys: The keys in the header to consider when apply rate limits (like 'sessionToken'). + :param data_keys: The keys in the header to consider when apply rate limits (like 'sessionToken'). + :param allow_if_exception: In case Redis is unresponsive, would you prefer allowing the request to pass through or + would you prefer the request getting blocked. + :param count_for_http_codes: If this is provided, the counter will be incremented only if the response code was one + of these values. If not provided, all requests will be counted. This can be used in cases when you want to count + only when the request was successfully served. + :return: The decorator factory. + """ + + # Param-cleaning: + rate_limit = max(1, rate_limit) + if header_keys is None: header_keys = [] + if data_keys is None: data_keys = [] + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # Note down the combination of values requested and the limits prescribed: + params = {"route": str(request.url_rule), "header": {}, "data": {}, "limit": rate_limit, "seconds": seconds} + for key in header_keys: params["header"][key] = kwargs["inbound_headers"].get(key) + for key in data_keys: params["data"][key] = kwargs["inbound_data"].get(key) + + # Now make a unique key from this combination: + params_json = json.to_string(params, no_space = True) + sha256_hash = hashlib.sha256() + sha256_hash.update(params_json.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # We first get the value of the counter: + app_attr = getattr(current_app, attr_name) + counter_value = await app_attr.count(base64_key, value = 1, expiry = seconds) + + # If any exception occurred in getting the count, + # and exceptions haven't been allowed: + if counter_value is None and not allow_if_exception: + response = ResponseModel( + status_code = StatusCodes.RATE_LIMIT_EXCEEDED, + message = "Please contact admin (E)" + ) + + # If the rate-limit has already been crossed, + # or when the counter was not fetched but exceptions are allowed: + elif (counter_value or 0) > rate_limit: + response = ResponseModel( + status_code = StatusCodes.RATE_LIMIT_EXCEEDED, + message = message or ", ".join([ + f"rate limit: {rate_limit} in {humanize.naturaldelta(datetime.timedelta(seconds = seconds))}", + f"this is your {humanize.ordinal(counter_value)} request in the given period" + ]) + ) + + # If the rate-limit hasn't been crossed: + else: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # If we have been told to count only for specific status codes, + # and if the HTTP code of this response is not in the list of codes, we reduce the counter by one: + if count_for_http_codes: + http_code = 200 if not isinstance(response, (list, tuple, set)) else response[1] + if http_code not in count_for_http_codes: + await app_attr.count(base64_key, value = -1, expiry = seconds) + + # Return the response from the function call: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): + + """ + Use this decorator to handle prematurely terminated requests. If your clean-up function needs access to variables, + consider using 'g' to hold data in the scope of the request. + :param cleanup_func: The function to call when the cancelled request needs graceful handling. + :param cleanup_coro: The coroutine to call when the cancelled request needs graceful handling. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + try: + + # we are ready to run the function that we are wrapping: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + # In case the client closes the connection pre-maturely: + except asyncio.CancelledError as exception: + if hasattr(current_app, "printer"): getattr(current_app, "printer")(exception) + if cleanup_func is not None: cleanup_func() + if cleanup_coro is not None: await cleanup_coro() + return ResponseModel( + api_version = kwargs.get("api_version"), + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ).for_quart() + + # We propagate any other kind of exception: + except Exception as exception: raise + + return wrapper + + return decorator + + +# --------------------------------------------------------------------------------------------------------------------- + + +def measure_metrics_for_prometheus(): + + """ + Use this decorator to automatically measure metrics for using in Prometheus. + :return: The decorator factory. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # Let the next in-line decorator know that it has been wrapped: + kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + + # We use the context of the measurement class: + async with MetricsAPI( + method = f"{request.method}", + endpoint = str(request.url_rule.rule) + ) as metrics: + + # Invoke the wrapped function: + response = await func(*args, **kwargs) + kwargs["decorator_count"] -= 1 + + # Interpret the HTTP code: + if isinstance(response, ResponseModel): _, metrics.http_code = response.for_quart() + elif isinstance(response, tuple): metrics.http_code = response[1] + else: metrics.http_code = 200 + + # Done here: + if ( + kwargs["decorator_count"] == 1 and + isinstance(response, ResponseModel) + ): response = response.for_quart() + return response + + return wrapper + + return decorator + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/api/codes.py b/api/codes.py new file mode 100644 index 0000000..5773cc9 --- /dev/null +++ b/api/codes.py @@ -0,0 +1,164 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Sept., 2024 + + OBJECTIVE: + + To maintain all status codes in one place. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +from enum import Enum, unique + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +@unique +class HttpCodes(Enum): + + """ + Commonly used standard HTTP status codes. + Can be sent after an API Call is processed. + Refer to: https://http.dev/status + NOTE: THIS LIST IS NOT EXHAUSTIVE! + """ + + # 1XX - Informational: + CONTINUE = 100 # .............. https://http.dev/102 + SWITCHING_PROTOCOLS = 101 # ... https://http.dev/101 + PROCESSING = 102 # ............ https://http.dev/102 + EARLY_HINTS = 103 # ........... https://http.dev/103 + + # 2XX - Success: + SUCCESS = 200 # .................. https://http.dev/200 + CREATED = 201 # .................. https://http.dev/201 + ACCEPTED = 202 # ................. https://http.dev/202 + NON_AUTHORITATIVE_INFO = 203 # ... https://http.dev/203 + NO_CONTENT = 204 # ............... https://http.dev/204 + RESET_CONTENT = 205 # ............ https://http.dev/205 + PARTIAL_CONTENT = 206 # .......... https://http.dev/206 + MULTI_STATUS = 207 # ............. https://http.dev/207 + ALREADY_REPORTED = 208 # ......... https://http.dev/208 + THIS_IS_FINE = 218 # ............. https://http.dev/218 + IM_USED = 226 # .................. https://http.dev/226 + + # 3XX - Redirection: + MULTIPLE_CHOICES = 300 # ..... https://http.dev/300 + MOVED_PERMANENTLY = 301 # .... https://http.dev/301 + MOVED_TEMPORARILY = 302 # .... https://http.dev/302 + SEE_OTHER = 303 # ............ https://http.dev/303 + NOT_MODIFIED = 304 # ......... https://http.dev/304 + USE_PROXY = 305 # ............ https://http.dev/305 + SWITCH_PROXY = 306 # ......... https://http.dev/306 + TEMPORARY_REDIRECT = 307 # ... https://http.dev/307 + PERMANENT_REDIRECT = 308 # ... https://http.dev/308 + + # 4XX - Client Errors: + BAD_REQUEST = 400 # ..................... https://http.dev/401 + UNAUTHORIZED = 401 # .................... https://http.dev/401 + PAYMENT_REQUIRED = 402 # ................ https://http.dev/402 + FORBIDDEN = 403 # ....................... https://http.dev/403 + NOT_FOUND = 404 # ....................... https://http.dev/404 + METHOD_NOT_ALLOWED = 405 # .............. https://http.dev/405 + NOT_ACCEPTABLE = 406 # .................. https://http.dev/406 + PROXY_AUTH_REQUIRED = 407 # ............. https://http.dev/407 + REQUEST_TIMEOUT = 408 # ................. https://http.dev/408 + CONFLICT = 409 # ........................ https://http.dev/409 + GONE = 410 # ............................ https://http.dev/410 + LENGTH_REQUIRED = 411 # ................. https://http.dev/411 + PRECONDITION_FAILED = 412 # ............. https://http.dev/412 + PAYLOAD_TOO_LARGE = 413 # ............... https://http.dev/413 + URI_TOO_LONG = 414 # .................... https://http.dev/414 + UNSUPPORTED_MEDIA_TYPE = 415 # .......... https://http.dev/415 + PAGE_EXPIRED = 419 # .................... https://http.dev/419 + TOO_MANY_REQUESTS = 429 # ............... https://http.dev/429 + UNAVAILABLE_FOR_LEGAL_REASONS = 451 # ... https://http.dev/451 + INVALID_TOKEN = 498 # ................... https://http.dev/498 + CLIENT_CLOSED_REQUEST = 499 # ........... https://http.dev/499 + + # 5XX - Server Errors: + INTERNAL_SERVER_ERROR = 500 # ........... https://http.dev/500 + NOT_IMPLEMENTED = 501 # ................. https://http.dev/501 + BAD_GATEWAY = 502 # ..................... https://http.dev/502 + SERVICE_UNAVAILABLE = 503 # ............. https://http.dev/503 + GATEWAY_TIMEOUT = 504 # ................. https://http.dev/504 + HTTP_VERSION_NOT_SUPPORTED = 505 # ...... https://http.dev/505 + VARIANT_ALSO_NEGOTIATES = 506 # ......... https://http.dev/506 + INSUFFICIENT_STORAGE = 507 # ............ https://http.dev/507 + LOOP_DETECTED = 508 # ................... https://http.dev/508 + BANDWIDTH_LIMIT_EXCEEDED = 509 # ........ https://http.dev/509 + WEB_SERVER_DOWN = 521 # ................. https://http.dev/521 + ORIGIN_IS_UNREACHABLE = 523 # ........... https://http.dev/523 + SERVICE_IS_OVERLOADED = 529 # ........... https://http.dev/529 + NETWORK_READ_TIMEOUT_ERROR = 598 # ...... https://http.dev/598 + NETWORK_CONNECT_TIMEOUT_ERROR = 599 # ... https://http.dev/599 + + +# --------------------------------------------------------------------------------------------------------------------- + + +@unique +class StatusCodes(Enum): + + """ + To be used internally within the context of your service. Customize these to match your service. + The format is: (SUCCESS_INDICATOR, INTERNAL_NUMERIC_CODE, HTTP_CODE) + Example: (True, 1, 200) + """ + + # Legacy Codes: + OK = (True, 1, HttpCodes.SUCCESS.value) + FAILED = (False, 0, HttpCodes.INTERNAL_SERVER_ERROR.value) + PARTIAL_SUCCESS = (True, 2, HttpCodes.PARTIAL_CONTENT.value) + PARTIAL_FAILURE = (False, 3, HttpCodes.PARTIAL_CONTENT.value) + + # Authentication Codes: + LOGGED_IN_SUCCESSFULLY = (True, 200, HttpCodes.SUCCESS.value) + LOGIN_FAILED = (False, 201, HttpCodes.UNAUTHORIZED.value) + INVALID_SESSION_TOKEN = (False, 202, HttpCodes.UNAUTHORIZED.value) + AUTHENTICATION_DETAILS_INCOMPLETE = (False, 203, HttpCodes.BAD_REQUEST.value) + + # Authorization Codes: + AUTHORIZED_SUCCESSFULLY = (True, 300, HttpCodes.SUCCESS.value) + NOT_ALLOWED = (False, 301, HttpCodes.FORBIDDEN.value) + AUTHORIZATION_DETAILS_INCOMPLETE = (False, 302, HttpCodes.BAD_REQUEST.value) + AUTHORIZATION_FAILED = (False, 303, HttpCodes.UNAUTHORIZED.value) + + # General failures: + DOWN_FOR_MAINTENANCE = (False, 800, HttpCodes.SERVICE_UNAVAILABLE.value) + UNKNOWN_ERROR = (False, 801, HttpCodes.INTERNAL_SERVER_ERROR.value) + DATA_INCOMPLETE = (False, 802, HttpCodes.BAD_REQUEST.value) + HEADERS_INCOMPLETE = (False, 803, HttpCodes.BAD_REQUEST.value) + FILES_MISSING = (False, 804, HttpCodes.BAD_REQUEST.value) + CLIENT_CLOSED_REQUEST = (False, 805, HttpCodes.CLIENT_CLOSED_REQUEST.value) + + # Validation failure: + DATA_VALIDATION_FAILURE = (False, 900, HttpCodes.BAD_REQUEST.value) + RATE_LIMIT_EXCEEDED = (False, 901, HttpCodes.TOO_MANY_REQUESTS.value) diff --git a/api/metrics_prometheus.py b/api/metrics_prometheus.py new file mode 100644 index 0000000..f7e9e3e --- /dev/null +++ b/api/metrics_prometheus.py @@ -0,0 +1,198 @@ +""" + + AUTHOR: + + Khushal P Soonderji + Sharvil J Daiya + + DATE: + + Saturday, 28th Sept., 2024 + + OBJECTIVE: + + To have one place from where several metrics are measured using easy to use context managers. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To measure time: +import time + +# To capture metrics for Prometheus: +from prometheus_client import Counter, Summary, Gauge + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Define the Prometheus metrics. +# FOR THE MICROSERVICE AS A WHOLE: +WORKER_COUNT = Counter( + name = "ms_workers_active_total", + documentation = "The number of threads for the microservice being monitored.", + labelnames = ["project_name", "service_name", "host_name"] +) + +# Define the Prometheus metrics. +# FOR INDIVIDUAL API ENDPOINTS: +REQUEST_LATENCY = Summary( + name = "http_request_latency_seconds", + documentation = "Latency of HTTP requests in seconds.", + labelnames = ["method", "endpoint", "http_status"] +) +TOTAL_REQUEST_COUNT = Counter( + name = "http_requests_total", + documentation = "Total HTTP requests.", + labelnames = ["method", "endpoint", "http_status"] +) +LIVE_REQUEST_COUNT = Gauge( + name = "http_requests_live_total", + documentation = "To check if an API endpoint is being served right now.", + labelnames = ["endpoint"] +) + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class MetricsAPI: + + """ + Use this class through its context manager to automatically measure all the metrics in one place. + This was originally created to measure the performance of API endpoints made in Quart, but it should work with + other frameworks as well. + """ + + def __init__( + self, + method = None, + endpoint = None, + raise_exception = False + ): + + # Make provisions for things to note. + # NOTE: THESE MUST BE SET FROM OUTSIDE: + self.method = method + self.endpoint = endpoint + self.http_code = None + self.__raise_exception = raise_exception + + async def __aenter__(self): + + # Note down the start time immediately: + self.__start_ts = time.perf_counter() + self.__cpu_start_ts = time.process_time() + + # Note down the metrics: + LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1) + + # Setup done: + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + + # Note down the metrics: + LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1) + + TOTAL_REQUEST_COUNT.labels( + self.method, + self.endpoint, + self.http_code + ).inc() + + REQUEST_LATENCY.labels( + self.method, + self.endpoint, + self.http_code + ).observe(time.perf_counter() - self.__start_ts) + + # Handle the exception as per the user's preference: + return False if self.__raise_exception else True + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + import random + from prometheus_client import generate_latest + + async def simulate_endpoint(): + + async with MetricsAPI( + method = random.choice(["GET", "POST"]), + endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}" + ) as metrics: + + # Simulate some action on some endpoint: + await asyncio.sleep(1.0) + + # Note down the values: + metrics.http_code = 200 + + async def main(): + + print("Simulating endpoints...") + tasks = [simulate_endpoint() for _ in range(250)] + await asyncio.gather(*tasks) + print("Done!") + + print("METRICS:") + print(generate_latest().decode()) + + asyncio.run(main()) diff --git a/api/response.py b/api/response.py new file mode 100644 index 0000000..3c51696 --- /dev/null +++ b/api/response.py @@ -0,0 +1,121 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Sept., 2024 + + OBJECTIVE: + + To have a structure to the response sent from the API calls. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +from pydantic import BaseModel +from typing import Any, Optional, List + +from utils_v2.api.codes import StatusCodes, HttpCodes + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class ResponseModel(BaseModel): + + status_code: StatusCodes + message: Optional[str | List] = None + data: Optional[Any] = None + seconds: Optional[float | int] = None + log_id: Optional[str] = None + http_code: Optional[HttpCodes] = None + api_version: Optional[str] = None + + def for_quart(self): + + # Construct the basic structure: + response_dict = { + "status": 1 if self.status_code.value[0] else 0, + "code": self.status_code.value[1], + "message": self.message or self.status_code.name.replace("_", " ").lower(), + "data": self.data, + "apiVer": self.api_version + } + + # Now add the additional fields: + if self.seconds is not None: response_dict["seconds"] = self.seconds + if self.log_id is not None: response_dict["logId"] = self.log_id + + # Figure out the HTTP code: + response_http_code = self.http_code.value if self.http_code is not None else self.status_code.value[2] + + # Done here: + return response_dict, response_http_code + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + my_response = ResponseModel( + status_code = StatusCodes.RATE_LIMIT_EXCEEDED + ) + my_response.log_id = "abc123" + + print(my_response.for_quart()) diff --git a/cache/__init__.py b/cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cache/async_redis_cache.py b/cache/async_redis_cache.py new file mode 100644 index 0000000..14df725 --- /dev/null +++ b/cache/async_redis_cache.py @@ -0,0 +1,451 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 11th April, 2024 + + OBJECTIVE: + + To provide an easy way to work with '.json' files in an async manner. + + REFERENCES: + + 1) https://www.w3schools.com/python/python_json.asp + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To use redis: +import redis.asyncio as redis + +# Other utils: +from utils_v2.string import json + +# For debugging: +from icecream import IceCreamDebugger + +# To make a decorator: +from functools import wraps + +# For hashing and shortening the hash: +import hashlib +import base64 + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** WRAPPERS *** +# ***** **** +# ***************************************************************************************************************** + + +def cache_it(cache = None, expiry = 120): + + """ + This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. + If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the + cache instead of going through the whole function again. + :param cache: The instance of 'AsyncRedisCache'. + :param expiry: The time in seconds after which the cached data must be cleared. + :return: The decorator that automatically caches your data. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(*args, **kwargs): + + # We first use the name of the function and the inputs given to it to generate a key for Redis with the + # simple hashing and shortening by way of base64 strings: + inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) + sha256_hash = hashlib.sha256() + sha256_hash.update(inputs_given.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Now we check if we have the value in cache: + try: response = await cache.get(base64_key, raise_exception = True) + + # If the key doesn't exist, we pass through the function and store the results. + except: + response = await func(*args, **kwargs) + await cache.set(key = base64_key, value = response, expiry = expiry) + + # Return the response from the wrapped function. + return response + + return wrapper + + return decorator + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncRedisCache: + + # Handle datatypes when setting values to cache: + __set_converters = { + "set": lambda x: list(x), + "tuple": lambda x: list(x), + "complex": lambda x: {"r": x.real, "i": x.imag} + } + + # Handle datatypes when getting from cache: + __get_converters = { + "set": lambda x: set(x), + "tuple": lambda x: tuple(x), + "complex": lambda x: complex(x["r"], x["i"]) + } + + def __init__( + self, + connection_string, + ping_counter = 1_000, + debug = False, + debug_prefix = "R-Cache | " + ): + + """ + Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools, + strings, dicts, lists, sets, and tuples :) + :param connection_string: The connection URL for connecting to Redis. + :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection. + :param debug: Whether, or not, you want to show debugging messages from the start. + :param debug_prefix: The prefix text to show with the debugging messages. + """ + + # Note down the configuration: + self.__client = None + self.__ping_counter = ping_counter + self.__requests_since_last_ping = 0 + self.__connection_string = connection_string + + # For debugging: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + @staticmethod + def make_key(*args, **kwargs): + + """ + Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the + inputs stay the same, the output stays the same. + :param args: Any number of args that you would like to use to generate the key. + :param kwargs: Any number of kwargs that you would like to use to generate the key. + :return: A string that can be used as a key to store values on Redis. + """ + + # Take everything into one plain text string: + plain_text = "".join(str(a) for a in args) + plain_text += json.to_string(kwargs, no_space = True) + + # Hash the plain text value, and create a key from it: + sha256_hash = hashlib.sha256() + sha256_hash.update(plain_text.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Done here: + return base64_key + + def add_converter( + self, + type_name, + set_converter_func, + get_converter_func, + ): + + """ + To add custom data converters to handle new values. + RULE: Each of the converter functions must take in exactly on argument. + :param type_name: The name of the type of the object to handle. HINT: type(obj).__name__. + :param set_converter_func: A converter function that converts from the custom datatype to a datatype that Redis + can work with. Try converting to an bool, int, float, str, list or dict. + :param get_converter_func: The inverse of the set converter. This function will be used to convert from a + datatype that Redis can work with to the custom datatype. + :return: None. + """ + + self.__set_converters[type_name] = lambda x: set_converter_func(x) + self.__get_converters[type_name] = lambda x: get_converter_func(x) + + async def connect(self): + + """ + To make an asynchronous connection request to the Redis server to establish a connection. + :return: True if connected, else False. + """ + + try: + self.__client = redis.from_url( + self.__connection_string + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def disconnect(self): + + """ + Close the connection to the Redis server. + :return: True if successful, else False. + """ + + if self.__client is not None: + try: await self.__client.close() + except Exception as exception: + self.__printer(exception) + return False + return True + return True + + async def ensure_connection(self): + + """ + To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected. + :return: True if connected, else False. + """ + + # If we are not connected, we try to establish a connection: + if self.__client is None: return await self.connect() + + # Else we check if we are connected. If not, we try to connect. + # But we check only once in a while. In the meantime, we assume that we are connected. + elif self.__requests_since_last_ping > self.__ping_counter: + try: + await self.__client.ping() + self.__requests_since_last_ping = 0 + return True + except Exception as exception: + self.__printer(exception) + return await self.connect() + else: + self.__requests_since_last_ping += 1 + return True + + async def set(self, key, value, expiry: float = 120, raise_exception = False): + + """ + Saves some value to the cache. + :param key: The key with which the data will be stored and retrieved. + :param value: The value to store. + :param expiry: The time in seconds after which the data will expire. Must be a positive number. + :param raise_exception: If you want to raise an exception if the process fails. + :return: True if cached, else False. + """ + + await self.ensure_connection() + + try: + + # We handle conversion for unsupported Pythonic datatypes: + original_type = type(value).__name__ + converter = self.__set_converters.get(original_type) + if converter is not None: value = converter(value) + + # Here we actually try to store the data: + response = await self.__client.setex( + key, + int(expiry), + json.to_string({ + "value": value, + "type": original_type + }, indent = 0, no_space = False) + ) + return response + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return False + + async def get(self, key, raise_exception = False, on_fail = None): + + """ + Retrieve the cached value. + :param key: The key with which the data was saved. + :param raise_exception: If you want to raise an exception if the process fails. + :param on_fail: What to return if the process fails due to an exception. + :return: The retrieved data or null if not found. + """ + + await self.ensure_connection() + + try: + data = await self.__client.get(key) + data = data.decode("utf-8") + data = json.from_string(data) + converter = self.__get_converters.get(data["type"]) + if converter is not None: return converter(data["value"]) + return data["value"] + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return on_fail + + async def delete(self, key, raise_exception = False): + + """ + Prematurely delete the value from the cache before it expires. + :param key: The key with which the data was saved. + :param raise_exception: If you want to raise an exception if the process fails. + :return: True if deleted, else False. + """ + + await self.ensure_connection() + + try: + response = await self.__client.delete(key) + return True if response else False + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return False + + async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False): + + """ + To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the + counter will be created and the value will be incremented from 0. + :param key: The name of the counter. + :param value: The amount to increment the value by. Send negative values to count backwards. + :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created. + This value has to be an integer. If a float is passed, the value will be rounded off. + :param raise_exception: If you want to raise an exception if the process fails. + :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed. + """ + + await self.ensure_connection() + + try: + + already_existed = await self.__client.exists(key) + new_value = await self.__client.incrby(key, value) + if expiry and not already_existed: await self.__client.expire(key, int(expiry)) + return new_value + + except Exception as exception: + self.__printer(exception) + if raise_exception: raise + else: return None + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + def set_complex(x): + return {"r": x.real, "i": x.imag} + + def get_complex(x): + return complex(x["r"], x["i"]) + + async def main(): + + my_cache = AsyncRedisCache( + connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0", + ping_counter = 100, + debug = False + ) + + my_cache.add_converter( + type_name = type(2j).__name__, + set_converter_func = set_complex, + get_converter_func = get_complex + ) + + value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624") + print("GET:", value) + print("TYP:", type(value), end = "\n\n") + + # success = await my_cache.set( + # key = "name", + # value = {"first": "John", "last": "Doe"}, + # expiry = 10 + # ) + # print("SET:", success, end = "\n\n") + # + # value = await my_cache.get(key = "name") + # print("GET:", value) + # print("TYP:", type(value), end = "\n\n") + # + # success = await my_cache.delete(key = "name") + # print("DEL:", success, end = "\n\n") + # + # value = await my_cache.get(key = "cnt") + # print("GET:", value) + # print("TYP:", type(value), end = "\n\n") + # + # await my_cache.delete(key = "cnt") + # for _ in range(50): + # await asyncio.sleep(1.0) + # counter = await my_cache.count(key = "cnt", value = 1, expiry = 10) + # print("COUNTER:", counter) + + asyncio.run(main()) diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database/__pycache__/__init__.cpython-310.pyc b/database/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..9b36a88 Binary files /dev/null and b/database/__pycache__/__init__.cpython-310.pyc differ diff --git a/database/__pycache__/async_mongo_v2.cpython-310.pyc b/database/__pycache__/async_mongo_v2.cpython-310.pyc new file mode 100644 index 0000000..fccfeaa Binary files /dev/null and b/database/__pycache__/async_mongo_v2.cpython-310.pyc differ diff --git a/database/async_firebase.py b/database/async_firebase.py new file mode 100644 index 0000000..a8065a1 --- /dev/null +++ b/database/async_firebase.py @@ -0,0 +1,504 @@ +""" + + AUTHOR: + + Khushal P Soonderji + Bhushan Thakkar + + DATE: + + Friday, 19th April, 2024 + + OBJECTIVE: + + To provide an easy interface to work with Firebase. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +# --- +import sys +sys.path.append(".") +sys.path.append("..") + +# For async operations: +# --- +import asyncio + +# For system-level activities: +# --- +import os + +# My async utils: +# --- +import async_json_utils + +# Firebase: +# --- +import firebase_admin +import firebase_admin.firestore_async +import firebase_admin.auth +import firebase_admin.db + +# For debugging and logging: +# --- +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncFirebase: + + def __init__( + self, + credentials_json_path, + app_name, + max_connections = 5, + debug = True + ): + + # Debugging print: + # --- + self.__printer = IceCreamDebugger(prefix = f"FBase ({app_name[:8]}) | ", includeContext = True) + if not debug: self.__printer.disable() + alert = f"Firebase session starting with max. {max_connections} connections." + self.__printer(alert) + + # Initialize the instance: + # --- + self.__semaphore = asyncio.Semaphore(max_connections) + credentials = firebase_admin.credentials.Certificate(credentials_json_path) + self.__firebase_app = firebase_admin.initialize_app(credentials, name = app_name) + self.__firestore = firebase_admin.firestore_async.client(self.__firebase_app) + + def __del__(self): + firebase_admin.delete_app(self.__firebase_app) + self.__firestore.close() + alert = "Firebase session ended." + self.__printer(alert) + + def __user_to_json(self, firebase_user): + + user_json = { + "uid": firebase_user.uid, + "email": firebase_user.email, + "emailVerified": firebase_user.email_verified, + "displayName": firebase_user.display_name, + "phoneNo": firebase_user.phone_number, + "photoUrl": firebase_user.photo_url, + "customClaims": firebase_user.custom_claims, + "disabled": firebase_user.disabled, + "providerId": firebase_user.provider_id, + "providerData": firebase_user.provider_data, + "tenantId": firebase_user.tenant_id + } + + return user_json + + async def create_custom_token(self, uid): + + async with self.__semaphore: + + try: + custom_token = firebase_admin.auth.create_custom_token( + uid = uid, + app = self.__firebase_app + ).decode("utf-8") + return custom_token + + except Exception as exception: + self.__printer(exception) + return None + + async def authenticate_token(self, token): + + """ + To authenticate the given session token. + :param token: The session token generated by Firebase on a successful sign-in. + :return: Either the retrieved user information or a blank dictionary. + """ + + try: + user_info = firebase_admin.auth.verify_id_token(token, app = self.__firebase_app) + return user_info + except Exception as exception: + self.__printer(exception) + return {} + + async def create_user( + self, + uid, + display_name = None, + password = None, + email = None, + phone_number = None + ): + + """ + Create a new user. + :param uid: The id to identify the user by. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + firebase_admin.auth.create_user( + uid = uid, + app = self.__firebase_app + ) + return True + + except firebase_admin.auth.UidAlreadyExistsError as excp: + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def delete_user(self, uid): + + async with self.__semaphore: + + try: + firebase_admin.auth.delete_user( + uid = uid, + app = self.__firebase_app + ) + return True + + except firebase_admin.auth.UserNotFoundError as excp: + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def get_user(self, uid): + + async with self.__semaphore: + + try: + user = firebase_admin.auth.get_user( + uid = uid, + app = self.__firebase_app + ) + return self.__user_to_json(user) + + except Exception as exception: + self.__printer(exception) + return None + + async def list_users(self, users_per_page, page_token = None): + + """ + To get a list of users. Firebase has a limit of 1000 per call of this API. So we use page-tokens to fetch next + pages of users. + :param users_per_page: How many users you want to list in this API call (Max. 1,000). + :param page_token: To be used in case of pagination. + :return: The list of users and the page token to be used for the next call. Will be None in case of failure. + """ + + async with self.__semaphore: + + try: + users = firebase_admin.auth.list_users( + page_token = page_token, + max_results = min(users_per_page, 1000), + app = self.__firebase_app + ) + next_page_token = users.next_page_token if users.has_next_page else None + users = users.users + users_json = [self.__user_to_json(user) for user in users] + return users_json, next_page_token + + except Exception as exception: + self.__printer(exception) + return None, None + + async def create_document(self, collection_path, document_name, document_data = None): + + """ + To create a new document with the specified data in an existing collection. + :param collection_path: The path of the collection (can be a sub-collection). + :param document_name: The name of the document you want to create. + :param document_data: The data that you want to populate in the document. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.collection( + collection_path + ).add( + document_id = document_name, + document_data = document_data or {} + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def delete_document(self, path): + + """ + To delete a document. + :param path: The path of the document. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).delete() + return True + + except Exception as excp: + print("FIRESTORE DOCUMENT DELETION EXCEPTION:", excp) + return False + + async def set_document(self, path, data): + + """ + To overwrite the data in a document. + :param path: The path of the document. + :param data: The data you want to update as a dictionary. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).set(data) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def update_document(self, path, data): + + """ + To update the data in a document. Provide only the fields that you want to update. + :param path: The path of the document. + :param data: The data you want to update as a dictionary. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).update(data) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def get_document(self, path): + + """ + To get the data in a document. + :param path: The path of the document. + :return: The dictionary of data as found in the path specified or None if the operation failed. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document(path).get() + return snapshot.to_dict() + + except Exception as exception: + self.__printer(exception) + return None + + async def get_document_fields(self, path, fields): + + """ + To get only specific fields (keys) in a document. Like how projections are used in Mongo. + :param path: The path of the document. + :param fields: The list of fields (keys) of the document that you want. + :return: The dictionary of data as found in the path specified. + """ + + async with self.__semaphore: + + try: + if type(fields) is not list: fields = [fields] + snapshot = await self.__firestore.document(path).get(fields) + return snapshot.to_dict() + + except Exception as exception: + self.__printer(exception) + return None + + async def create_collection(self, collection_name, document_name, document_data): + + """ + Creates a new collection in the root of the database. Note that Firebase doesn't allow creating new empty + collections, so we must add one first document in it. + :param collection_name: The name of the collection you want to create. + :param document_name: The name of the first document you want to put in the collection. + :param document_data: The data that you want to put in the first document of the new collection. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.collection( + collection_name + ).add( + document_id = document_name, + document_data = document_data + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def create_sub_collection( + self, + document_path, + sub_collection_name, + sub_document_name = None, + sub_document_data = None + ): + + """ + Firebase allows you to create collections inside documents. This method is built for that. Note that Firebase + doesn't allow creating new empty collections, so we must add one first document in it. + :param document_path: The path of the document in which you want to create a new collection. + :param sub_collection_name: The name of the collection you want to create. + :param sub_document_name: The name of the first document you want to put in the collection. + :param sub_document_data: The data that you want to put in the first document of the new collection. + :return: True or False based on the success of the operation. + """ + + async with self.__semaphore: + + try: + snapshot = await self.__firestore.document( + document_path + ).collection( + sub_collection_name + ).add( + document_id = sub_document_name, + document_data = sub_document_data or {} + ) + return True + + except Exception as exception: + self.__printer(exception) + return False + + async def get_collection(self, path): + + """ + Get a whole collection's data. + :param path: The path of the collection. + :return: The collection. + """ + + async with self.__semaphore: + + try: + snapshots = await self.__firestore.collection(path).get() + return {snapshot.id: snapshot.to_dict() for snapshot in snapshots} + + except Exception as exception: + self.__printer(exception) + return None + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import time + from shared import constants + + user_data = { + "firebaseUid": None, + "roles": "ClientAdmin", + "sessionToken": "siddhesh_20240419" + } + + async def test(): + my_firebase = AsyncFirebase( + f"{constants.PROJECT_DIRECTORY}/utils/cred/firebase_certs.json", + "myFire", + max_connections = 5 + ) + print("\n\n---\n\n") + results = await my_firebase.get_document("testCollection/testDoc") + print(async_json_utils.to_json_string(results)) + print("\n\n---\n\n") + results = await my_firebase.get_document_fields("testCollection/testDoc", ["sampleMap"]) + print(async_json_utils.to_json_string(results)) + # await my_firebase.create_sub_collection( + # "myDeepCollection/deepDocId", + # "subCollection3", + # "subDoc", + # {"sub_key": "sub_val"} + # ) + # await my_firebase.create_collection( + # "rootCollection", + # "subDoc", + # {"sub_key": "sub_val"} + # ) + # await my_firebase.delete_document("activeSessions/9d402f1502dfd55a4326fa7fc8e6cb7d") + # print(async_json_utils.to_json_string(await my_firebase.get_collection("myDeepCollection"))) + + start_time = time.time() + asyncio.run(test()) + print(f"FINISHED IN {time.time() - start_time} SECONDS.") diff --git a/database/async_mongo.py b/database/async_mongo.py new file mode 100644 index 0000000..c052fac --- /dev/null +++ b/database/async_mongo.py @@ -0,0 +1,843 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 28th May, 2024 + + OBJECTIVE: + + To have one central place from where all async database connectivity happens. + + REFERENCES: + + 01. https://motor.readthedocs.io/en/stable/ + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For async behaviour: +import asyncio + +# For datetime handling: +import pytz +import datetime + +# MongoDB: +from motor.motor_asyncio import AsyncIOMotorClient +from bson.objectid import ObjectId +from bson.json_util import dumps, loads + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class MongoFindException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo find operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoInsertException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo insert operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoUpdateException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo update operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoReplaceException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo replace operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoDeleteException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo delete operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MongoException(Exception): + + def __init__(self, hint = None, origin = None): + self.__hint = hint + self.__origin = origin + + def __str__(self): + message = "mongo operation failed" + if self.__hint is not None: message = f"{message} ({self.__hint})" + if self.__origin is not None: message = f"{self.__origin} --> {message}" + return message + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongo: + + __db = None + __db_name = None + __client = None + + def __init__( + self, + database_name = "myDb", + max_connections = 5, + debug = True, + debug_only_errors = True, + host = "localhost", + port = 27017, + connection_string = None + ): + + # Basic config: + self.__max_connections = max_connections + self.__db_name = database_name + self.__host = host, + self.__port = port + self.__connection_string = connection_string + + # For debugging: + self.__debug_only_errors = debug_only_errors + self.__printer = IceCreamDebugger(prefix = f"Mongo ({self.__db_name}) | ", includeContext = True) + if not debug: self.__printer.disable() + + # Rate/access control: + self.__exclusive_semaphore = asyncio.Semaphore(1) + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + @staticmethod + def generate_id(): + + """ + Just generates an '_id' in MongoDB style. + :return: The '_id' in MongoDB style. + """ + + return str(ObjectId()) + + async def connect(self): + try: + if self.__connection_string is None: + self.__client = AsyncIOMotorClient( + self.__host, + self.__port, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + else: + self.__client = AsyncIOMotorClient( + self.__connection_string, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + self.__db = self.__client.get_database(self.__db_name) + if not self.__debug_only_errors: + server_info = await self.__client.server_info() + self.__printer(server_info) + except Exception as exception: self.__printer(exception) + + async def ensure_connection(self): + if self.__db is None: + async with self.__exclusive_semaphore: + await self.connect() + + @property + async def client(self): + await self.ensure_connection() + return self.__client + + @staticmethod + def dict_to_dot_notation(input_dict, parent_key = "", separator = "."): + items = [] + for k, v in input_dict.items(): + new_key = f"{parent_key}{separator}{k}" if parent_key else k + if isinstance(v, dict) and v: + items.extend(AsyncMongo.dict_to_dot_notation(v, new_key, separator = separator).items()) + else: + items.append((new_key, v)) + return dict(items) + + @staticmethod + def normalize_date_time(document): + + """ + MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it. + This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was + timezone naive, UTC timezone will be applied to it without changing the time value. + :param document: The document that you want to normalize the date-time in. + :return: The document with normalized datetime. + """ + + if isinstance(document, datetime.datetime): + utc_tz = pytz.timezone("UTC") + if document.tzinfo is None: document = utc_tz.localize(document) + else: document = document.astimezone(utc_tz) + + if type(document) is list: + document = [AsyncMongo.normalize_date_time(item) for item in document] + + if type(document) is dict: + document = { + AsyncMongo.normalize_date_time(k): AsyncMongo.normalize_date_time(v) + for k, v in document.items() + } + + return document + + @staticmethod + def __from_json_string(json_data): + return loads(json_data) + + @staticmethod + def __to_json_string(python_data, indent = 4, default = None): + return dumps(python_data, indent = indent, default = default) + + async def list_indexes( + self, + collection, + session = None, + raise_exception = False + ): + + """ + Lists out the indexes of a collection. + :param collection: The collection whose indexes you want to list out, + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The list of indexes or None if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + indexes = None + + # Try to list the indexes: + try: + responses = await self.__db[collection].list_indexes(session = session).to_list(None) + indexes = [{key: value for key, value in response.items()} for response in responses] + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not indexes: raise MongoException(hint = "list indexes") + return indexes + + async def create_index( + self, + collection, + keys, + options = None, + session = None, + raise_exception = False + ): + + """ + Creates an index on a collection. + :param collection: The collection to create the index on. + :param keys: The keys (and sorting) to implement the index on. + :param options: Additional config. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the execution. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the insertion: + try: + options = options or {} + keys = [(k, v) for k, v in keys.items()] + response = await self.__db[collection].create_index(keys, session = session, **options) + if response: success = True + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not success: raise MongoException(hint = "create index") + return success + + async def insert_one( + self, + collection, + document, + session = None, + raise_exception = False + ): + + """ + Insert data into a collection. + :param collection: The collection you want to feed the data into. + :param document: The data to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_id = None + + # Try to make the insertion: + try: + response = await self.__db[collection].insert_one(document.copy(), session = session) + inserted_id = response.inserted_id + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not inserted_id: raise MongoInsertException(hint = f"{collection}") + return inserted_id + + async def insert_many( + self, + collection, + documents, + session = None, + raise_exception = False + ): + + """ + Insert a lot of data into a collection. + :param collection: The collection you want to feed the data into. + :param documents: The data to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_ids = [] + + # Try to make the insertion: + try: + response = await self.__db[collection].insert_many(documents, session = session) + inserted_ids = response.inserted_ids + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not inserted_ids: raise MongoInsertException(hint = f"{collection}") + return inserted_ids + + async def update_one( + self, + collection, + filter_json = None, + update_json = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update one document. + :param collection: The collection you want to update. + :param filter_json: The selection criteria to locate the document to update. + :param update_json: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the insertion: + try: + response = await self.__db[collection].update_one( + filter_json, + update_json, + upsert = upsert, + session = session + ) + success = False if response.modified_count == 0 else True + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not success: raise MongoUpdateException(hint = f"{collection}") + return success + + async def update_many( + self, + collection, + filter_json = None, + update_json = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update many documents. + :param collection: The collection you want to update. + :param filter_json: The selection criteria to locate the document to update. + :param update_json: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + update_count = 0 + + # Try to make the insertion: + try: + response = await self.__db[collection].update_many( + filter_json, + update_json, + upsert = upsert, + session = session + ) + update_count = response.modified_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not update_count: raise MongoUpdateException(hint = f"{collection}") + return update_count + + async def count( + self, + collection, + filter_json = None + ): + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + if filter_json is None: filter_json = {} + count = await self.__db[collection].count_documents(filter_json) + except Exception as exception: self.__printer(exception) + + # Done here: + return count + + async def bulk_write( + self, + collection, + requests, + session = None, + raise_exception = False + ): + + """ + To perform various individual operations in one go. You will have to import individual actions like "UpdateOne" + and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method. + :param collection: The collection you want to run the requests on. + :param requests: The array of requests (operations) to be performed. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + response = await self.__db[collection].bulk_write(requests, session = session) + count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not count: raise MongoException(hint = "bulk write") + return count + + async def find_many( + self, + collection, + filter_json, + projections = None, + skip = 0, + limit = None, + sort = None, + session = None, + as_json_string = False, + indent = 4, + raise_exception = False + ): + + """ + Finds one or more records that match the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter_json: The filter criteria. + :param projections: What parts of the matching data you want to fetch. + :param skip: The no. of records to skip before picking next ones. Needed for pagination. + :param limit: The max. no. of records you want to fetch. + :param sort: The sorting rules to apply. + :param session: The session if you need to do this in a transaction. + :param as_json_string: Whether you want it as a JSON string or a Python dict/list. + :param indent: The indentation to use if you want it as a JSON string. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # Assume defaults: + if sort is None: sort = {"_id": -1} + if limit is None: limit = 10 + + # try to query the data: + try: + results = await self.__db[collection].find( + filter_json, + projections, + session = session + ).sort(sort).skip(skip).limit(limit).to_list(None) + if as_json_string: results = self.__to_json_string(results, indent = indent, default = str) + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not results: raise MongoFindException(hint = f"{collection}") + return results + + async def find_one( + self, + collection, + filter_json, + projections = None, + session = None, + as_json_string = False, + indent = 4, + raise_exception = False + ): + + """ + Finds one record that matches the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter_json: The filter criteria. + :param projections: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param as_json_string: Whether you want it as a JSON string or a Python dict/list. + :param indent: The indentation to use if you want it as a JSON string. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to query the data: + try: + results = await self.__db[collection].find_one(filter_json, projections, session = session) + if as_json_string: results = self.__to_json_string(results, indent = indent, default = str) + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not results: raise MongoFindException(hint = f"{collection}") + return results + + async def replace_one( + self, + collection, + filter_json, + replacement_json, + upsert = False, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter_json: The filter criteria. + :param replacement_json: The data to put in place of the existing document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # try to query the data: + try: + result = await self.__db[collection].replace_one( + filter_json, + replacement_json, + upsert = upsert, + session = session + ) + if result.modified_count or result.upserted_id: success = True + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not success: raise MongoReplaceException(hint = f"{collection}") + return success + + async def delete_one( + self, + collection, + filter_json, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter_json: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + result = await self.__db[collection].delete_one(filter_json, session = session) + count = result.deleted_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}") + return count + + async def delete_many( + self, + collection, + filter_json, + session = None, + raise_exception = False + ): + + """ + To delete many documents from a collection. + WARNING: sending {} in the filter would mean deleting ALL the documents. + :param collection: The collection from which you want to delete many records. + :param filter_json: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + result = await self.__db[collection].delete_many(filter_json, session = session) + count = result.deleted_count + except Exception as exception: + self.__printer(exception) + + # Check results and return: + if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}") + return count + + async def aggregate( + self, + collection, + pipeline, + limit = None, + raise_exception = False + ): + + """ + Perform an advance query on the data. + :param collection: The collection to perform the query on. + :param pipeline: The pipeline of actions to take. Must be a list. + :param limit: The max. no. of records to retrieve. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to perform the aggregation action: + try: results = await self.__db[collection].aggregate(pipeline).to_list(limit) + except Exception as exception: self.__printer(exception) + + # Check results and return: + if raise_exception and not results: raise MongoException(hint = "aggregation") + return results + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/database/async_mongo_storage.py b/database/async_mongo_storage.py new file mode 100644 index 0000000..94df2d8 --- /dev/null +++ b/database/async_mongo_storage.py @@ -0,0 +1,358 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 28th May, 2024 + + OBJECTIVE: + + To have one central place from where all async database connectivity happens. + + REFERENCES: + + 01. https://motor.readthedocs.io/en/stable/ + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +# --- +import sys +sys.path.append(".") +sys.path.append("..") + +# For system-level activity: +import io + +# For async behaviour: +import asyncio + +# MongoDB for File Storage: +from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket +from bson.objectid import ObjectId + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongoStorage: + + __db = None + __db_name = None + __client = None + __store = None + + def __init__( + self, + connection_string = None, + max_connections = 5, + host_name = "localhost", + port = 27017, + database_name = "fileStore", + debug = True, + debug_prefix = "GridFS (M) | ", + debug_only_errors = True + ): + + # Database Initialization: + self.__host_name = host_name + self.__port = port + self.__db_name = database_name + self.__connection_string = connection_string + self.__max_connections = max_connections + + # Debugging: + self.__debug_only_errors = debug_only_errors + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + @staticmethod + def generate_id(): + + """ + Just generates an '_id' in MongoDB style. + :return: The '_id' in MongoDB style. + """ + + return str(ObjectId()) + + async def connect(self): + + """ + Initialize the database connection. + :return: Nothing. + """ + + if self.__connection_string is None: + self.__client = AsyncIOMotorClient( + self.__host_name, + self.__port, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + else: + self.__client = AsyncIOMotorClient( + self.__connection_string, + maxPoolSize = self.__max_connections, + minPoolSize = self.__max_connections + ) + self.__db = self.__client.get_database(self.__db_name) + self.__store = AsyncIOMotorGridFSBucket(self.__db) + + @property + def fs(self): + + """ + To access the features that have not been wrapped in this reportlab directly. + This could include things like streaming files chunk-by-chunk. + :return: The file-store instance. + """ + + return self.__store + + async def write_from_memory(self, file_name, file_data, metadata_json = None): + + """ + Save a file (from RAM) to Mongo. Suitable for smaller files (a few MBs max.). + :param file_name: The name of the file. + :param file_data: The data of the file (held in RAM). + :param metadata_json: A JSON of metadata information that can later be used to search files (RECOMMENDED). + :return: The file's id as a string (if it gets saved) or None. + """ + + if self.__store is None: await self.connect() + + file_data.seek(0) + file_size = file_data.__sizeof__() + + file_id = None + try: file_id = await self.__store.upload_from_stream(file_name, file_data, metadata = metadata_json) + except Exception as exception: self.__printer(exception, file_name, file_size, file_id) + if not self.__debug_only_errors: self.__printer(file_name, file_size, file_id) + return str(file_id) + + async def read_to_memory(self, file_identifier, by_id = True): + + """ + To retrieve a file (in RAM) based on the provided identifier. + Suitable for smaller files (a few MBs max.). + :param file_identifier: Either the name or the "_id" of the file. + :param by_id: Set to True if you are fetching by the "_id" of the file. + :return: Either the file (in RAM) or None. + """ + + if self.__store is None: await self.connect() + + file_data = None + + try: + if by_id: grid_out = await self.__store.open_download_stream(ObjectId(file_identifier)) + else: grid_out = await self.__store.open_download_stream_by_name(file_identifier) + file_data = io.BytesIO(await grid_out.read()) + file_data.seek(0) + + except Exception as exception: + file_data = None + self.__printer(exception, file_identifier, by_id) + + if not self.__debug_only_errors: self.__printer(file_identifier, by_id) + return file_data + + async def delete_file_by_id(self, file_id): + + """ + Tries to delete one file by the id. + :param file_id: The id of the file in the database. + :return: True or False based on the success of the operation. + """ + + if self.__store is None: await self.connect() + + deleted = False + + try: + response = await self.__store.delete(file_id = ObjectId(file_id)) + deleted = True + except Exception as exception: + self.__printer(exception, file_id, deleted) + + return deleted + + def __format_metadata_json(self, metadata_json): + + """ + NOTE: ONLY USE WHEN SEARCHING FILES BY METADATA. + MongoDB expects dot-notation while searching for files by the metadata. We are making a function to search + files assuming that the conditions are to be applied to the metadata itself. So this function add the + dot-notation to the right places to conduct a successful search. + :param metadata_json: The JSON to format. + :return: The formatted JSON that has the right dot-notation. + """ + + formatted_metadata_json = {} + + for key, value in metadata_json.items(): + if not key.startswith("$"): key = f"metadata.{key}" + else: + if type(value) is dict: value = self.__format_metadata_json(value) + if type(value) is list: value = [self.__format_metadata_json(item) for item in value] + formatted_metadata_json[key] = value + + return formatted_metadata_json + + async def find_file_by_metadata(self, metadata_json, limit = None, skip = None, sort = None): + + """ + This method only lists the files that match the criteria mentioned in the metadata JSON. + :param metadata_json: The JSON that describes what you want to find. + :param limit: Max. no. of records to retrieve. + :param skip: No. of starting results to skip. Useful for pagination. + :param sort: The sorting conditions to follow. + :return: A list of (JSONs of) files that match the conditions. The list can be empty. + """ + + if self.__store is None: await self.connect() + + files_list = [] + + try: + limit = limit or 10 + skip = skip or 0 + sort = {"_id": -1} if not isinstance(sort, dict) else sort + formatted_metadata_json = self.__format_metadata_json(metadata_json) + return await self.__store.find( + formatted_metadata_json + ).sort(sort).skip(skip).limit(limit).to_list(None) + + except Exception as exception: self.__printer(exception, metadata_json, len(files_list)) + + if not self.__debug_only_errors: self.__printer(metadata_json, len(files_list)) + return files_list + + async def find_file_by_id(self, file_id): + + """ + This method allows you to get the file's info from the id of the file. + :param file_id: The id that was assigned by Mongo during upload. + :return: The file's info or None if the file doesn't exist. + """ + + if self.__store is None: await self.connect() + + file_info = None + + try: + formatted_metadata_json = {"_id": ObjectId(file_id)} + file_info = (await self.__store.find(formatted_metadata_json).to_list(1))[0] + + except Exception as exception: self.__printer(exception, file_id, file_info) + + return file_info + + async def get_file_name(self, file_id): + + """ + Returns the file name if the id of the file is known. + :param file_id: The id of the file as assigned by MongoDB when the file was stored. + :return: The file's name (if it exists), or None. + """ + + # Ensure that we are connected: + if self.__store is None: await self.connect() + + # Ensure that the input given is of 'ObjectId' type: + if type(file_id) is not ObjectId: file_id = ObjectId(str(file_id)) + + # Fetch and return the file name: + files_list = await self.__store.find( + {"_id": file_id}, + {"filename": True} + ).sort({"_id": -1}).limit(1).to_list(None) + try: file_name = files_list[0]["filename"] + except: file_name = None + return file_name + + async def aggregate( + self, + collection, + pipeline, + limit = None + ): + + """ + Perform an advance query on the data. + :param collection: The collection to perform the query on. + :param pipeline: The pipeline of actions to take. Must be a list. + :param limit: The max. no. of records to retrieve. + :return: The array of matching records or null if there was an exception. + """ + + if self.__store is None: await self.connect() + + results = None + try: results = await self.__db[collection].aggregate(pipeline).to_list(limit) + except Exception as exception: self.__printer(exception) + return results + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/database/async_mongo_v2.py b/database/async_mongo_v2.py new file mode 100644 index 0000000..dc9955a --- /dev/null +++ b/database/async_mongo_v2.py @@ -0,0 +1,1732 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Original: Tuesday, 28th May, 2024 + Modified: Tuesday, 17th Sept., 2024 + + OBJECTIVE: + + To have one central place from where all async MongoDB activity happens. + + REFERENCES: + + 01. https://motor.readthedocs.io/en/stable/ + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For system-level activity: +import io + +# For async behaviour: +import asyncio + +# MongoDB for File Storage: +from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket +from bson.objectid import ObjectId +from bson.json_util import dumps, loads + +# My utils: +from utils_v2.string import json + +# For datetime handling: +import pytz +import datetime + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** EXCEPTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMongoBase: + + def __init__( + self, + connection_string = None, + max_connections = 5, + host_name = "localhost", + port = 27017, + database_name = "myDb", + debug = True, + debug_prefix = "Mongo | ", + debug_only_errors = True + ): + + # Basic variables that will be needed later: + self._client = None + self._db = None + self._fs = None + + # Database Initialization: + self._host_name = host_name + self._port = port + self._db_name = database_name + self._connection_string = connection_string + self._max_connections = max_connections + + # Debugging: + self._debug_only_errors = debug_only_errors + self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self._printer.disable() + + def enable_debug(self): + self._printer.enable() + + def disable_debug(self): + self._printer.disable() + + async def connect(self): + + """ + Initialize the database connection. + :return: None. + """ + + # If a connection string is given, + # give preference to it: + if self._connection_string is None: + self._client = AsyncIOMotorClient( + self._host_name, + self._port, + maxPoolSize = self._max_connections, + minPoolSize = self._max_connections + ) + + # In the absense of a connection string, + # try to connect through the other credentials: + else: + self._client = AsyncIOMotorClient( + self._connection_string, + maxPoolSize = self._max_connections, + minPoolSize = self._max_connections + ) + + # Debugging print: + if not self._debug_only_errors: + server_info = await self._client.server_info() + self._printer(server_info) + + # Now we connect to the database and the file-store: + self._db = self._client.get_database(self._db_name) + self._fs = AsyncIOMotorGridFSBucket(self._db) + + async def ensure_connection(self): + + """ + Call this at the start of every function to automatically connect to the database when the user of the library + forgets to explicitly connect to the database. + :return: None. + """ + + if self._client is None: await self.connect() + + @staticmethod + def generate_id(as_str = False): + + """ + Just generates an '_id' in MongoDB style. + :param as_str: Set to True to convert the generated id to a string; and False to receive it as an instance of + 'ObjectId'. This is useful when you need to pre-assign ids to files. + :return: The '_id' in MongoDB style. + """ + + generated_id = ObjectId() + return str(generated_id) if as_str else generated_id + + @property + async def client(self): + + """ + returns the client to perform activities that have not been explicitly wrapped in the class. + :return: The client's instance. + """ + + await self.ensure_connection() + return self._client + + @property + async def db(self): + + """ + returns the database connector to perform activities that have not been explicitly wrapped in the class. + :return: The database's connection instance. + """ + + await self.ensure_connection() + return self._fs + + @property + async def fs(self): + + """ + returns the file-store to perform activities that have not been explicitly wrapped in the class. + :return: The file-store instance. + """ + + await self.ensure_connection() + return self._fs + + @staticmethod + def from_json_string(json_data): + + """ + Converts from a JSON string to BSON. + :param json_data: The JSON string to convert to BSON. + :return: The BSON interpretation of the input JSON string. + """ + + return loads(json_data) + + @staticmethod + def to_json_string(data, indent = 4, default = None): + + """ + Converts from a BSON to JSON string. + :param data: The input BSON data. + :param indent: The no. of spaces to put into the string for pretty print. + :param default: The default function to apply to data that cannot be converted directly. + :return: The JSON string from the input data. + """ + + return dumps(data, indent = indent, default = default) + + @staticmethod + def dict_to_dot_notation(input_dict, pk = "", s = "."): + + """ + Converts an input dict to dot notation format. Can be used as a utility to perform searches. + :param input_dict: The dict that you want to convert to dot notation. + :param pk: Parent Key. DO NOT TOUCH (meant to be used during recursion). + :param s: Separator. DO NOT TOUCH (meant to be used during recursion). + :return: The dot notation representation of the input dict. + """ + + items = [] + for k, v in input_dict.items(): + new_key = f"{pk}{s}{k}" if pk else k + if isinstance(v, dict) and v: items.extend(AsyncMongoBase.dict_to_dot_notation(v, new_key, s = s).items()) + else: items.append((new_key, v)) + return dict(items) + + @staticmethod + def normalize_date_time(document): + + """ + MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it. + This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was + timezone naive, UTC timezone will be applied to it without changing the time value. + :param document: The document that you want to normalize the date-time in. + :return: The document with normalized datetime. + """ + + if isinstance(document, datetime.datetime): + utc_tz = pytz.timezone("UTC") + if document.tzinfo is None: document = utc_tz.localize(document) + else: document = document.astimezone(utc_tz) + + if type(document) is list: + document = [AsyncMongoBase.normalize_date_time(item) for item in document] + + if type(document) is dict: + document = { + AsyncMongoBase.normalize_date_time(k): AsyncMongoBase.normalize_date_time(v) + for k, v in document.items() + } + + return document + + @staticmethod + def read_to_ram(file_path): + + """ + Reads a file into a BytesIO object in RAM. + :param file_path: The path to the file on disk. + :return: The file in a BytesIO object. + """ + + with open(file_path, "rb") as file: file_data = file.read() + file_in_ram = io.BytesIO(file_data) + file_in_ram.seek(0) + return file_in_ram + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncMongo(AsyncMongoBase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + async def list_indexes( + self, + collection, + session = None, + raise_exception = False + ): + + """ + Lists out the indexes of a collection. + :param collection: The collection whose indexes you want to list out, + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The list of indexes or None if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + indexes = None + + # Try to list the indexes: + try: + responses = await self._db[collection].list_indexes(session = session).to_list(None) + indexes = [{key: value for key, value in response.items()} for response in responses] + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return indexes + + async def create_index( + self, + collection, + keys, + options = None, + session = None, + raise_exception = False + ): + + """ + Creates an index on a collection. + :param collection: The collection to create the index on. + :param keys: The keys (and sorting) to implement the index on. + :param options: Additional config. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the execution. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the insertion: + try: + options = options or {} + keys = [(k, v) for k, v in keys.items()] + response = await self._db[collection].create_index(keys, session = session, **options) + if response: success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def count( + self, + collection, + filter, + session = None, + raise_exception = False + ): + + """ + Counts the no. of documents that match the given filter condition. + :param collection: The name of the collection to count in. + :param filter: The filter criteria that the documents must satisfy to be counted. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The count if the operation was performed successfully or None is something went wrong and the exception + was suppressed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # try to query the data: + try: count = await self._db[collection].count_documents(filter, session = session) + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def insert_one( + self, + collection, + document, + session = None, + raise_exception = False + ): + + """ + Insert data into a collection. + :param collection: The collection you want to feed the data into. + :param document: The data to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_id = None + + # Try to make the insertion: + try: + response = await self._db[collection].insert_one(document.copy(), session = session) + inserted_id = response.inserted_id + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return inserted_id + + async def insert_many( + self, + collection, + documents, + session = None, + raise_exception = False + ): + + """ + Insert a lot of data into a collection. + :param collection: The collection you want to feed the data into. + :param documents: The list of documents to be stored. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted data, or null if the action fails. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_ids = [] + + # Try to make the insertion: + try: + response = await self._db[collection].insert_many(documents, session = session) + inserted_ids = response.inserted_ids + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return inserted_ids + + async def find_one( + self, + collection, + filter, + projection = None, + session = None, + raise_exception = False + ): + + """ + Finds one record that matches the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The matching record or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to query the data: + try: results = await self._db[collection].find_one(filter, projection, session = session) + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Check results and return: + return results + + async def find_many( + self, + collection, + filter, + projection = None, + skip = 0, + limit = None, + sort = None, + session = None, + raise_exception = False + ): + + """ + Finds one or more records that match the given conditions. + :param collection: The name of the collection to perform the search in. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param skip: The no. of records to skip before picking next ones. Needed for pagination. + :param limit: The max. no. of records you want to fetch. + :param sort: The sorting rules to apply. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # Assume defaults: + if sort is None: sort = {"_id": -1} + if limit is None: limit = 10 + + # try to query the data: + try: + results = await self._db[collection].find( + filter, + projection, + sort = sort, + skip = skip, + limit = limit, + session = session + ).to_list(None) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return results + + async def update_one( + self, + collection, + filter = None, + update = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update one document. + :param collection: The collection you want to update. + :param filter: The selection criteria to locate the document to update. + :param update: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the update: + try: + response = await self._db[collection].update_one( + filter, + update, + upsert = upsert, + session = session + ) + success = False if response.modified_count == 0 else True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def update_many( + self, + collection, + filter = None, + update = None, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update many documents. + :param collection: The collection you want to update. + :param filter: The selection criteria to locate the document to update. + :param update: The values you want to update. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + update_count = 0 + + # Try to make the insertion: + try: + response = await self._db[collection].update_many( + filter, + update, + upsert = upsert, + session = session + ) + update_count = response.modified_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return update_count + + async def find_one_and_update( + self, + collection, + filter = None, + update = None, + projection = None, + return_updated = False, + upsert = False, + session = None, + raise_exception = False + ): + + """ + Update one document. + :param collection: The collection you want to update. + :param filter: The selection criteria to locate the document to update. + :param update: The values you want to update. + :param projection: What parts of the matching data you want to fetch. + :param return_updated: To choose whether you want to retrieve the original document or the updated document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + document = None + + # Try to make the update: + try: + document = await self._db[collection].find_one_and_update( + filter, + update, + projection = projection, + return_document = return_updated, + upsert = upsert, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return document + + async def replace_one( + self, + collection, + filter, + replacement, + upsert = False, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param replacement: The data to put in place of the existing document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # try to query the data: + try: + result = await self._db[collection].replace_one( + filter, + replacement, + upsert = upsert, + session = session + ) + if result.modified_count or result.upserted_id: success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def find_one_and_replace( + self, + collection, + filter, + replacement, + projection = None, + return_replaced = False, + upsert = False, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param replacement: The data to put in place of the existing document. + :param projection: What parts of the matching data you want to fetch. + :param return_replaced: To choose whether you want to retrieve the original document or the updated document. + :param upsert: If you want to insert if the document doesn't already exist. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + document = None + + # try to query the data: + try: + document = await self._db[collection].find_one_and_replace( + filter, + replacement, + projection = projection, + return_document = return_replaced, + upsert = upsert, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return document + + async def delete_one( + self, + collection, + filter, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # try to query the data: + try: + result = await self._db[collection].delete_one(filter, session = session) + success = True if result.deleted_count else False + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def delete_many( + self, + collection, + filter, + session = None, + raise_exception = False + ): + + """ + To delete many documents from a collection. + WARNING: sending {} in the filter would mean deleting ALL the documents. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + result = await self._db[collection].delete_many(filter, session = session) + count = result.deleted_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def find_one_and_delete( + self, + collection, + filter, + projection = None, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param collection: The collection from which you want to delete many records. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + document = None + + # try to query the data: + try: + document = await self._db[collection].find_one_and_delete( + filter, + projection = projection, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return document + + async def bulk_write( + self, + collection, + requests, + session = None, + raise_exception = False + ): + + """ + To perform various individual operations in one go. You will have to import individual actions like "UpdateOne" + and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method. + :param collection: The collection you want to run the requests on. + :param requests: The array of requests (operations) to be performed. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # try to query the data: + try: + response = await self._db[collection].bulk_write(requests, session = session) + count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def aggregate( + self, + collection, + pipeline, + limit = None, + session = None, + raise_exception = False + ): + + """ + Perform an advance query on the data. + :param collection: The collection to perform the query on. + :param pipeline: The pipeline of actions to take. Must be a list. + :param limit: The max. no. of records to retrieve. preferably apply the limit from within the pipeline. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to perform the aggregation action: + try: results = await self._db[collection].aggregate(pipeline, session = session).to_list(limit) + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return results + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncMongoStorage(AsyncMongoBase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + async def easy_upload( + self, + source, + file_name: str, + file_metadata: dict = None, + file_id = None, + chunk_size: int = None, + session = None, + raise_exception = False + ): + + """ + Easily write one file to MongoDB's GridFS. Ideal for directly uploading small files without having to worry + about any internal mechanisms. + :param source: The actual file, supplied as either a path string or a file-like object, to be written to the + database. + :param file_name: The name of the file as it will be stored on (and retrieved from) GridFS. + :param file_metadata: Any metadata to later search the file by. + :param file_id: Any custom id to be given to the file. TRY STICKING TO THE ID GENERATED BY 'generate_id'. + :param chunk_size: The chunk size (in bytes) to use for storing this file. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The id of the inserted file or None if the upload failed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + inserted_id = None + + # Try to write the file: + try: + + # Input pre-processing: + if isinstance(source, io.BytesIO): source.seek(0) + elif isinstance(source, str): source = open(source, mode = "rb") + + # If no file id was supplied: + if file_id is None: + inserted_id = await self._fs.upload_from_stream( + filename = file_name, + source = source, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + + # If a file id was supplied: + else: + await self._fs.upload_from_stream_with_id( + file_id = file_id, + filename = file_name, + source = source, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + inserted_id = file_id + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return inserted_id + + async def easy_download( + self, + destination, + file_id = None, + file_name = None, + session = None, + raise_exception = False + ): + + """ + Easily read one file from MongoDB's GridFS. Ideal for directly downloading small files without having to worry + about any internal mechanisms. + :param destination: The path on the local disk or a buffer in RAM to save the downloaded data to. + :param file_id: (RECOMMENDED) the id of the save file. + :param file_name: The name of the saved file. NOT RECOMMENDED because you could have many files with the same + name. The best way to tell files apart if from the id. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. The contents of the stored file are written + directly to the destination. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to read the file: + try: + + # Input pre-processing: + if isinstance(destination, str): destination = open(destination, mode = "wb") + + # If a file id is supplied (preferred way): + if file_id is not None: + await self._fs.download_to_stream( + destination = destination, + file_id = file_id, + session = session + ) + if isinstance(destination, io.BytesIO): destination.seek(0) + success = True + + # If a file name is supplied: + elif file_name is not None: + await self._fs.download_to_stream_by_name( + destination = destination, + filename = file_name, + session = session + ) + if isinstance(destination, io.BytesIO): destination.seek(0) + success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def count( + self, + filter, + session = None, + raise_exception = False + ): + + """ + Counts the no. of documents that match the given filter condition. + :param filter: The filter criteria that the documents must satisfy to be counted. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The count if the operation was performed successfully or None is something went wrong and the exception + was suppressed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # try to query the data: + try: count = await self._db["fs.files"].count_documents(filter, session = session) + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def find_one( + self, + filter, + projection = None, + session = None, + raise_exception = False + ): + + """ + Finds one record that matches the given conditions. This does NOT return the file itself, it returns the record + that describes the file. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The matching record or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # try to query the data: + try: results = await self._db["fs.files"].find_one(filter, projection, session = session) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Check results and return: + return results + + async def find_many( + self, + filter, + projection = None, + skip = 0, + limit = None, + sort = None, + session = None, + raise_exception = False + ): + + """ + Finds one or more records that match the given conditions. This doesn't return any actual files directly, it + returns the records that describe the files. + :param filter: The filter criteria. + :param projection: What parts of the matching data you want to fetch. + :param skip: The no. of records to skip before picking next ones. Needed for pagination. + :param limit: The max. no. of records you want to fetch. + :param sort: The sorting rules to apply. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The array of matching records or null if there was an exception. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + results = None + + # Assume defaults: + if sort is None: sort = {"_id": -1} + if limit is None: limit = 10 + + # try to query the data: + try: + results = await self._db["fs.files"].find( + filter, + projection, + session = session + ).sort(sort).skip(skip).limit(limit).to_list(None) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return results + + async def update_metadata_for_one( + self, + filter: dict, + unset_data: dict = None, + set_data: dict = None, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for one file. + :param filter: The conditions to filter the files by. + :param unset_data: The fields that you want to discard. + :param set_data: The fields that you want to add or update. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the update: + try: + + # Input pre-processing: + update = {} + if unset_data is not None: update["$unset"] = self.dict_to_dot_notation({"metadata": unset_data}) + if set_data is not None: update["$set"] = self.dict_to_dot_notation({"metadata": set_data}) + + # Actual update happens here: + if update: + response = await self._db["fs.files"].update_one( + filter, + update, + session = session + ) + success = False if response.modified_count == 0 else True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def update_metadata_for_many( + self, + filter: dict, + unset_data: dict = None, + set_data: dict = None, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for many files. Practically the same as 'update_metadata_for_one' except that the scope of + the modifications is far wider. + :param filter: The conditions to filter the files by. + :param unset_data: The fields that you want to discard. + :param set_data: The fields that you want to add or update. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # Try to make the update: + try: + + # Input pre-processing: + update = {} + if unset_data is not None: update["$unset"] = self.dict_to_dot_notation({"metadata": unset_data}) + if set_data is not None: update["$set"] = self.dict_to_dot_notation({"metadata": set_data}) + + # Actual update happens here: + if update: + response = await self._db["fs.files"].update_many( + filter, + update, + session = session + ) + count = response.modified_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def replace_metadata_for_one( + self, + filter: dict, + replacement, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for one file. + :param filter: The conditions to filter the files by. + :param replacement: The new metadata to put inplace of the old one. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to make the update: + try: + + # Actual update happens here: + result = await self._db["fs.files"].update_one( + filter, + {"$set": {"metadata": replacement}}, + upsert = False, + session = session + ) + if result.modified_count or result.upserted_id: success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def replace_metadata_for_many( + self, + filter: dict, + replacement, + session = None, + raise_exception = False + ): + + """ + Updates the metadata for one file. + :param filter: The conditions to filter the files by. + :param replacement: The new metadata to put inplace of the old one. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = None + + # Try to make the update: + try: + + # Actual update happens here: + response = await self._db["fs.files"].update_many( + filter, + {"$set": {"metadata": replacement}}, + session = session + ) + count = response.modified_count + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return count + + async def delete_by_id( + self, + file_id, + session = None, + raise_exception = False + ): + + """ + Deletes one file by the file's id. Deleting files is different from deleting simple documents because, in files, + you also need to clear out the chunks (which are stored in a separate collection). The built-in mechanism of + Motor only provides support to delete by the file's id. + :param file_id: The id of the file that you want to delete. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Try to read the file: + try: + + # Delete the file: + await self._fs.delete(file_id = file_id, session = session) + success = True + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return success + + async def delete_one( + self, + filter, + session = None, + raise_exception = False + ): + + """ + To delete one document from a collection. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: True or False based on the success of the operation. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + success = False + + # Find one file that matches the given filter: + file = await self.find_one( + filter = filter, + session = session, + raise_exception = raise_exception + ) + + # If we have a list of files to work with: + if file: success = await self.delete_by_id(file["_id"]) + + # Done here: + return success + + async def delete_many( + self, + filter, + session = None, + raise_exception = False + ): + + """ + To delete many documents from a collection. + WARNING: sending {} in the filter would mean deleting ALL the documents. + :param filter: The filter criteria. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The number of files deleted (can be zero) or None if something failed while searching the files. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + count = 0 + + # Find all the files that match the given filter: + files = await self.find_many( + filter = filter, + session = session, + raise_exception = raise_exception + ) + + # If we have a list of files to work with: + if files is not None: + tasks = [self.delete_by_id(file["_id"]) for file in files] + results = await asyncio.gather(*tasks) + count = sum(results) + + # Done here: + return count + + async def get_upload_stream( + self, + file_name: str, + file_metadata: dict = None, + file_id = None, + chunk_size: int = None, + session = None, + raise_exception = False + ): + + """ + Returns a GridIn object so that you can perform your own upload using the built-in writing methods. You must use + the 'write' method to write data to the file by passing it either a string of bytes or a file-like object. When + the file has been fully written, you must call the 'close' method to finish the operation. In case you need to + cancel the operation, you can call the 'abort' method to delete all the already written data and stop uploading + new data. + :param file_name: The name of the file as it will be stored on (and retrieved from) GridFS. + :param file_metadata: Any metadata to later search the file by. + :param file_id: Any custom id to be given to the file. TRY STICKING TO THE ID GENERATED BY 'generate_id'. + :param chunk_size: The chunk size (in bytes) to use for storing this file. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The upload stream that implements the 'write', 'close', and 'abort' methods, or None if something + failed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + stream = None + + # Try to open the upload stream: + try: + + # If no file id was supplied: + if file_id is None: + stream = await self._fs.open_upload_stream( + filename = file_name, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + + # If a file id was supplied: + else: + stream = await self._fs.open_upload_stream_with_id( + file_id = file_id, + filename = file_name, + metadata = file_metadata, + chunk_size_bytes = chunk_size, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return stream + + async def get_download_stream( + self, + file_id = None, + file_name = None, + session = None, + raise_exception = False + ): + + """ + Returns a GridOut object so that you can implement your own download logic using the built-in 'read' method. + Once the reading is done, use the 'close' method to release the resources used by the stream. + :param file_id: (RECOMMENDED) the id of the save file. + :param file_name: The name of the saved file. NOT RECOMMENDED because you could have many files with the same + name. The best way to tell files apart if from the id. + :param session: The session if you need to do this in a transaction. + :param raise_exception: Whether, or not, you want to raise an exception when something fails. + :return: The download stream that implements the 'read' and 'close' methods, or None if something failed. + """ + + # Ensure you are connected: + await self.ensure_connection() + + # Assume failure: + stream = None + + # Try to open a download stream: + try: + + # If a file id is supplied (preferred way): + if file_id is not None: + stream = await self._fs.open_download_stream( + file_id = file_id, + session = session + ) + + # If a file name is supplied: + elif file_name is not None: + stream = await self._fs.open_download_stream_by_name( + filename = file_name, + session = session + ) + + # When something goes wrong: + except Exception as exception: + self._printer(exception) + if raise_exception: raise + + # Done here: + return stream + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + from shared import constants + + async def main(): + + # Create an instance of the database connector: + my_fs = AsyncMongoStorage( + connection_string = constants.MONGO_FILE_CONNECTION_STRING, + database_name = constants.MONGO_FILE_DATABASE_NAME, + max_connections = 10, + debug = True + ) + + # Connect to the database: + await my_fs.connect() + + # Keep performing the changes in batches till you have corrections to make: + while True: + + # Find all the files that have their metadata as a string: + files = await my_fs.find_many( + filter = { + "metadata": {"$type": "string"} + }, + limit = 10 + ) + + print(my_fs.to_json_string(files)) + break + + # # If no matches were found: + # if not files: break + # + # # Fix the metadata file-by-file: + # for file in files: + # file_id = file["_id"] + # success = await my_fs.replace_metadata_for_one( + # filter = {"_id": file_id}, + # replacement = json.from_string(file["metadata"]) + # ) + # print(file_id, ":", success) + + print("Fixes done!") + + asyncio.run(main()) diff --git a/database/async_mysql_v2.py b/database/async_mysql_v2.py new file mode 100644 index 0000000..911bfaf --- /dev/null +++ b/database/async_mysql_v2.py @@ -0,0 +1,311 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 30th Aug., 2024 + + OBJECTIVE: + + To be able to access SQL-based databases from python in a simple way. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# MySQL Database: +import aiomysql + +# For data-crunching: +import pandas as pd + +# For time-keeping: +import time + +# OS-level operations: +import os + +# My utils: +from utils_v2.string import json + +# For async activities: +import asyncio + +# For debugging: +from icecream import IceCreamDebugger +import traceback + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncMySQL: + + def __init__( + self, + pool_size, + *args, + **kwargs + ): + + """ + A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as + JSON-like structures (list or dict). + :param pool_size: The number of connections to maintain n a pool. + :param args: Any arguments to pass. Not used. + :param kwargs: Pass the connection configuration from here. + """ + + # Set up the variables: + self.__args = args + self.__kwargs = kwargs + self.__min_pool_size = 10 + self.__max_pool_size = max(pool_size, self.__min_pool_size) + self.__pool = None + + # Set up the debugging tools: + self.__printer = IceCreamDebugger(prefix = "MySQL | ", includeContext = True) + + def __del__(self): + pass + + async def connect(self): + + """ + Establish a connection and create a pool of connections to call from. + :return: None. + """ + + try: + + self.__kwargs["db"] = self.__kwargs.pop("database") + self.__pool = await aiomysql.create_pool( + minsize = self.__min_pool_size, + maxsize = self.__max_pool_size, + loop = asyncio.get_event_loop(), + **self.__kwargs + ) + + except Exception as exception: + self.__printer(exception) + self.__pool = None + + async def ensure_connection(self): + + """ + Tries to ensure that a connection is present. + Can be called before every function to make sure that our pool is established. + :return: None. + """ + + if self.__pool is None: await self.connect() + + async def call_procedure(self, procedure_name, procedure_args): + + """ + To call stored procedures and retrieve all the responses. + :param procedure_name: + :param procedure_args: + :return: + """ + + # Make sure we have a connection: + await self.ensure_connection() + + # Make a variable to hold all the result sets. + # Needed for when the procedure responds with many "tables": + all_result_sets = [] + + # Call the procedure and get the results: + async with self.__pool.acquire() as connection: + async with connection.cursor() as cursor: + await cursor.callproc(procedure_name, procedure_args) + + # Iterate over all result sets, + # and process them one-by-one: + while True: + this_result_set = [] + result = await cursor.fetchall() + if not result: break + columns = [desc[0] for desc in cursor.description] + for row in result: this_result_set.append(dict(zip(columns, row))) + all_result_sets.append(this_result_set) + await cursor.nextset() + + # Done here: + return all_result_sets + + async def call_procedure_and_get_json( + self, + procedure_name, + procedure_args, + retry_count = 1, + backoff_seconds = 0.5, + backoff_multiplier = 1.1, + return_exception = False + ): + + # Note down the start time: + start_ts = time.perf_counter() + + # Try to get the data from the database: + results = [] + exception = None + for _ in range(retry_count): + try: results = await self.call_procedure( + procedure_name = procedure_name, + procedure_args = procedure_args, + ) + except Exception as exc: exception = exc + if exception is None: break + await asyncio.sleep(backoff_seconds) + backoff_seconds = backoff_seconds * backoff_multiplier + + print("LEN:", len(results)) + + # If the results are blank: + if len(results) == 0: return { + "status": results[0][0]["status"], + "message": results[0][0].get("message", "please contact admin (E)"), + "seconds": None, + "data": {} + } + + # Extract the very basic success or failure indicators: + formatted_results = { + "status": results[0][0]["status"], + "message": results[0][0].get("message", "ok"), + "seconds": 0.0, + "data": {} + } + + # Handle the remaining keys of the zeroth result set: + for key, value in results[0][0].items(): + if key not in formatted_results.keys(): + formatted_results["data"][key] = value + + # Format + for index in range(len(results)): + if index > 0: formatted_results["data"][f"rs{index-1}"] = results[index] + + # Note down the time taken: + formatted_results["seconds"] = time.perf_counter() - start_ts + + # Done here: + print(f"{procedure_name}:") + print(json.to_string(formatted_results)) + print("\n") + if return_exception: return formatted_results, exception + else: return formatted_results + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + async def test(connector): + pass + + async def multi_test(count = 1): + + """ + Test asynchronous behaviour. + :return: None. + """ + + cred_json = { + "host": "del.ditscentre.in", + "user": "bicree", + "port": 3306, + "password": "9c3b2808a4aa281129d399fe09e69b53", + "database": "bicree" + } + + db_conn = AsyncMySQL( + pool_size = 25, + **cred_json + ) + + await db_conn.connect() + + # result = await db_conn.call_procedure_and_get_json( + # procedure_name = "login", + # procedure_args = ( + # "shree", + # "shree", + # "aiomysql", + # "127.0.0.1" + # ) + # ) + + result = await db_conn.call_procedure_and_get_json( + procedure_name = "listSummary", + procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",) + ) + + + start_time = time.time() + asyncio.run(multi_test(count = 1)) + print(f"ASYNC HITS DONE IN: {time.time() - start_time} seconds.") + diff --git a/date_time/__init__.py b/date_time/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/date_time/__pycache__/__init__.cpython-310.pyc b/date_time/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..98444f4 Binary files /dev/null and b/date_time/__pycache__/__init__.cpython-310.pyc differ diff --git a/date_time/__pycache__/date_time.cpython-310.pyc b/date_time/__pycache__/date_time.cpython-310.pyc new file mode 100644 index 0000000..33397e9 Binary files /dev/null and b/date_time/__pycache__/date_time.cpython-310.pyc differ diff --git a/date_time/date_time.py b/date_time/date_time.py new file mode 100644 index 0000000..7437fc7 --- /dev/null +++ b/date_time/date_time.py @@ -0,0 +1,270 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 21st jun, 2024 + + OBJECTIVE: + + To provide an easy way to work with time. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For date and time handling: +import pytz +from datetime import datetime, timedelta +import dateparser + +# To handle date-time objects from a Pandas Dataframe: +import pandas as pd + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Date-Time Formats: +DATE_TIME_FORMATS = ( + "%d/%m/%y", + "%d-%b-%y", + "%d-%m-%y", + "%d.%m.%y", + "%d/%m/%Y", + "%d-%b-%Y", + "%d-%m-%Y", + "%d.%m.%Y", + "%d/%b", + "%d%m%Y", + "%Y%m%d", + "%Y-%m-%d %H:%M:%S" +) + +# Useful Timezones: +TIMEZONE_UTC = pytz.timezone("UTC") +TIMEZONE_IST = pytz.timezone("Asia/Kolkata") +TIMEZONE_ET = pytz.timezone("America/New_York") +TIMEZONE_CT = pytz.timezone("America/Chicago") +TIMEZONE_MT = pytz.timezone("America/Denver") +TIMEZONE_PT = pytz.timezone("America/Los_Angeles") +TIMEZONE_JST = pytz.timezone("Asia/Tokyo") +TIMEZONE_CET = pytz.timezone("Europe/Paris") +TIMEZONE_GMT = pytz.timezone("GMT") +TIMEZONE_AEST = pytz.timezone("Australia/Sydney") +TIMEZONE_NZST = pytz.timezone("Pacific/Auckland") +TIMEZONE_CST = pytz.timezone("Asia/Shanghai") +TIMEZONE_KST = pytz.timezone("Asia/Seoul") +TIMEZONE_MSK = pytz.timezone("Europe/Moscow") +TIMEZONE_BRT = pytz.timezone("America/Sao_Paulo") +TIMEZONE_GST = pytz.timezone("Asia/Dubai") +TIMEZONE_SAST = pytz.timezone("Africa/Johannesburg") +TIMEZONE_AST = pytz.timezone("Asia/Riyadh") + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def translate_date_time_string( + datetime_string, + source_format = None, + destination_format = "%Y-%m-%dT%H:%M:%S" +): + + """ + To convert an input datetime string to a different format. + :param datetime_string: The datetime string to translate. + :param source_format: The current format of the string. If not provided, dateparser will be used. + :param destination_format: The format to convert to. + :return: The converted datetime string. + """ + + try: + if source_format is None: datetime_obj = dateparser.parse(datetime_string) + else: datetime_obj = datetime.strptime(datetime_string, source_format) + return datetime_obj.strftime(destination_format) + except Exception as exception: + return None + + +# --------------------------------------------------------------------------------------------------------------------- + + +def parse_date_time(input_value, timezone = None, date_formats = None): + + """ + To take any kind of input and interpret the datetime from it. + :param input_value: Either a string or an integer or some form of datetime representation. + :param timezone: The timezone to apply to the interpreted datetime. EXISTING TIMEZONE INFO WILL BE OVERWRITTEN. + :param date_formats: The string formats to consider when parsing a string input. + :return: The parsed datetime or null. + """ + + datetime_object = None + date_formats = date_formats or DATE_TIME_FORMATS + + # It could either be in seconds or milliseconds from epoch time's base date (January 1, 1970), + # or it could be days since Microsoft Excel's base date (December 31, 1899). + if type(input_value) in [int, float]: + if input_value > 9999999999.0: datetime_object = datetime.fromtimestamp(input_value / 1000.0) + if input_value > 999999.0: datetime_object = datetime.fromtimestamp(input_value) + else: datetime_object = datetime.fromtimestamp(input_value * 24 * 60 * 60.0) - timedelta(days = 25569) + + # The input can even be a pre-formatted date: + if type(input_value) is str: + datetime_object = dateparser.parse(input_value, date_formats = date_formats) + + # If the type is a datetime object, then return it as it is: + if type(input_value) is datetime: + datetime_object = input_value + + # If the type is the native datetime format of pandas: + if type(input_value) is pd._libs.tslibs.timestamps.Timestamp: + datetime_object = input_value.to_pydatetime() + + # Process the timezone: + if datetime_object is not None and timezone is not None: + datetime_object = as_if_timezone(datetime_object, timezone) + + # Done here: + return datetime_object + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_current_date_time(timezone = None, as_string = False): + + """ + Returns the current time as a datetime object. + :param timezone: The timezone to apply to the returned datetime. + :param as_string: Whether, or not, you want the output as a string. + :return: The datetime object/string representing the current time. + """ + + if timezone is not None and isinstance(timezone, str): timezone = pytz.timezone(timezone) + now = datetime.now(timezone) + return now.isoformat() if as_string else now + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_current_ist_date_time(as_string = False): + + """ + Gives out the current time in IST timezone. + :param as_string: Whether, or not, you want the output as a string. + :return: The datetime object or string representing the current time. + """ + + return get_current_date_time( + timezone = TIMEZONE_IST, + as_string = as_string + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_current_utc_date_time(as_string = False): + + """ + Gives out the current time in UTC timezone. + :param as_string: Whether, or not, you want the output as a string. + :return: The datetime object or string representing the current time. + """ + + return get_current_date_time( + timezone = TIMEZONE_UTC, + as_string = as_string + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def as_if_timezone(datetime_object, timezone): + + """ + Ignores existing timezone info and applies the intended timezone. + The time stays the same, only the timezone marker changes. + e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-09 00:00:00+00:00 + HINT: IT PRETENDS "AS IF" THE TIMEZONE WAS THE INPUT TIMEZONE. + :param datetime_object: The datetime object on which the timezone needs to be applied. + :param timezone: The timezone that needs to be applied. + :return: A timezone-aware datetime object. + """ + + tz_object = pytz.timezone(timezone) if isinstance(timezone, str) else timezone + return tz_object.localize(datetime_object.replace(tzinfo = None)) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_timezone(datetime_object, timezone): + + """ + Converts from one timezone to another. The time is adjusted by computing the difference between the two timezones. + NOTE: THIS FUNCTION ASSUMES THE INPUT WAS IN UTC IF THE INPUT WAS TIMEZONE-NAIVE. + e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-08 18:30:00+00:00 + :param datetime_object: The datetime object on which the timezone needs to be applied. + :param timezone: The timezone that needs to be applied. + :return: A timezone-aware datetime object. + """ + + if isinstance(timezone, str): timezone = pytz.timezone(timezone) + if datetime_object.tzinfo is None: return datetime_object.replace(tzinfo = TIMEZONE_UTC) + return datetime_object.astimezone(timezone) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/image/__init__.py b/image/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/image/scanner/__init__.py b/image/scanner/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/image/scanner/scanner.py b/image/scanner/scanner.py new file mode 100644 index 0000000..424ddd1 --- /dev/null +++ b/image/scanner/scanner.py @@ -0,0 +1,410 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 19th Sept., 2024 + + OBJECTIVE: + + To provide a way to easily scan documents like printouts and visiting cards. + + REFERENCES: + + 01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085 + 02. https://github.com/JaidedAI/EasyOCR + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys + +from diffusers.utils.import_utils import candidates + +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To work with image processing: +import cv2 +import imutils +import numpy as np + +# To run OCR: +import easyocr + +# To download images from the web: +import requests + +# My utils: +from utils_v2.string import json +from utils_v2.ai.object_detection.yolo import YoloDetect + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DocumentScanner: + + def __init__( + self, + layout_detection_yolo = None, + whitelisted_yolo_classes = None, + ocr_languages = None + ): + + self.__yolo = YoloDetect( + model_file = layout_detection_yolo, + debug = False + ) + self.__whitelisted_yolo_classes = whitelisted_yolo_classes or [] + self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"]) + pass + + @staticmethod + def open_image(image): + + """ + Opens an image from various sources. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :return: The image as a CV2 object. + """ + + # When the image is provided as a string, + # it could be either as a URL or a path to a local file: + if isinstance(image, str): + if image.startswith("https://") or image.startswith("http://"): + image = io.BytesIO(requests.get(image).content) + else: image = cv2.imread(image) + + # If the image is provided as a io.BytesIO buffer: + if isinstance(image, io.BytesIO): + image.seek(0) + image = np.asarray(bytearray(image.read()), dtype = np.uint8) + image = cv2.imdecode(image, cv2.IMREAD_COLOR) + + # Done here: + return image + + @staticmethod + def polygon_area(points): + + """ + Computes the area occupied by a shape described by the array of points. + Example input: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]] + :return: The area of the shape. + """ + + n = len(points) + area = 0.0 + for i in range(n): + j = (i + 1) % n # next vertex + area += points[i][0] * points[j][1] + area -= points[j][0] * points[i][1] + return abs(area) / 2.0 + + @staticmethod + def show(image, title = "Preview", wait = True): + + """ + Just a quick wrapper to show the image in a window. + :param image: The image that you want to show. + :param title: The title of the window. + :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many + windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the + very last one. + :return: None. + """ + + cv2.imshow(title, image) + if wait: + cv2.waitKey() + cv2.destroyAllWindows() + + @staticmethod + def enhance(image): + + """ + Applies a form of contrast boost to make the edges more easily visible. + :param image: The OpenCV image that needs to be enhanced. + :return: The enhanced CV2 image. + """ + + return cv2.detailEnhance( + src = image, + sigma_s = 20, + sigma_r = 0.15 + ) + + @staticmethod + def get_edges(image): + + """ + Gets the edges in an image. + :param image: A CV2 image. + :return: The image with the edges detected. + """ + + # Denoise the image: + processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + processed_image = cv2.GaussianBlur( + processed_image, + ksize = (5, 5), + sigmaX = 0 + ) + + # Detect edges in the image: + processed_image = cv2.Canny( + processed_image, + threshold1 = 50, + threshold2 = 200 + ) + + # Close small gaps in the edges: + kernel = np.ones((5, 5), np.uint8) + processed_image = cv2.dilate(processed_image, kernel, iterations = 1) + # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel) + # processed_image = cv2.erode(processed_image, kernel, iterations = 1) + + # Done here: + return processed_image + + def scan(self, image, min_area = 0.125): + + """ + Looks for a rectangle in an image and flattens it out. No colour filters are applied here. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_area: The minimum area occupied by the document in the image. + :return: The flattened image if any, else None. + """ + + # Open the image and read the data: + image = self.open_image(image) + if image is None: return None + height, width = image.shape[:2] + image_area = height * width + + # Enhance the image to have better visibility of edges: + # processed_image = self.enhance(image.copy()) + + # Detect edges in the image: + processed_image = self.get_edges(image.copy()) + + # Find the contours in the edges, + # and sort them in ascending order: + contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + contours = imutils.grab_contours(contours) + contours = sorted(contours, key = cv2.contourArea, reverse = True) + + # We try to get clean and passable contours. + # For our purposes, 'clean' is when it can be approximated to exactly 4 sides, + # and 'passable' is when the approximation comes close to it: + clean_rects = [] + passable_rects = [] + for contour in contours: + perimeter = cv2.arcLength(contour, closed = True) + approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True) + if len(approximation) == 4: clean_rects.append(approximation) + elif len(approximation) <= 32: + candidate_contour = cv2.minAreaRect(approximation) + candidate_contour = cv2.boxPoints(candidate_contour) + candidate_contour = np.intp(candidate_contour) + passable_rects.append(candidate_contour) + clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True) + passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True) + + # Now we decide between the best candidate: + document_outline = None + if len(clean_rects) > 0: + candidate_rect = np.array([p[0] for p in clean_rects[0]]) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + if document_outline is None and len(passable_rects) > 0: + candidate_rect = cv2.minAreaRect(passable_rects[0]) + candidate_rect = cv2.boxPoints(candidate_rect) + candidate_rect = np.intp(candidate_rect) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + + # If there is no rectangular contour, + # we exit with failure: + if document_outline is None: return None + + # Since we did get the best candidate for the document, + # we figure out which point n the contour is which corner of the document: + document_rectangle = np.zeros((4, 2), dtype = "float32") + sum_points = document_outline.sum(axis = 1) + document_rectangle[0] = document_outline[np.argmin(sum_points)] + document_rectangle[2] = document_outline[np.argmax(sum_points)] + diff_points = np.diff(document_outline, axis = 1) + document_rectangle[1] = document_outline[np.argmin(diff_points)] + document_rectangle[3] = document_outline[np.argmax(diff_points)] + (top_left, top_right, bottom_right, bottom_left) = document_rectangle + + # let's compute the dimensions of the document in the image: + left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2)) + right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2)) + top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2)) + bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2)) + max_height = max(int(left_height), int(right_height)) + max_width = max(int(top_width), int(bottom_width)) + + # We compute the destination of the transform: + destination = np.array( + object = [ + [0, 0], + [max_width - 1, 0], + [max_width - 1, max_height - 1], + [0, max_height - 1] + ], + dtype = "float32" + ) + + # We apply the transform that flattens out the document: + matrix = cv2.getPerspectiveTransform(document_rectangle, destination) + flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height)) + + # Done here: + return flattened_image + + def extract_text( + self, + image, + min_confidence = 0.5, + margin = 0.05 + ): + + """ + To run OCR on the input image. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_confidence: The minimum amount of confidence for detected text to be considered. + :param margin: The extra pixels to include when cropping into a section for OCR. + :return: The extracted text. + """ + + # Make a variable that will hold the final result: + extracted_text = {} + + # Open the image and read the data: + image = self.open_image(image) + height, width = image.shape[:2] + + # Run OCR on the entire canvas: + ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence] + extracted_text["fullDoc"] = " ".join(ocr_result) + + # Get the doc layout in the image: + document_sections = self.__yolo.predict(image) + document_sections = document_sections["boxes"] + + # For each section, we run the OCR process individually: + section_results = [] + for section in document_sections: + if section["class"] in self.__whitelisted_yolo_classes: + section_width = section["x2"] - section["x1"] + section_height = section["y2"] - section["y1"] + x1 = max(int(section["x1"] - (margin * section_width)), 0) + y1 = max(int(section["y1"] - (margin * section_height)), 0) + x2 = min(int(section["x2"] + (margin * section_width)), width) + y2 = min(int(section["y2"] + (margin * section_height)), height) + sub_image = image[y1:y2, x1:x2] + ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence] + section_results.append({ + "text": " ".join(ocr_result), + "class": section["class"], + "className": section["className"], + "x1": x1, + "y1": y1, + "x2": x2, + "y2": y2 + }) + extracted_text["bySection"] = section_results + + # Done here: + return extracted_text + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import time + + my_scanner = DocumentScanner( + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], + # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], + ocr_languages = ["en"] + ) + + image = my_scanner.open_image(r"/home/developer/Downloads/talkaholics_card.jpg") + document_image = my_scanner.scan(image) + if document_image is not None: + cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image) + start_time = time.time() + document_text = my_scanner.extract_text( + document_image, + min_confidence = 0.5 + ) + print("OCR RESULT:") + print(document_text) + + print(f"FINISHED IN {time.time() - start_time} SECONDS!") + + else: print("No image") diff --git a/image/scanner/scanner_v2.py b/image/scanner/scanner_v2.py new file mode 100644 index 0000000..2a5b9fc --- /dev/null +++ b/image/scanner/scanner_v2.py @@ -0,0 +1,379 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 24th Sept., 2024 + + OBJECTIVE: + + To provide a way to easily scan documents like printouts and visiting cards. + + REFERENCES: + + 01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085 + 02. https://github.com/JaidedAI/ + 03. https://github.com/criistian14/flutter_document_scanner/blob/master/flutter_document_scanner_android/android/src/main/kotlin/com/christian/flutterDocumentScanner/OpenCVPlugin.kt + 04. https://www.geeksforgeeks.org/python-bilateral-filtering/ + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To work with image processing: +import cv2 +import imutils +import numpy as np + +# To run OCR: +import easyocr + +# To download images from the web: +import requests + +# My utils: +from utils_v2.string import json +from utils_v2.ai.object_detection.yolo import YoloDetect + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class DocumentScanner: + + def __init__( + self, + layout_detection_yolo = None, + whitelisted_yolo_classes = None, + ocr_languages = None + ): + + self.__yolo = YoloDetect( + model_file = layout_detection_yolo, + debug = False + ) + self.__whitelisted_yolo_classes = whitelisted_yolo_classes or [] + self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"]) + pass + + @staticmethod + def open_image(image): + + """ + Opens an image from various sources. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :return: The image as a CV2 object. + """ + + # When the image is provided as a string, + # it could be either as a URL or a path to a local file: + if isinstance(image, str): + if image.startswith("https://") or image.startswith("http://"): + image = io.BytesIO(requests.get(image).content) + else: image = cv2.imread(image) + + # If the image is provided as a io.BytesIO buffer: + if isinstance(image, io.BytesIO): + image.seek(0) + image = np.asarray(bytearray(image.read()), dtype = np.uint8) + image = cv2.imdecode(image, cv2.IMREAD_COLOR) + + # Done here: + return image + + @staticmethod + def show(image, title = "Preview", wait = True): + + """ + Just a quick wrapper to show the image in a window. + :param image: The image that you want to show. + :param title: The title of the window. + :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many + windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the + very last one. + :return: None. + """ + + cv2.imshow(title, image) + if wait: + cv2.waitKey() + cv2.destroyAllWindows() + + @staticmethod + def get_edges(image): + + """ + Apply all the preprocessing filters on the image before sending it off for contour-finding. + :param image: The image to pre-process. + :return: The pre-processed image. + """ + + # Denoise the image: + processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + processed_image = cv2.bilateralFilter( + processed_image, + d = 9, + sigmaColor = 75, + sigmaSpace = 75 + ) + + # Detect edges in the image: + processed_image = cv2.Canny( + processed_image, + threshold1 = 75, + threshold2 = 200 + ) + + # Close small gaps in the edges: + kernel = np.ones((5, 5), np.uint8) + processed_image = cv2.dilate(processed_image, kernel, iterations = 1) + # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel) + # processed_image = cv2.erode(processed_image, kernel, iterations = 1) + + # Done here: + return processed_image + + def scan(self, image, min_area = 0.125): + + """ + Looks for a rectangle in an image and flattens it out. No colour filters are applied here. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_area: The minimum area occupied by the document in the image. + :return: The flattened image if any, else None. + """ + + # Open the image and read the data: + image = self.open_image(image) + if image is None: return None + height, width = image.shape[:2] + image_area = height * width + + # image = self.enhance(image) + processed_image = self.get_edges(image) + # self.show(processed_image) + + # Find the contours in the edges, + # and sort them in ascending order: + contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + contours = imutils.grab_contours(contours) + contours = sorted(contours, key = cv2.contourArea, reverse = True) + + # We try to get clean and passable contours. + # For our purposes, 'clean' is when it can be approximated to exactly 4 sides, + # and 'passable' is when the approximation comes close to it: + clean_rects = [] + passable_rects = [] + for contour in contours: + perimeter = cv2.arcLength(contour, closed = True) + approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True) + if len(approximation) == 4: clean_rects.append(approximation) + elif len(approximation) <= 32: + candidate_contour = cv2.minAreaRect(approximation) + candidate_contour = cv2.boxPoints(candidate_contour) + candidate_contour = np.intp(candidate_contour) + passable_rects.append(candidate_contour) + clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True) + passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True) + + # Now we decide between the best candidate: + document_outline = None + if len(clean_rects) > 0: + candidate_rect = np.array([p[0] for p in clean_rects[0]]) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + if document_outline is None and len(passable_rects) > 0: + candidate_rect = cv2.minAreaRect(passable_rects[0]) + candidate_rect = cv2.boxPoints(candidate_rect) + candidate_rect = np.intp(candidate_rect) + rectangle_area = cv2.contourArea(candidate_rect) + if rectangle_area / image_area >= min_area: document_outline = candidate_rect + + # If there is no rectangular contour, + # we exit with failure: + if document_outline is None: return None + + # Since we did get the best candidate for the document, + # we figure out which point n the contour is which corner of the document: + document_rectangle = np.zeros((4, 2), dtype = "float32") + sum_points = document_outline.sum(axis = 1) + document_rectangle[0] = document_outline[np.argmin(sum_points)] + document_rectangle[2] = document_outline[np.argmax(sum_points)] + diff_points = np.diff(document_outline, axis = 1) + document_rectangle[1] = document_outline[np.argmin(diff_points)] + document_rectangle[3] = document_outline[np.argmax(diff_points)] + (top_left, top_right, bottom_right, bottom_left) = document_rectangle + + # let's compute the dimensions of the document in the image: + left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2)) + right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2)) + top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2)) + bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2)) + max_height = max(int(left_height), int(right_height)) + max_width = max(int(top_width), int(bottom_width)) + + # We compute the destination of the transform: + destination = np.array( + object = [ + [0, 0], + [max_width - 1, 0], + [max_width - 1, max_height - 1], + [0, max_height - 1] + ], + dtype = "float32" + ) + + # We apply the transform that flattens out the document: + matrix = cv2.getPerspectiveTransform(document_rectangle, destination) + flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height)) + + # Done here: + return flattened_image + + def extract_text( + self, + image, + min_confidence = 0.5, + margin = 0.05 + ): + + """ + To run OCR on the input image. + :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. + :param min_confidence: The minimum amount of confidence for detected text to be considered. + :param margin: The extra pixels to include when cropping into a section for OCR. + :return: The extracted text. + """ + + # Make a variable that will hold the final result: + extracted_text = {} + + # Open the image and read the data: + image = self.open_image(image) + height, width = image.shape[:2] + + # Run OCR on the entire canvas: + ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence] + extracted_text["fullDoc"] = " ".join(ocr_result) + + # Get the doc layout in the image: + document_sections = self.__yolo.predict(image) + document_sections = document_sections["boxes"] + + # For each section, we run the OCR process individually: + section_results = [] + for section in document_sections: + if section["class"] in self.__whitelisted_yolo_classes: + section_width = section["x2"] - section["x1"] + section_height = section["y2"] - section["y1"] + x1 = max(int(section["x1"] - (margin * section_width)), 0) + y1 = max(int(section["y1"] - (margin * section_height)), 0) + x2 = min(int(section["x2"] + (margin * section_width)), width) + y2 = min(int(section["y2"] + (margin * section_height)), height) + sub_image = image[y1:y2, x1:x2] + ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence] + section_results.append({ + "text": " ".join(ocr_result), + "class": section["class"], + "className": section["className"], + "x1": x1, + "y1": y1, + "x2": x2, + "y2": y2 + }) + extracted_text["bySection"] = section_results + + # Done here: + return extracted_text + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import time + + my_scanner = DocumentScanner( + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], + # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], + ocr_languages = ["en"] + ) + + image = my_scanner.open_image(r"/home/developer/Downloads/sushmita_card.jpg") + # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card.jpg") + # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card_2.jpg") + # image = my_scanner.open_image(r"/home/developer/Downloads/card_square.jpg") + document_image = my_scanner.scan(image) + if document_image is not None: + cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image) + start_time = time.time() + document_text = my_scanner.extract_text( + document_image, + min_confidence = 0.5 + ) + print("OCR RESULT:") + print(document_text) + + print(f"FINISHED IN {time.time() - start_time} SECONDS!") + + else: print("No image") diff --git a/mail/__init__.py b/mail/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mail/async_mail.py b/mail/async_mail.py new file mode 100644 index 0000000..839873c --- /dev/null +++ b/mail/async_mail.py @@ -0,0 +1,440 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 16th Jul, 2024 + + OBJECTIVE: + + To be able to send out mails from code. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For working with mails: +import aiosmtplib +from email.mime.multipart import MIMEMultipart +from email.mime.image import MIMEImage +from email.mime.text import MIMEText +from email.mime.base import MIMEBase +from email import encoders + +# My utils: +from utils import rate_limit_utils + +# Common: +from shared.statuses import StatusCodes + +# For random strings: +import string +import random + +# For system-level activities: +import os + +# For working with files in RAM: +import io + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class MailMessage: + + def __init__(self, to_email, subject): + + """ + Create an instance of the message that you would like to send. + :param to_email: The EMail ID of th recipient. + :param subject: The subject of the mail. + """ + + self.message = MIMEMultipart() + self.message["To"] = to_email + self.message["Subject"] = subject + + def add_text(self, text): + + """ + Add plain-text to the mail body. + :param text: The text to add to the mail body. + :return: None. + """ + + self.message.attach(MIMEText(text, "plain")) + + def add_html(self, html_text): + + """ + Add HTML text to the mail body. + :param html_text: The HTML text to add to the mail body. + :return: None. + """ + + self.message.attach(MIMEText(html_text, "html")) + + def add_inline_image(self, image_file, content_id = None): + + """ + Add an inline image to the body of the mail. + NOTE: This is NOT the same as sending an image as an attachment. + :param image_file: The image data to attach to the mail body. + :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not + specified, I will generate a random string. You may write a custom value here if you know what you are + doing. For most use cases, please ignore this field. + :return: None. + """ + + # Read the image as bytes: + image_bytes = None + if type(image_file) is str: + with open(image_file, "rb") as opened_image_file: + image_bytes = opened_image_file.read() + if type(image_file) is io.BytesIO: + image_file.seek(0) + image_bytes = image_file.getvalue() + + # Declare the part to be attached to the multipart message: + if image_bytes is not None: + + # Create the HTML block if the image pointer is blank: + if content_id is None: + content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) + self.add_html(f""" + + +

+ + + """) + + # Then add the image: + image_part = MIMEImage(image_bytes) + image_part.add_header("Content-ID", f"<{content_id}>") + self.message.attach(image_part) + + def add_attachment(self, attachment_file, file_name = None): + + # Declare the part to be attached to the multipart message: + part = MIMEBase("application", "octet-stream") + + # If the attachment is a file stored in the local disk: + if type(attachment_file) is str: + file_name = file_name or os.path.split(attachment_file)[-1] + with open(attachment_file, "rb") as attachment: + part.set_payload(attachment.read()) + + # If the file is held in RAM: + if type(attachment_file) is io.BytesIO(): + attachment_file.seek(0) + part.set_payload(attachment_file.read()) + + # Encode and attach the file: + encoders.encode_base64(part) + part.add_header( + "Content-Disposition", + f"attachment; filename= {file_name}", + ) + self.message.attach(part) + + def get_message(self): + return self.message + + +# --------------------------------------------------------------------------------------------------------------------- + + +class AsyncMailClient: + + # Constants: + SMTP_TLS_PORT = 587 + SMTP_SSL_PORT = 465 + + # variables: + __smtp = None + + def __init__( + self, + email, + password, + server, + port = 587, + rate_limiters = None, + wait_for_turn = True, + debug = True, + debug_prefix = "Mail (C) | " + ): + + """ + Set up the mail client. + :param email: The Email ID to use when sending out mails. + :param password: The password of the EMail ID that is being used. + :param server: The EMail server. + :param port: The port number to connect to the host. + :param rate_limiters: The rate limiters to use. Must have "get_turn" and "has_turn" methods. "get_turn" method + must wait for the turn, and "has_turn" method must only check if a turn is available. + :param wait_for_turn: To wait for turn if the rate limit has been exceeded, or to return with failure. + :param debug: Whether, or not, you want to print debugging messages. + :param debug_prefix: The prefix to identify the debugging messages. + """ + + # Initialize the debugger: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # Note down the credentials and other details: + self.__email = email + self.__password = password + self.__server = server + self.__port = port + self.__rate_limiters = rate_limiters if type(rate_limiters) is list else ([rate_limiters] if rate_limiters is not None else []) + self.__wait_for_turn = wait_for_turn + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + async def login(self): + + """ + To connect to the mail server and authenticate the user. + :return: True if authenticated, else False. + """ + + # Initialize the SMTP connection, + # and return with success if all goes well: + try: + self.__smtp = aiosmtplib.SMTP( + hostname = self.__server, + port = self.__port, + use_tls = False, + start_tls = False + ) + await self.__smtp.connect() + await self.__smtp.starttls() + await self.__smtp.login(self.__email, self.__password) + return True + + # Return with failure if something goes wrong: + except Exception as exception: + self.__printer(exception) + try: await self.__smtp.quit() + except Exception as exception: self.__printer(exception) + self.__smtp = None + return False + + async def ensure_connection(self): + + """ + Can be run before the sending operation to confirm that we are yet connected to the server. + If not connected, this code will reattempt to connect and log-in. + :return: True if connected, else False. + """ + + # If the login had failed initially, the object will be set to null. + # In such a case, we make an attempt to login: + if self.__smtp is None: + return await self.login() + + # If the login was successful, we check if the connection is active. + # If not, we try to re-login: + if self.__smtp.is_connected: + return True + else: + try: + await self.__smtp.connect() + await self.__smtp.starttls() + await self.__smtp.login(self.__email, self.__password) + return True + except Exception as exception: + self.__printer(exception) + try: await self.__smtp.quit() + except Exception as exception: self.__printer(exception) + self.__smtp = None + return False + + async def logout(self): + + """ + Closes the connection to the SMTP client. + :return: True by default. + """ + + if self.__smtp is not None: + try: await self.__smtp.quit() + except Exception as exception: self.__printer(exception) + self.__smtp = None + return True + + async def send(self, mail: MailMessage): + + """ + Send out the mail. + :param mail: The instance of 'MailMessage' with all the content populated. + :return: A dict with 'success' and 'message'. + """ + + # Return with failure if we aren't connected, + # and our attempt to (re)connect fails: + if not await self.ensure_connection(): + return { + "success": False, + "message": "login failed" + } + + # Comply with the rate-limit: + for rate_limiter in self.__rate_limiters: + if not self.__wait_for_turn: + if not await rate_limiter.has_turn(): return False + got_turn = await rate_limiter.get_turn() + if not got_turn: + return { + "success": False, + "message": "rate-limit wait timeout" + } + + # Try to send the message: + try: + mail.message["From"] = self.__email + await self.__smtp.send_message(mail.message) + return { + "success": True, + "message": "mail sent" + } + + # If something goes wrong: + except Exception as exception: + self.__printer(exception) + return { + "success": False, + "message": str(exception) + } + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + from utils_v2.string import json + from utils_v2.mail.mail_message import MailMessage + + async def test(): + + rate_lim = rate_limit_utils.TokenBucket( + rate_limit = 1, + seconds = 60.0, + ) + + mail_client = AsyncMailClient( + email = "sender@gmail.com", + password = "secret_password", + server = "smtp.gmail.com", + rate_limiters = rate_lim + ) + + my_mail = MailMessage( + to_email = "recipient@gmail.com", + subject = "Bhopli is the best!" + ) + my_mail.add_html( + """ + + + + + Sample HTML String + + + +

Hello, Bhopli!

Bhopli is the best, most well-behaved cat in the known universe.

Hello, Bhopli!

Bhopli is the best, most well-behaved cat in the known universe.

+ + + """ + ) + my_mail.add_text("This is how you should pet her 👇") + my_mail.add_inline_image(r"/path/to/image/cat_petting.png") + my_mail.add_attachment(r"/path/to/file/sample_label.pdf") + + await mail_client.login() + result = await mail_client.send(my_mail) + print("MAIL RESULT:", json_utils.to_json_string(result)) + await mail_client.logout() + + asyncio.run(test()) diff --git a/network/__init__.py b/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/network/ip.py b/network/ip.py new file mode 100644 index 0000000..134078f --- /dev/null +++ b/network/ip.py @@ -0,0 +1,173 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 12th Jul., 2024 + + OBJECTIVE: + + To provide an easy way to get geolocation information of an IP address. + + REFERENCES: + + 1) https://medium.com/@tubelwj/how-to-retrieve-ip-geolocation-information-in-python-929e15041e3e + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For working with IP Addresses: +import ipaddress + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def ipv4_to_int(ip_string): + + """ + Converts an IP (v4) string to an integer value. + :param ip_string: The IP address (v4) that you want to convert to integer format. + :return: An integer representation of the IP (v4) address. + """ + + ip_numerical = int(ipaddress.IPv4Address(ip_string)) + return ip_numerical + + +# --------------------------------------------------------------------------------------------------------------------- + + +def int_to_ipv4(ip_numerical): + + """ + Interprets the IP (v4) value from the given integer value. + :param ip_numerical: The integer value that represents an IP (v4) address. + :return: + """ + + ip_string = str(ipaddress.IPv4Address(ip_numerical)) + return ip_string + + +# --------------------------------------------------------------------------------------------------------------------- + + +def ipv4_to_bin(ip_string): + + """ + Converts an input IP (v4) address to the binary string that represents the 32 bits. + :param ip_string: The IP (v4) string in a format like "192.168.0.1" + :return: The binary representation (as a string) of the input IP address. + """ + + ip_binary = bin(int(ipaddress.IPv4Address(ip_string)))[2:].zfill(32) + return ip_binary + + +# --------------------------------------------------------------------------------------------------------------------- + + +def bin_to_ipv4(ip_binary): + + """ + Interprets the IP (v4) value from the given binary string. + :param ip_binary: The string of 1s and 0s that represents the IP (v4) address. + :return: The IP (v4) address as a string. + """ + + ip_string = str(ipaddress.IPv4Address(int(ip_binary, 2))) + return ip_string + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_ipv4_range(ip_string, as_string = True): + + """ + Given a network description in the format "88.95.100.128/25", this function tells you the first and last IP + addresses of that network. Useful for determining if an IP address lies in a network. + :param ip_string: The input network description in the format "88.95.100.128/25" + :param as_string: To select between integer and string formats for the IP range output. + :return: The first and last IP addresses of the input network, and the count. + """ + + # Extract the components of the string: + ip_components = ip_string.split("/") + ip_addr = ipv4_to_int(ip_components[0]) + ip_bits = int(ip_components[1]) + + # Convert the mask number to binary representation: + ip_mask = (1 << ip_bits) - 1 + ip_mask = ip_mask << (32 - ip_bits) + inv_ip_mask = (~ip_mask) & 0xFFFF + + # Figure out the start and end IP addresses: + start_ip = ip_addr & ip_mask + end_ip = ip_addr | inv_ip_mask + count = end_ip - start_ip + 1 + + # If the IPs are needed as strings, we perform the conversion: + if as_string: + start_ip = int_to_ipv4(start_ip) + end_ip = int_to_ipv4(end_ip) + + # Done here: + return start_ip, end_ip, count + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + print(ipv4_to_int("255.255.255.255")) + print(ipv4_to_int("x.x.x.x")) diff --git a/network/ping.py b/network/ping.py new file mode 100644 index 0000000..39e42c7 --- /dev/null +++ b/network/ping.py @@ -0,0 +1,218 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 28th Jun, 2024 + + OBJECTIVE: + + To provide a way to ping a server and get the traceroute dump. + + REFERENCES: + + 1) https://www.geeksforgeeks.org/traceroute-implementation-on-python/ + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +# --- +import sys +sys.path.append(".") +sys.path.append("..") + +# Utils: +# --- +from utils import json_utils +from utils import time_utils +from utils import regex_utils + +# For networking: +# --- +import socket +from scapy.all import * + +# For running the script from the terminal: +# --- +import argparse + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_name_and_addr(destination): + + # If the user provided the IP address: + # --- + if ( + regex_utils.match(destination, regex_utils.REGEX_IPV4) or + regex_utils.match(destination, regex_utils.REGEX_IPV6) + ): + try: destination_name = socket.gethostbyaddr(destination)[0] + except Exception as exception: destination_name = "*" + destination_ip = destination + + # If the provided destination was the domain name: + # --- + else: + destination_name = destination + try: destination_ip = socket.gethostbyname(destination) + except Exception as exception: destination_ip = "*" + + # Done here: + # --- + return destination_name, destination_ip + + +# --------------------------------------------------------------------------------------------------------------------- + + +def tracert( + destination, + max_hops = 30, + timeout = 2.0, + port = 33434 +): + + # Initialize the variables: + # --- + destination_name, destination_ip = get_name_and_addr(destination) + full_trace = [] + ttl = 1 + + # Keep noting hops till the limit is reached: + # --- + while ttl <= max_hops: + + # Create a JSON for this stage: + # --- + this_hop = { + "destAddr": destination_ip, + "destName": destination_name, + "hopNo": ttl - 1, + "isDest": False, + "hopAddr": None, + "hopName": None, + "ts": None + } + + # Create the IP and UDP headers and combine them: + # --- + ip_packet = IP(dst = destination, ttl = ttl) + udp_packet = UDP(dport = port) + trace_packet = ip_packet / udp_packet + + # Send the packet and receive a reply and note down the timestamp: + # --- + reply = sr1(trace_packet, timeout = timeout, verbose = 0) + this_hop["ts"] = time_utils.get_current_utc_datetime(as_string = True) + + # No response: + # --- + if reply is None: this_hop["hopAddr"] = this_hop["hopName"] = "*" + + # If some response was received, we note the values and break out if this was the destination hop: + # --- + else: + this_hop["hopName"], this_hop["hopAddr"] = get_name_and_addr(f"{reply.src}") + if reply.type == 3: + this_hop["isDest"] = True + full_trace.append(this_hop) + break + + # Carry on to the next hop: + # --- + full_trace.append(this_hop) + ttl += 1 + + # Done here: + # --- + return full_trace + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description = "Traceroute Implementation in Python!") + + parser.add_argument( + "dest", + help = "Destination (Name or IP address)." + ) + parser.add_argument( + "-m", + "--max-hops", + type = int, + default = 30, + help = + "Maximum number of hops (default: 30)." + ) + parser.add_argument( + "-t", + "--timeout", + type = float, + default = 2.0, + help = "Timeout for each packet in seconds (default: 2.0)." + ) + parser.add_argument( + "-p", + "--port", + type = int, + default = 33434, + help = "Timeout for each packet in seconds (default: 33434)." + ) + + args = parser.parse_args() + + trace = tracert( + destination = args.dest, + max_hops = args.max_hops, + timeout = args.timeout, + port = args.port + ) + print("TRACE:") + print(json_utils.to_json_string(trace)) diff --git a/pdf/__init__.py b/pdf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pdf/pdf_maker.py b/pdf/pdf_maker.py new file mode 100644 index 0000000..5b42c79 --- /dev/null +++ b/pdf/pdf_maker.py @@ -0,0 +1,1570 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 13th Aug., 2024 + + OBJECTIVE: + + To generate custom PDFs. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For conversion to PDF: +from PyPDF2 import PdfMerger, PdfReader, PdfWriter, Transformation + +# For barcode and QR code generation: +import xml.etree.ElementTree as ET +from barcode import Code128 +from barcode.writer import ImageWriter, SVGWriter +import segno +from PIL import Image, ExifTags + +# System-level activities: +import os +import io +import inspect +import contextlib + +# For calculations: +import math + +# For PDF generation: +import fitz +from reportlab.pdfgen import canvas +from reportlab.lib.units import mm, cm, inch +from reportlab.lib.styles import ParagraphStyle +from reportlab.platypus import Paragraph +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.lib.utils import ImageReader +from reportlab.graphics import renderSVG + +# My utils: +from utils_v2.string import json +from utils_v2.system import files +from utils_v2.string import regex + +# For debugging: +from icecream import IceCreamDebugger +import traceback + +# For random string generation: +import random +import string + +# To validate the font file: +from fontTools import ttLib + +# To make http calls: +import requests +import httpx + +# To work with base64 coding: +import base64 + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make async API calls: +max_connections = 5 +limits = httpx.Limits( + max_keepalive_connections = max_connections, + max_connections = max_connections, + keepalive_expiry = 3600 +) +http_client = httpx.AsyncClient( + limits = limits, + follow_redirects = True +) + +# Headers for downloading files: +HEADERS_DOWNLOAD = { + "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0", + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate" +} + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class PDFMaker: + + def __init__( + self, + width, + height, + unit = "mm", + dpi = 72, + debug = True, + debug_prefix = "PDF (M) | ", + raise_exception = True, + **kwargs + ): + + """ + IMPORTANT: To keep things intuitively oriented (left-to-right, and top-to-bottom), the internal bottom-up + coordinates have been flipped mathematically. All measurements start from the top-left corner. Moreover, I have + ensured that all numbers follow the same units. So, if you set your unit to "inch", everything will be measured + in the same unit. I hope this creates maximum consistency for you. + A NOTE ON IMAGES: All images will be measured in pixels when operating on them directly (like cropping and + resizing), but will be measured in the 'unit' of the canvas when placing it in the PDF. + A NOTE OF FONT SIZES: Font sizes (and their line spacings) are measured in 'points'. As per the standard, there + are 72 'points' in one inch. While everything else in this utility uses the unit of measurement defined in the + 'unit' param, fonts need to be specified in 'points' to maintain compatibility with HTML rendering engines to + work with well-formatted paragraphs. + WARNING: I STRONGLY RECOMMEND NOT USING DPI AS A SETTING. THE PDF FORMAT ITSELF DOESN'T HAVE ANY PROVISION FOR + PIXEL DENSITY. THIS SETTING HAS BEEN MADE AVAILABLE FOR PLACEBO EFFECT WHEN CLIENTS ASK FOR VERY HIGH RESOLUTION + FILES WITHOUT WANTING TO UNDERSTAND ANY TECHNICALITY. USE WITH CAUTION. + :param width: [int|float] The width of the page (in the specified unit). + :param height: [int|float] The height of the page (in the specified unit). + :param unit: [str] The unit ("mm", "cm", "inch") to use. + :param dpi: [int|float] The pixel density. AVOID USING THIS. + :param debug: [bool] Whether, or not, you want to print debugging messages. Can be changed on the fly. + :param debug_prefix: [str] The prefix to show in debugging messages. + :param raise_exception: If set to True, any failure will immediately stop the PDF building process. If set to + False, the PDF building will continue by ignoring failed steps. Set to True by default for an all-or-nothing + output behaviour. + :param kwargs: Just a placeholder, does nothing for now (20240904). + """ + + # For debugging: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # Note down the exception-handling choice: + self.__raise_exception = raise_exception + + # We pick the unit from the user's choice: + self.__unit = { + "mm": mm, + "cm": cm, + "inch": inch + }[unit] + + # To know the canvas size multipliers, we need a few parameters: + # 01. The DPI Factor is the multiplier to apply to achieve the target DPI considering a default DPI of 72, + # 02. The Unit Factor is the multiplier to apply to the unit to achieve a high-res result. + self.__dpi_factor = (dpi / 72.0) + self.__unit_factor = self.__unit * self.__dpi_factor + + # Now we create the canvas: + self.__pdf_in_ram = io.BytesIO() + self.__canvas_width = width * self.__unit_factor + self.__canvas_height = height * self.__unit_factor + self.__canvas = canvas.Canvas( + self.__pdf_in_ram, + pagesize = (self.__canvas_width, self.__canvas_height) + ) + + def enable_debug(self): + + """ + To enable the debugging text in the terminal. + :return: None. + """ + + self.__printer.enable() + + def disable_debug(self): + + """ + To disable the debugging text in the terminal. + :return: None. + """ + + self.__printer.disable() + + # ┏┓┓ ┓ ┳┳ •┓• • + # ┗┓┣┓┏┓┏┓┏┓┏┫ ┃┃╋┓┃┓╋┓┏┓┏ + # ┗┛┛┗┗┻┛ ┗ ┗┻ ┗┛┗┗┗┗┗┗┗ ┛ + + @staticmethod + async def download_from_url(url, follow_redirects = False): + + """ + Download a file from a URL. + :param url: The URL to download the file from. + :param follow_redirects: Whether, or not, to follow along with any redirects when downloading the file. + :return: The downloaded file or None. + """ + + file = io.BytesIO((await http_client.get( + url, + headers = HEADERS_DOWNLOAD, + follow_redirects = follow_redirects + )).content) + file.seek(0) + return file + + @staticmethod + def read_to_ram(file_path): + + """ + Reads a file into a BytesIO object in RAM. + NOTE: The method doesn't itself verify if the file is a PDF. + :param file_path: The path to the file on disk. + :return: The file in a BytesIO object. + """ + + with open(file_path, "rb") as file: + file_data = file.read() + file_in_ram = io.BytesIO(file_data) + file_in_ram.seek(0) + return file_in_ram + + def __parse_color(self, input_color): + + """ + The reportlab 'reportlab' needs the colour to be specified in an array of RGS values where each value is a + number in the range 0-1. + :param input_color: Either a hex string or an array of numbers. + :return: An array that can be used by 'reportlab'. + """ + + # Start by assuming failure: + processed_color = None + + # If the input color is a string, we convert from hex to an array of hex codes: + if isinstance(input_color, str): + input_color = input_color.lstrip("#") + color_tuple = ( + input_color[0:2].upper(), + input_color[2:4].upper(), + input_color[4:6].upper() + ) + input_color = color_tuple + + # If the input color is a list/tuple of hex-strings: + if isinstance(input_color, (list, tuple)): + try: + if all(isinstance(item, str) for item in input_color): + processed_color = [int(item, 16) / 255.0 for item in input_color] + elif all(isinstance(item, (float, int)) for item in input_color): + processed_color = [max(min(item, 255.0), 0.0) / 255.0 for item in input_color] + except: + if self.__raise_exception: raise + + # Done here: + return processed_color + + def __parse_x(self, *args): + + """ + To adjust the horizontal coordinates for the canvas. + :param args: As many horizontal coordinates as you would like to adjust. + :return: The adjusted coordinates. + """ + + if len(args) == 0: return None + parsed_x = [x * self.__unit_factor for x in args] + parsed_x = [x if x >= 0 else x + self.__canvas_width for x in parsed_x] + return tuple(parsed_x) if len(parsed_x) > 1 else parsed_x[0] + + def __parse_y(self, *args, flip = False): + + """ + To adjust the vertical coordinates for the canvas. + :param args: As many vertical coordinates as you would like to adjust. + :param flip: To convert from bottom-up system to top-down system. + :return: The adjusted coordinates. + """ + + if len(args) == 0: return None + parsed_y = [y * self.__unit_factor for y in args] + parsed_y = [y if y >= 0 else y + self.__canvas_height for y in parsed_y] + if flip: parsed_y = [self.__canvas_height - y for y in parsed_y] + return tuple(parsed_y) if len(parsed_y) > 1 else parsed_y[0] + + def __parse_font_size(self, *args): + + """ + Adjust the font size. + :param args: The font sizes to adjust + :return: The adjusted font-size. + """ + + if len(args) == 0: return None + parsed_font_sizes = args + return parsed_font_sizes if len(parsed_font_sizes) > 1 else parsed_font_sizes[0] + + @staticmethod + def _parse_coordinates_for_rotation( + original_x, + original_y, + angle + ): + + """ + Adjusts the x and y coordinates to compensate for the effect of a rotation operation. + :param original_x: The x coordinate before applying rotation. + :param original_y: The y coordinate before applying rotation. + :param angle: The angle (in degrees) by which the rotation will be applied. + :return: The adjusted x and y coordinates that compensate for the effect of the canvas's rotation. + """ + + # Get the angle in radians: + angle_radians = math.radians(angle) + angle_cos = math.cos(angle_radians) + angle_sin = math.sin(angle_radians) + + # Compute the inverse rotation: + adjusted_x = original_x * angle_cos + original_y * angle_sin + adjusted_y = -original_x * angle_sin + original_y * angle_cos + + # Done here: + return adjusted_x, adjusted_y + + # ┳ ┏┓ • + # ┃┏┓┏╋┏┓┏┓┏┏┓ ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏ + # ┻┛┗┛┗┗┻┛┗┗┗ ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛ + # ┛ + + @property + def canvas(self): + + """ + Use this to fetch just the canvas with all the inputs applied to it. + :return: The canvas (page) that is being used currently in the instance. + """ + + return self.__canvas + + @property + def unit_factor(self): + + """ + The scaling factor after taking into consideration the physical unit of measurement being use (e.g.: "mm"). + :return: The unit-factor. + """ + + return self.__unit_factor + + @property + def dpi_factor(self): + + """ + The scaling factor derived from comparing the default DPI of 72 and a custom DPI specified when creating the + instance. Since the PDF format doesn't have a native DPI configuration, this scaling factor will help in + adjusting the sizes of whatever you wish to add to the canvas. + :return: The DPI scaling factor. + """ + + return self.__dpi_factor + + @property + def shape(self): + + """ + an array of the dimensions (width, height) of the canvas. + :return: The width and height of the canvas. + """ + + return self.__canvas_width, self.__canvas_height + + # ┏┓ ┏┓ ┏┓•┓ + # ┃ ┏┓┏┓┓┏┏┓┏ ┣╋ ┣ ┓┃┏┓ + # ┗┛┗┻┛┗┗┛┗┻┛ ┗┻ ┻ ┗┗┗ + + def save(self, path: str = None): + + """ + Save the Canvas as a PDF either to RAM or a file on disk. + :param path: The path you want to save the PDF to on disk. If not specified, the PDF will be saved to RAM and + returned as a BytesIO object. + :return: A BytesIO object, or True/False if a path is supplied and the file gets saved. + """ + + # Save the changes to the file in RAM. + self.__canvas.save() + self.__pdf_in_ram.seek(0) + + # If the user has not specified a path, return the buffer: + if path is None: return self.__pdf_in_ram + + # In case a path was specified, + # We try to save the file in the specified path: + try: + with open(path, "wb") as file: file.write(self.__pdf_in_ram.getvalue()) + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def next_page(self): + + """ + To start working on the next page. + The library that is being used doesn't support moving back and forth between pages in any random order, it, + instead, needs to go in a sequential fashion from one page to the next. + :return: None. + """ + + self.__canvas.showPage() + + # ┏┓ + # ┣ ┏┓┏┓╋┏ + # ┻ ┗┛┛┗┗┛ + + @staticmethod + async def register_font_from_url(name, url, follow_redirects = False): + + """ + Download a font from a URL and then register it for use. + :param name: The name you would later refer to the font by. + :param url: The URL to download the font from. + :param follow_redirects: Whether, or not, to follow along with any redirects when downloading the file. + :return: True if registered, else False. + """ + + try: return PDFMaker.register_font( + name, + await PDFMaker.download_from_url(url, follow_redirects = follow_redirects) + ) + except Exception as exception: + return False + + @staticmethod + def register_fonts_from_directory(directory): + + """ + Register all the fonts from a directory to be used later in the PDF. + :param directory: The directory that has the TTF files to register. + :return: A dict describing the fonts that were registered. + """ + + # List out all the files in the folder: + all_files = files.list_files(directory, full_path = True) + + # Keep only TTF files and make a name-to-path map: + font_map = { + os.path.split(file)[-1].split(".")[0].lower(): file + for file in all_files if file.lower().rstrip().endswith(".ttf") + } + + # Register the custom fonts: + registered_fonts = [] + for font_name, font_path in font_map.items(): + if PDFMaker.register_font(font_name, font_path): + registered_fonts.append({"name": font_name, "file": font_path}) + + # Done here: + return registered_fonts + + @staticmethod + def register_font(name, file): + + """ + Register one TTF font from either a BytesIO object or a path on the disk. + :param name: The name of the font as you would like to use when invoking it. + :param file: The font file (TTF format). + :return: True if registered, else False. + """ + + try: + + if isinstance(file, io.BytesIO): file.seek(0) + pdfmetrics.registerFont(TTFont(name, file)) + return True + + except Exception as exception: + return False + + @staticmethod + def get_font_height(font, size, count_ascent = True, count_descent = True): + + """ + Compute the height of the font for the given size. + :param font: [str] The name of the font (as registered). + :param size: [int|float] The size of the font (in points). + :param count_ascent: [bool] Whether, or not, you would like to consider the ascent of the font. + :param count_descent: [bool] Whether, or not, you would like to consider the descent of the font. + :return: The height of the font in points. + """ + + # Get font's information and compute the height: + ascent, descent = pdfmetrics.getAscentDescent(font, size) + if not count_ascent: ascent = 0.0 + if not count_descent: descent = 0.0 + height = ascent - descent + return height + + @staticmethod + def font_is_registered(font): + + """ + Checks if a particular font is available for use. + :param font: [str] The name of the font to check. + :return: True if registered, else False + """ + + return font in pdfmetrics.getRegisteredFontNames() + + @staticmethod + def get_first_registered_font(): + + """ + Returns the name of the first registered font. Useful for cases like those when your font fails to load and you + need a default to fall back on. + :return: The name of the first font that has been registered. + """ + + return pdfmetrics.getRegisteredFontNames()[0] + + @staticmethod + def list_registered_fonts(): + + """ + Returns the list of names of all the registered fonts. + :return: The list of names of the fonts that have been registered. + """ + + return pdfmetrics.getRegisteredFontNames() + + @staticmethod + def is_valid_ttf(file): + + """ + Checks if a font file is valid and usable or not. + :param file: [str|io.BytesIO] The file to check. + :return: True if valid, else False. + """ + + if isinstance(file, io.BytesIO): file.seek(0) + try: font = ttLib.TTFont(file)["head"].fontRevision + except Exception as exception: return False + return True + + # ┳ + # ┃┏┳┓┏┓┏┓┏┓┏ + # ┻┛┗┗┗┻┗┫┗ ┛ + # ┛ + + @staticmethod + def make_barcode_image( + data, + bar_width = 2, + bar_height = 125, + border = 10, + bar_color = "#000000", + background_color = "#FFFFFF", + dpi = 300, + format = "png", + as_pil = True + ): + + """ + Makes a barcode in Code128 format. + :param data: [str] The data to encode in the barcode. + :param bar_width: [int] The base bar-module width in pixels. + :param bar_height: [int] The bar-module height in pixels. + :param border: [int] The size of the border around all sides of the barcode in pixels. + :param bar_color: [str] The colour of the bars as a hex string. + :param background_color: [str] The colour of the background as a hex string. + :param dpi: [int] The pixel density to note in the image. Leave it to default for most use cases. + :param format: [str] The file format to save the image in. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :return: Either a PIL object or a PNG file in RAM. + """ + + # Create the barcode in RAM: + barcode_in_ram = io.BytesIO() + barcode_options = { + "module_width": (bar_width / dpi) * 25.4, + "module_height": (bar_height / dpi) * 25.4, + "quiet_zone": 0, + "foreground": bar_color, + "background": background_color, + "format": format, + "dpi": dpi + } + Code128(str(data), writer = ImageWriter()).write(barcode_in_ram, options = barcode_options) + barcode_in_ram.seek(0) + + # Remove the rasterized text at the bottom, and create a border of the desired size: + barcode_image = Image.open(barcode_in_ram, formats = [format]) + width, height = barcode_image.size + barcode_image = barcode_image.crop(( + 0, + int(dpi * 0.0393701), + width, + int(dpi * 0.0393701) + bar_height + )) + width, height = barcode_image.size + background_image = Image.new( + mode = "RGB", + size = (width + border + border, height + border + border), + color = background_color + ) + background_image.paste(barcode_image, (border, border)) + barcode_image = background_image + + # If the user has asked for a PIL object: + if as_pil: return barcode_image + + # Else, ave the PNG in ram: + barcode_in_ram = io.BytesIO() + barcode_image.save(barcode_in_ram, format = "png") + barcode_in_ram.seek(0) + + # Done: + return barcode_in_ram + + @staticmethod + def make_qr_image( + data, + scale = 5, + border = 2, + foreground_color = "#000000", + background_color = "#FFFFFF", + border_color = "#FFFFFF", + background_art = None, + format = "png", + as_pil = True + ): + + """ + Create a QR code. + :param data: [str] The data that you want to encode in the QR code. + :param scale: [int] The size (in pixels) for each tiny block in the QR code. + :param border: [int] The size (in multiples of scale) of the quiet zone. + :param foreground_color: [str] The color of the foreground, a.k.a. the dark color. + :param background_color: [str] The color of the background, a.k.a. the light color. + :param border_color: [str] The color of the quiet zone. + :param background_art: [io.BytesIO|PIL.Image] Any image that you would like to feed into the background instead + of a plain QR code. This can be an animated GIF image as well. + :param format: [str] The type of output file that you want. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :return: Either a PIL object or a PNG file in RAM. + """ + + qr_code = segno.make_qr(str(data)) + params = { + "scale": scale, + "border": border, + "dark": foreground_color, + "light": background_color, + "quiet_zone": border_color, + "kind": format + } + + qr_image = io.BytesIO() + + if background_art is not None: + if isinstance(background_art, Image.Image): + buffer = io.BytesIO() + background_art.save(buffer, format = format) + buffer.seek(0) + background_art = buffer + params["background"] = background_art + params["target"] = qr_image + qr_code.to_artistic(**params) + + else: + params["out"] = qr_image + qr_code.save(**params) + + qr_image.seek(0) + if as_pil: qr_image = Image.open(qr_image, formats = [format]) + return qr_image + + async def image_from_url(self, url, follow_redirects = False, as_pil = True, format = "png"): + + """ + Asynchronously downloads an image and returns it as a PIL object. + Use this instead of just passing the URL to 'draw_image' for better efficiency. + :param url: [str] The URL to download the image from. + :param follow_redirects: [bool] Whether, or not, to follow redirect URLs when downloading the file. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :param format: [str] The type of output file that you want. Not applicable for PIL objects. + :return: The image as a PIL object or as a file in RAM. + """ + + try: + + pil_image = Image.open(await PDFMaker.download_from_url(url, follow_redirects = follow_redirects)) + if as_pil: return pil_image + image_in_memory = io.BytesIO() + pil_image.save(image_in_memory, format = format) + image_in_memory.seek(0) + return image_in_memory + + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + + def image_from_base64(self, data, as_pil = True): + + """ + Converts a base64 string to an image. + :param data: [str] The base-64 representation of the image data. + :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). + :return: The image as a PIL object or as a file in RAM. + """ + + try: + + image = io.BytesIO(base64.b64decode(data)) + image.seek(0) + if not as_pil: return image + image = Image.open(image) + return image + + except Exception as exception: + if self.__raise_exception: raise + + @staticmethod + def crop_image_to_aspect_ratio(image, target_aspect_ratio, anchor = "c"): + + """ + Shaves off pixels from an image to achieve the target aspect ratio. + :param image: [PIL.Image] The image to work on, held in a PIL object. + :param target_aspect_ratio: [int|float] The aspect ratio you want to achieve. + :param anchor: [str] Which part of the image you want to retain. Imagine it like a compass - "nw", "n", "ne", + "e", "se", "s", "sw", "w", and "c". + :return: The adjusted image. + """ + + # Get the existing figures: + width, height = image.size + original_aspect_ratio = width / height + + # Compute the new dimensions: + if original_aspect_ratio > target_aspect_ratio: + scaling_factor = target_aspect_ratio / original_aspect_ratio + new_width, new_height = int(width * scaling_factor), height + else: + scaling_factor = original_aspect_ratio / target_aspect_ratio + new_width, new_height = width, int(height * scaling_factor) + + # Now we figure out the start and end points of cropping: + match anchor.strip().lower(): + case "nw": + start_x = 0 + start_y = 0 + case "n": + start_x = int(abs(new_width - width) / 2.0) + start_y = 0 + case "ne": + start_x = int(abs(new_width - width)) + start_y = 0 + case "e": + start_x = int(abs(new_width - width)) + start_y = int(abs(new_height - height) / 2.0) + case "se": + start_x = int(abs(new_width - width)) + start_y = int(abs(new_height - height)) + case "s": + start_x = int(abs(new_width - width) / 2.0) + start_y = int(abs(new_height - height)) + case "sw": + start_x = 0 + start_y = int(abs(new_height - height)) + case "w": + start_x = 0 + start_y = int(abs(new_height - height) / 2.0) + case _: + start_x = int(abs(new_width - width) / 2.0) + start_y = int(abs(new_height - height) / 2.0) + + end_x = start_x + new_width + end_y = start_y + new_height + + # And we finally crop and return the image: + return image.crop((start_x, start_y, end_x, end_y)) + + @staticmethod + def crop_image(image, start_x, start_y, end_x, end_y): + + """ + Crops an image. + :param image: [PIL.Image] The image to work on, held in a PIL object. + :param start_x: [int|float] The left coordinate (in the same unit as the dimension of the canvas). + :param start_y: [int|float] The top coordinate (in the same unit as the dimension of the canvas). + :param end_x: [int|float] The right coordinate (in the same unit as the dimension of the canvas). + :param end_y: [int|float] The bottom coordinate (in the same unit as the dimension of the canvas). + :return: The cropped image. + """ + + return image.crop((start_x, start_y, end_x, end_y)) + + @staticmethod + def resize_image_to_aspect_ratio(image, target_aspect_ratio): + + """ + Resizes the image (by stretching and squishing) till it fits the target aspect ratio. + :param image: [PIL.Image] The image to adjust. + :param target_aspect_ratio: [int|float] The aspect ratio that you want to achieve. + :return: The adjusted image. + """ + + # Get the existing figures: + width, height = image.size + original_aspect_ratio = width / height + + # Compute the new dimensions: + if original_aspect_ratio > target_aspect_ratio: + new_width, new_height = width, int(width / target_aspect_ratio) + else: + new_width, new_height = int(height * target_aspect_ratio), height + + # Resize and return the image: + return image.resize((new_width, new_height)) + + @staticmethod + def resize_image(image, width, height): + + """ + Resizes an image to the specified width and height. + :param image: [PIL.Image] The image as a PIL object. + :param width: [int|float] The new width. + :param height: [int|float] The new height. + :return: The resized image. + """ + + return image.resize((width, height)) + + @staticmethod + def adjust_image_orientation_from_exif(image): + + """ + Rotates the image as per the instructions in the EXIF tags. + :param image: The PIL object that holds the image data. + :return: The adjusted image. + """ + + # Guard clause to return if the attribute is not even found: + if not hasattr(image, "getexif"): return image + + # Else we start extracting EXIF data: + exif = image.getexif() + if exif is None: return image + + # We look for the tag that indicates orientation: + # ORIENTATION CODES: + # 1: Normal (Landscape) + # 2: Flipped horizontally + # 3: Upside down (Reverse Landscape) + # 4: Flipped vertically + # 5: Rotated 90° clockwise and flipped horizontally + # 6: Rotated 90° clockwise (Portrait) + # 7: Rotated 90° counter-clockwise and flipped horizontally + # 8: Rotated 90° counter-clockwise (Reverse Portrait) + orientation = exif.get(274, 0) + if orientation == 1: pass + elif orientation == 2: image = image.transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 3: image = image.rotate(180, expand = True) + elif orientation == 4: image = image.rotate(180, expand = True).transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 5: image = image.rotate(270, expand = True).transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 6: image = image.rotate(270, expand = True) + elif orientation == 7: image = image.rotate(90, expand = True).transpose(Image.FLIP_LEFT_RIGHT) + elif orientation == 8: image = image.rotate(90, expand = True) + + # Done here: + return image + + # ┏┓┳┓┏ + # ┃┃┃┃╋ + # ┣┛┻┛┛ + + def draw_line( + self, + start_x, + start_y, + end_x, + end_y, + thickness = 1.0, + color = (0, 0, 0), + alpha = 1.0 + ): + + """ + Draw a line in the PDF page. + :param start_x: [int|float] The left coordinate. + :param start_y: [int|float] The top coordinate. + :param end_x: [int|float] The right coordinate. + :param end_y: [int|float] The bottom coordinate. + :param thickness: [int|float] The thickness of the line (in the same unit as the canvas's dimensions). + :param color: [str] The RGB color in an array. + :param alpha: [int|float] The opacity of the line. 1 is 100% and 0 is 0% + :return: True or False based on the success of the operation. + """ + + # Handle color: + color = self.__parse_color(color) + + # Adjust the dimensions: + thickness *= self.__unit_factor + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = True) + + try: + + self.__canvas.setLineWidth(thickness) + self.__canvas.setStrokeColorRGB(color[0], color[1], color[2], alpha) + self.__canvas.line( + start_x, + start_y, + end_x, + end_y + ) + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def draw_circle( + self, + x, + y, + radius, + fill_color = (255, 255, 255), + fill_alpha = 0.0, + stroke_color = (0, 0, 0), + stroke_alpha = 1.0, + thickness = 1.0 + ): + + """ + Draw a circle in the PDF page. + :param x: [int|float] The horizontal coordinate of the center of the circle. + :param y: [int|float] The vertical coordinate of the center of the circle. + :param radius: [int|float] The radius of the circle. + :param fill_color: [str] The color-array (0-255) or hex string to define the inner fill color of the rectangle. + :param fill_alpha: [int|float] The opacity of the inner fill color. + :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. + :param stroke_alpha: [int|float] The opacity of the outline color. + :param thickness: [int|float] The thickness of the line (in the same unit as the canvas's dimensions). + :return: True or False based on the success of the operation. + """ + + try: + + # Handle color: + fill_color = self.__parse_color(fill_color) + stroke_color = self.__parse_color(stroke_color) + + # Adjust the dimensions: + x = self.__parse_x(x) + y = self.__parse_y(y, flip = True) + thickness *= self.__unit_factor + radius *= self.__unit_factor + + # Draw the circle here: + self.__canvas.setLineWidth(thickness) + self.__canvas.setFillColorRGB(fill_color[0], fill_color[1], fill_color[2]) + self.__canvas.setFillAlpha(fill_alpha) + self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) + self.__canvas.setStrokeAlpha(stroke_alpha) + self.__canvas.circle(x, y, radius, stroke = 1, fill = 1) + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def draw_rect( + self, + start_x, + start_y, + end_x, + end_y, + radius = 0.0, + fill_color = (255, 255, 255), + fill_alpha = 0.0, + stroke_color = (0, 0, 0), + stroke_alpha = 1.0, + thickness = 1.0, + ): + + """ + Draws a rectangle on the page. + :param start_x: [int|float] The left coordinate. + :param start_y: [int|float] The top coordinate. + :param end_x: [int|float] The right coordinate. + :param end_y: [int|float] The bottom coordinate. + :param radius: [int|float] The corner radius. + :param fill_color: [str] The color-array (0-255) or hex string to define the inner fill color of the rectangle. + :param fill_alpha: [int|float] The opacity of the inner fill color. + :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. + :param stroke_alpha: [int|float] The opacity of the outline color. + :param thickness: [int|float] The line thickness of the outline (in the same unit as the canvas's dimensions). + :return: True or False based on the success of the operation. + """ + + try: + + # Handle color: + fill_color = self.__parse_color(fill_color) + stroke_color = self.__parse_color(stroke_color) + + # Adjust the dimensions: + thickness *= self.__unit_factor + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = False) + + # Compute the width and height desired by the user: + width = end_x - start_x + height = end_y - start_y + + # Draw the rectangle here: + self.__canvas.setLineWidth(thickness) + self.__canvas.setFillColorRGB(fill_color[0], fill_color[1], fill_color[2]) + self.__canvas.setFillAlpha(fill_alpha) + self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) + self.__canvas.setStrokeAlpha(stroke_alpha) + self.__canvas.roundRect( + start_x, + self.__canvas_height - start_y - height, + width, + height, + radius * self.__unit_factor, + stroke = 1, + fill = 1 + ) + + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def make_paragraph_style( + self, + font_name = "Helvetica", + font_size = 12, + font_color = (0, 0, 0), + line_spacing = 12, + border_size = 0.0, + border_color = (255, 255, 255), + align = "left", + style_name = "CustomStyle" + ): + + """ + Creates a text style for using in a paragraph. + :param font_name: [str] The name of the font. Has to be registered. + :param font_size: [int|float] The size that you want the font to be printed in. + :param font_color: [str] The color-array (0-255) or hex string that you want the font to be printed in. + :param line_spacing: [int|float] The line spacing to apply. + :param border_size: [int|float] The thickness of the border. + :param border_color: [str] The color-array (0-255) or hex string of the color you wan the border to be in. + :param align: [str] Text alignment ("left", "center", or "right"). + :param style_name: [str] A name that you would like to give this style. + :return: The paragraph style. + """ + + # Make adjustments: + font_color = self.__parse_color(font_color) + font_size = self.__parse_font_size(font_size) + border_color = self.__parse_color(border_color) + border_size *= self.__unit_factor + + return ParagraphStyle( + name = style_name, + fontName = font_name, + fontSize = font_size, + textColor = (font_color[0], font_color[1], font_color[2]), + alignment = { + "left": 0, + "center": 1, + "right": 2 + }[align], + leading = line_spacing * font_size, + spaceBefore = 0, + spaceAfter = 0, + borderWidth = border_size, + borderColor = border_color + ) + + def write_paragraph( + self, + text, + start_x, + start_y, + end_x, + end_y, + alpha = 1.0, + style = None + ): + + """ + To write text into a bounding box in the PDF. + This can be either plaintext or an HTML string. + :param text: [str] The text to type in the bounding box. + :param start_x: [int|float] The left coordinate (in the same unit as used while setting up the canvas). + :param start_y: [int|float] The top coordinate (in the same unit as used while setting up the canvas). + :param end_x: [int|float] The right coordinate (in the same unit as used while setting up the canvas). + :param end_y: [int|float] The bottom coordinate (in the same unit as used while setting up the canvas). + :param alpha: [int|float] The opacity of the text. 1 is 100% and 0 is 0% + :param style: The paragraph-style generated by using "make_paragraph_style" method. Avoid using this if you are + passing in an HTML string. + :return: True or False based on the success of the operation. You will get a False even if the text flows out of + the bounding box (which will cause it to not render). + """ + + try: + + # Adjust the dimensions: + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = False) + + # Compute the width and height desired by the user: + width = end_x - start_x + height = end_y - start_y + + # Make the paragraph: + paragraph = Paragraph(text, style = style) + + # Wrap the paragraph to compute the needed dimensions: + needed_width, needed_height = paragraph.wrap(width, height) + + # Adjust the y coordinates (for top-down system): + start_y = self.__canvas_height - start_y - needed_height + + # Write the text to the canvas if the text fits. + # Return true in case of success, else false: + if width >= needed_width and height >= needed_height: + self.__canvas.setFillAlpha(alpha) + paragraph.drawOn(self.__canvas, start_x, start_y, _sW = 0) + return True + else: return False + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def write_string( + self, + text, + x, + y, + font = "Helvetica", + size = 12, + color = (0, 0, 0), + alpha = 1.0, + align = "left", + angle = 0 + ): + + """ + Writes a string onto the canvas. + :param text: [str] The text to be written. + :param x: [int|float] The horizontal coordinate (of the baseline). + :param y: [int|float] The vertical coordinate (of the baseline). + :param font: [str] The name of the font to be used. + :param size: [int|float] The size of the font to be used (in points). + :param color: [str] The color-array (0-255) or hex string that you want the font to be printed in. + :param alpha: [int|float] The opacity of the text. 1 is 100% and 0 is 0% + :param align: [str] The alignment to use ("left", "center", "right"). + :param angle: The angle at which you want the text to be printed in. + :return: True or False based on the success of the operation. + """ + + try: + + # Handle the color + color = self.__parse_color(color) + + # Adjust the dimensions: + x = self.__parse_x(x) + y = self.__parse_y(y, flip = True) + size = self.__parse_font_size(size) + + # Adjust the canvas's values: + self.__canvas.setFont(font, size) + self.__canvas.setFillColorRGB(color[0], color[1], color[2]) + self.__canvas.setFillAlpha(alpha) + + # Handle angular adjustments: + self.__canvas.rotate(angle) + x, y = self._parse_coordinates_for_rotation( + original_x = x, + original_y = y, + angle = angle + ) + + # Adjust the font coordinates as per the angle: + text_width = self.__canvas.stringWidth(text, fontName = font, fontSize = size) + if align == "center": x -= text_width / 2 + elif align == "right": x -= text_width + + # Draw the string: + self.__canvas.drawString(x, y, text) + + # Reset angular adjustments: + self.__canvas.rotate(-angle) + + # Return with success if nothing broke till here: + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + def draw_image( + self, + image, + start_x, + start_y, + end_x, + end_y, + crop = False, + crop_anchor = "c", + alpha = 1.0, + stroke_color = (0, 0, 0), + stroke_alpha = 0.0, + thickness = 1.0, + identifier: str = None, + format = "jpeg" + ): + + """ + Draw an image onto the PDF page in the specified bounding box. The image can be drawn in either "fit" mode where + the image is scaled down to fit in the bounding box, or in "fill" mode where the image is cropped to fill up the + space in the bounding box by matching their aspec ratios. + :param image: The image data as a PIL object or a valid URL or a file path. NOTE: If given as a URL, the image + will be downloaded synchronously. + :param start_x: [int|float] The left coordinate. + :param start_y: [int|float] The top coordinate. + :param end_x: [int|float] The right coordinate. + :param end_y: [int|float] The bottom coordinate. + :param crop: [bool] Set to True for "fill" mode, False for "fit" mode. + :param crop_anchor: [str] If cropping is to be performed, what should the anchor be for cropping. Refer to the + description in 'crop_image_to_aspect_ratio'. + :param alpha: [int|float] The opacity of the image. 1 is 100% and 0 is 0% + :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. + :param stroke_alpha: [int|float] The opacity of the outline color. + :param thickness: [int|float] The line thickness of the outline (in the same unit as the canvas's dimensions). + :param identifier: [str] A unique name for the image. + :param format: The file format of the image (when it will be embedded in the PDF). + :return: True or False based on the success of the operation. + """ + + try: + + # Cleaning: + format = format.strip().lower().split(".")[-1] + + # Handle color: + stroke_color = self.__parse_color(stroke_color) + + # The identifier can allow us to use the same image multiple times in a PDF file. + # If the identifier is not provided, we give the file a random identifier. + if identifier is None: identifier = "".join([random.choice(string.ascii_lowercase) for _ in range(8)]) + + # Adjust the dimensions: + start_x, end_x = self.__parse_x(start_x, end_x) + start_y, end_y = self.__parse_y(start_y, end_y, flip = True) + + # Compute the width and height desired by the user: + width = end_x - start_x + height = end_y - start_y + + # In case the image is passed as a URL, we try to download it and open it as a PIL object: + if isinstance(image, str): + if image.startswith("https://") or image.startswith("http://"): + image = Image.open(io.BytesIO(requests.get(image).content)) + else: image = Image.open(image) + + # The image can have orientation data in its EXIF tags. + # If so, we adjust for it so that reportlab can work with it properly: + image = self.adjust_image_orientation_from_exif(image) + + # If the image needs to be cropped, we compute the aspect ratio and get the image adjusted: + if crop: image = self.crop_image_to_aspect_ratio( + image, + abs(end_x - start_x) / abs(end_y - start_y), + anchor = crop_anchor + ) + + # We set the image opacity and outline settings here: + self.__canvas.setFillAlpha(alpha) + self.__canvas.setLineWidth(thickness * self.__unit_factor) + self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) + self.__canvas.setStrokeAlpha(stroke_alpha) + + # Draw the image onto the canvas: + if ( + image.mode == "P" or + format in ["jpeg", "jpg"] + ): + image = image.convert("RGB") + image_buffer = io.BytesIO() + image.save(image_buffer, format = format) + image_buffer.seek(0) + self.__canvas.drawImage( + ImageReader(image_buffer, ident = identifier), + start_x, + start_y, + width = width, + height = height, + preserveAspectRatio = True, + mask = "auto", + anchor = "c", + showBoundary = True + ) + + return True + + # In case something goes wrong: + except Exception as exception: + self.__printer(exception) + if self.__raise_exception: raise + return False + + # def draw_svg( + # self, + # svg, + # start_x, + # start_y, + # end_x, + # end_y + # ): + # + # renderSVG.draw(svg, sta) + + # ┏┓┳┓┏ ┏┳┓ ┓ + # ┃┃┃┃╋ ┃ ┏┓┏┓┃┏ + # ┣┛┻┛┛ ┻ ┗┛┗┛┗┛ + + @staticmethod + def make_thumbnails( + pdf_file, + pages = None, + scale = 1.0, + as_pil = True, + format = "png" + ): + + """ + Makes thumbnails of pages of the PDF and returns them + :param pdf_file: The PDF file whose pages must be converted to thumbnails. + :param pages: The list of page nos. to make thumbnails of. If not specified, all pages will be converted. + :param scale: The scaling multiplier, applied to both axes, to change the output size. + :param as_pil: If True, the output will be an array of PIL objects, else the output will be an array of + file-like BytesIO objects. + :param format: The file format of the output image. + :return: The array of thumbnails, either as PIL objects, or as BytesIO objects. + """ + + # Open the PDF, and ensure that we have the pages numbers: + if isinstance(pdf_file, io.BytesIO): + pdf_file.seek(0) + pdf_document = fitz.open(stream = pdf_file) + pdf_file.seek(0) + else: pdf_document = fitz.open(pdf_file) + if pages is None: pages = list(range(len(pdf_document))) + + # Iterate over the pages and make the thumbnails: + thumbnails = [] + for page_no in pages: + img = pdf_document.load_page(page_no).get_pixmap(matrix = fitz.Matrix(scale, scale)) + img = io.BytesIO(img.tobytes(format)) + if as_pil: img = Image.open(img, formats = [format]) + thumbnails.append(img) + + # Done here: + return thumbnails + + @staticmethod + def join_from_ram(pdf_files, raise_exception = True): + + """ + Joins a set of PDF files held in RAM (as BytesIO objects). + :param pdf_files: The list of PDF files (as BytesIO objects). + :param raise_exception: Set to True for an all-or-nothing process. + :return: The joined PDF file (as a BytesIo object). + """ + + try: + + # Create a file in RAM that will hold the merged PDF: + joined_pdf_in_ram = io.BytesIO() + + # Merge the files into one: + merger = PdfMerger() + for pdf_file in pdf_files: merger.append(pdf_file) + merger.write(joined_pdf_in_ram) + merger.close() + + # Done here: + joined_pdf_in_ram.seek(0) + return joined_pdf_in_ram + + # In case something goes wrong: + except Exception as exception: + if raise_exception: raise + return None + + @staticmethod + def grid_from_ram( + pdf_files, + input_width, + input_height, + output_width, + output_height, + cutting_width, + cutting_height, + unit = "mm", + dpi = 72 + ): + + """ + Lays out a set of input PDFs into a larger pdf in a grid. + NOTE: The 'unit' and 'dpi' choice will be used for both (input and output). + WARNING: I STRONGLY RECOMMEND NOT USING DPI AS A SETTING. THE PDF FORMAT ITSELF DOESN'T HAVE ANY PROVISION FOR + PIXEL DENSITY. THIS SETTING HAS BEEN MADE AVAILABLE FOR PLACEBO EFFECT WHEN CLIENTS ASK FOR VERY HIGH RESOLUTION + FILES WITHOUT WANTING TO UNDERSTAND ANY TECHNICALITY. USE WITH CAUTION. + :param pdf_files: The list of PDF files (as BytesIO objects) to lay onto the larger canvas. + :param input_width: The width of the input PDFs. + :param input_height: The height of the input PDFs + :param output_width: The width of the large PDF on which the smaller PDFs will be laid out. + :param output_height: The height of the large PDF on which the smaller PDFs will be laid out. + :param cutting_width: The horizontal buffer to leave between the grid. + :param cutting_height: The vertical buffer to leave between the grid. + :param unit: The unit to use (common to the input and output PDFs). + :param dpi: The pixel density to consider (common to the input and output PDFs). + :return: The PDF grid. + """ + + # Create the base PDF: + base_canvas = PDFMaker( + width = output_width, + height = output_height, + unit = unit, + dpi = dpi + ) + base_canvas.draw_line(0, 1, 1, 0, alpha = 0.0) + base_pdf = base_canvas.save() + + # Adjust all units to match the pixel density adjustments: + input_width *= base_canvas.unit_factor + input_height *= base_canvas.unit_factor + output_width *= base_canvas.unit_factor + output_height *= base_canvas.unit_factor + cutting_width *= base_canvas.unit_factor + cutting_height *= base_canvas.unit_factor + + # Calculate how many input pdfs will fit and the page count needed: + per_row = math.floor((output_width - cutting_width) / (input_width + cutting_width)) + per_column = math.floor((output_height - cutting_height) / (input_height + cutting_height)) + per_page = per_row * per_column + output_page_count = math.ceil(len(pdf_files) / per_page) + + # Calculate the starting point on the page from where the pasting will start: + page_start_x = (output_width - (per_row * (input_width + cutting_width)) + cutting_width) / 2.0 + page_start_y = (output_height - (per_column * (input_height + cutting_height)) + cutting_height) / 2.0 + + # We make a writer object to write pages to the output file, + # and we make a list that will hold the individual output pages: + grid_pdf = PdfWriter() + + # We create each needed page: + for page_number in range(output_page_count): + + # Open the base PDF and create a page here: + grid_page = PdfReader(base_pdf).pages[0] + + # Shortlist the input pdfs to be pasted on this page: + starting_offset = page_number * per_page + ending_offset = starting_offset + per_page + ending_offset = min(ending_offset, len(pdf_files)) + pdf_files_for_page = pdf_files[starting_offset:ending_offset] + + # We paste all the input PDFs one-by-one: + for index, input_pdf in enumerate(pdf_files_for_page): + + # Compute the coordinates to paste on: + row_number = math.floor(index / per_row) + column_number = index if index < per_row else index % per_row + paste_x = page_start_x + (column_number * input_width) + (column_number * cutting_width) + paste_y = page_start_y + (row_number * input_height) + (row_number * cutting_height) + + # The default system for the reportlab is bottom-up, + # we convert to top-down for intuitive operations: + paste_y = base_canvas.shape[1] - paste_y - input_height + + # Open th smaller PDF and paste it on those coordinates: + input_page = PdfReader(input_pdf).pages[0] + input_page.add_transformation(Transformation().translate( + tx = paste_x, + ty = paste_y + ), expand = True) + grid_page.merge_page(input_page) + + # Add the grid page to the grid PDF: + grid_pdf.add_page(grid_page) + + # Join and return the result: + grid_pdf_in_ram = io.BytesIO() + grid_pdf.write(grid_pdf_in_ram) + return grid_pdf_in_ram + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass + diff --git a/queue/__init__.py b/queue/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/queue/async_kafka.py b/queue/async_kafka.py new file mode 100644 index 0000000..d6c0a82 --- /dev/null +++ b/queue/async_kafka.py @@ -0,0 +1,554 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 13th Jul, 2024 + + OBJECTIVE: + + To be able to work with keys + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To use Kafka: +from aiokafka import AIOKafkaProducer +from aiokafka import AIOKafkaConsumer + +# For working with JSON strings: +from utils_v2.string import json +from utils_v2.serialization.json_serializer import JSONSerializer + +# For debugging: +from icecream import IceCreamDebugger + +# For SSL security: +import ssl + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_ssl_context( + ca_file, + cert_file, + key_file +): + + """ + Generate the SSL context to use with the Kafka instances. + :param ca_file: The Certificate Authority file as a path to a local file. + :param cert_file: The Certificate file as a path to a local file. + :param key_file: The Key file as a path to a local file. + :return: The SSL context instance as a path to a local file. + """ + + ssl_context = ssl.create_default_context() + ssl_context.load_verify_locations(ca_file) + ssl_context.load_cert_chain(certfile = cert_file, keyfile = key_file) + return ssl_context + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class ProducerKafka: + + def __init__( + self, + topic, + serializer = None, + debug = True, + debug_prefix = "Kafka (P) | ", + **kwargs + ): + + """ + Create a Kafka Producer. + :param topic: The topic to produce on. + :param serializer: The serializer to use. + :param debug: Whether, or not, you want to print the debug strings. + :param debug_prefix: The prefix to use while debugging. + :param kwargs: Any configuration parameters for the Kafka instances. + """ + + # Initialize the debugger: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # initialize the Kafka producer: + self.__topic = topic + self.__kwargs = kwargs + self.__producer = None + self.__connected = False + self.__serializer = serializer or JSONSerializer() + + # For establishing connection: + self.__exclusive_semaphore = asyncio.Semaphore(1) + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + async def connect(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + async with self.__exclusive_semaphore: + if not self.__connected: + try: + self.__producer = AIOKafkaProducer(**self.__kwargs) + await self.__producer.start() + self.__connected = True + except Exception as exception: self.__printer(exception) + return self.__connected + + async def ensure_connection(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + if not self.__connected: await self.connect() + return self.__connected + + async def close(self): + + """ + Terminates the connection. + :return: None. + """ + + if self.__connected: + try: + await self.__producer.stop() + self.__printer("Producer closed!") + self.__connected = False + except Exception as exception: self.__printer(exception) + + async def produce(self, value, key = None, topic = None, encoding = "utf-8"): + + """ + Sends one message to the Kafka server on the topic that has been set for this instance. + :param value: The message to send. + :param key: The key to use when you want the messages to follow an order. + :param topic: A custom topic for this message, else the topic defined during the creation of this instance will + be used by default. + :param encoding: The encoding format. + :return: True or False based on the success of the operation. + """ + + # Ensure connectivity to the server. + # If not connected, return with failure immediately. + if not await self.ensure_connection(): return False + + try: + + # Send the message: + await self.__producer.send_and_wait( + topic = topic or self.__topic, + value = self.__serializer.serialize(data = value, encoding = encoding), + key = key + ) + + # Return with success if no exception occurred: + return True + + # Return with failure if something went wrong: + except Exception as exception: + self.__printer(exception, self.__topic, type(value), value) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +class ConsumerKafka: + + def __init__( + self, + topic, + serializer = None, + debug = True, + debug_prefix = "Kafka (C) | ", + **kwargs + ): + + """ + Create a Kafka Consumer. + :param topic: The topic to consumer on. + :param serializer: The serializer to use. + :param debug: Whether, or not, you want to print the debug strings. + :param debug_prefix: The prefix to use while debugging. + :param kwargs: Any configuration parameters for the Kafka instances. + """ + + # Initialize the debugger: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # initialize the Kafka producer: + self.__topic = topic + self.__kwargs = kwargs + self.__consumer = None + self.__connected = False + self.__serializer = serializer or JSONSerializer() + + # For establishing connection: + self.__exclusive_semaphore = asyncio.Semaphore(1) + + def enable_debug(self): + self.__printer.enable() + + def disable_debug(self): + self.__printer.disable() + + async def connect(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + async with self.__exclusive_semaphore: + if not self.__connected: + try: + self.__consumer = AIOKafkaConsumer(self.__topic, **self.__kwargs) + await self.__consumer.start() + self.__connected = True + except Exception as exception: self.__printer(exception) + return self.__connected + + async def ensure_connection(self): + + """ + Connects to the Kafka server if not connected. + :return: True or False based on the success of the operation. + """ + + if not self.__connected: await self.connect() + return self.__connected + + async def close(self): + + """ + Terminates the connection. + :return: None. + """ + + if self.__connected: + try: + await self.__consumer.stop() + self.__printer("Consumer closed!") + self.__connected = False + except Exception as exception: self.__printer(exception) + + async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"): + + """ + Get messages from the Kafka server. + :param count: The number of messages to get from the Kafka server. + :param timeout: The time in seconds to wait for retrieval. + :param encoding: The encoding to use. + :return: The messages that were received. If no messages are available, an empty list will be returned. + """ + + # Ensure connectivity to the server. + # If not connected, return with failure immediately. + if not await self.ensure_connection(): return [] + + # Make a variable that will hold the final results: + messages = [] + + try: + + # Read some messages: + results = await self.__consumer.getmany( + max_records = max(1, count), + timeout_ms = int(timeout * 1_000) + ) + + # Format the received messages: + if results: + for topic_partition, records in results.items(): + for record in records: + record_dict = record.__dict__ + record_dict["value"] = self.__serializer.deserialize( + data = record_dict["value"], + encoding = encoding + ) + messages.append(record_dict) + + # Debugging print if something went wrong: + except Exception as exception: self.__printer(exception) + + # Done here: + return messages + + +# --------------------------------------------------------------------------------------------------------------------- + + +class BidirectionalKafka: + + # The 'roles' that the instance can take. + # The master talks on the channel (topic) that the slave listens on and vice versa. + # Master-Slave is only for deciding who talks on which channel and who listens on which. + # In a two-party system, one must be the master, the other must be the slave. + # There are no extra privileges that the master enjoys. The naming convention was borrowed from common protocols + # used in electronics (like I2C). + ROLE_MASTER = 1 + ROLE_SLAVE = 0 + + def __init__( + self, + role, + topic, + ack_topic: str = None, + group: str = None, + serializer = None, + debug = True, + debug_prefix = "Kafka (B) | ", + **kwargs + ): + + """ + Creates a walkie-talkie type setup to use Kafka in a bidirectional manner. Fo more information on all the + individual methods, please read through the doc-strings of the component classes 'ProducerKafka', and + 'ConsumerKafka'. + :param role: Select from "ROLE_MASTER" and "ROLE_SLAVE". Between the two parties that are talking, one will be + the master and the other will be the slave. The channel that the master uses to speak will the one the slave + uses to listen, and vice versa. + :param topic: The topic to communicate on. Will be the same between the master and the slave. + :param ack_topic: Explicitly provide this for the second channel, or it will be created from the name of the + topic itself. Will be the same between the master and the slave. + :param group: The group to assign the instance to. + :param debug: Whether, or not, you want to print the debug strings. + :param debug_prefix: The prefix to use while debugging. + :param kwargs: Any configuration parameters for the Kafka instances. + """ + + # Not down the basic variables: + self.__role = role + self.__topic = topic + self.__ack_topic = ack_topic or topic + "Ack" + self.__group = group + + # In case the current instance is the master, + # it will talk on "topic", and listen on "ack_topic": + if self.__role == self.ROLE_MASTER: + self.__producer_kwargs = kwargs.copy() + self.__producer = ProducerKafka( + topic = self.__topic, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (P) | ", + **self.__producer_kwargs + ) + self.__consumer_kwargs = kwargs.copy() + self.__consumer_kwargs["group_id"] = self.__group + self.__consumer = ConsumerKafka( + topic = self.__ack_topic, + group = group, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (C) | ", + **self.__consumer_kwargs + ) + + # On the other hand, if the current instance is a slave, + # It will listen on "topic", and talk on "ack_topic": + else: + self.__producer_kwargs = kwargs.copy() + self.__producer = ProducerKafka( + topic = self.__ack_topic, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (P) | ", + **self.__producer_kwargs + ) + self.__consumer_kwargs = kwargs.copy() + self.__consumer_kwargs["group_id"] = self.__group + self.__consumer = ConsumerKafka( + topic = self.__topic, + group = group, + serializer = serializer, + debug = debug, + debug_prefix = debug_prefix.strip() + " (C) | ", + **self.__consumer_kwargs + ) + + def enable_debug(self): + self.__producer.enable_debug() + self.__consumer.enable_debug() + + def disable_debug(self): + self.__producer.disable_debug() + self.__consumer.disable_debug() + + async def ensure_connection(self): + await self.__producer.ensure_connection() + await self.__consumer.ensure_connection() + + async def close(self): + await self.__producer.close() + await self.__consumer.close() + + async def produce(self, message, encoding = "utf-8"): + return await self.__producer.produce(message, encoding = encoding) + + async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"): + return await self.__consumer.consume(count = count, timeout = timeout, encoding = encoding) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + import time + from data_models.kafka_message import KafkaMessage + + ssl_ctx = get_ssl_context( + ca_file = r"/home/developer/PycharmProjects/utils/cred/kafka/cert_authority.pem", + cert_file = r"/home/developer/PycharmProjects/utils/cred/kafka/fullchain.pem", + key_file = r"/home/developer/PycharmProjects/utils/cred/kafka/privkey.pem" + ) + + async def consumer_test(): + + consumer = ConsumerKafka( + topic = "kft_file_upload", + # group_id = "assessImg", + group_id = "updateMedia", + bootstrap_servers = "wtt.ditscentre.in:9092", + security_protocol = "SSL", + ssl_context = ssl_ctx + ) + await consumer.connect() + await asyncio.sleep(1.5) + print("READY!") + + while True: + messages = await consumer.consume(count = 1) + if len(messages) > 0: print("MESSAGE:", json.to_string(messages[0], default=str)) + await asyncio.sleep(1.0) + + async def producer_test(): + + producer = ProducerKafka( + topic = "kft_file_upload", + bootstrap_servers = "del.ditscentre.in:9092", + security_protocol = "SSL", + ssl_context = ssl_ctx + ) + await producer.connect() + print("READY!") + + while True: + my_msg = KafkaMessage( + data = { + "accepted": False, + "reason": "low resolution" + }, + media = { + "name": "pikachu_poster.jpg", + "ext": "jpg", + "url": "https://nexcom.ditscentre.in/utils/files/small/download/66ded1c1c1c05139a618b5ff", + "attr": { + "user": "SarangKabir", + "project": "ACE-PGP", + "id": 173, + "campaignActivityId": "25", + "idCampaign": 49, + "phoneNo": "7977821877" + } + }, + appId = "aceWockhardt", + proc = { + "name": "_assessImg", + "attr": { + "blurThreshold": 0.25, + "clarityThreshold": 0.65, + "nsfwThreshold": 0.25, + "minWidth": 512, + "minHeight": 512 + } + }, + ack = None + ) + success = await producer.produce(my_msg.model_dump()) + print("produced...") + time.sleep(1.0) + break + + await producer.close() + + + asyncio.run(consumer_test()) diff --git a/rate_limit/__init__.py b/rate_limit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rate_limit/async_token_bucket.py b/rate_limit/async_token_bucket.py new file mode 100644 index 0000000..4fb85e7 --- /dev/null +++ b/rate_limit/async_token_bucket.py @@ -0,0 +1,212 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Wednesday, 15th May, 2024 + + OBJECTIVE: + + To provide a rate-limiting mechanism using the simple token bucket algorithm. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For date and time keeping: +import datetime +import time + +# For asynchronous activities: +import asyncio + +# For mathematical operations: +import math + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class TokenBucket: + + """ + MODES: "add" - Add 'rate_limit' no. of tokens to the bucket every interval. + "reset" - Reset the token count to 'rate_limit' every interval + """ + + MODE_RESET = 0 + MODE_ADD = 1 + + ONE_YEAR = 3_15_36_000.0 + ONE_MONTH = 26_78_400.0 + ONE_WEEK = 6_04_800.0 + ONE_DAY = 86_400.0 + ONE_HOUR = 3_600.0 + ONE_MINUTE = 60.0 + ONE_SECOND = 1.0 + + def __init__(self, rate_limit, seconds = 1.0, mode = "reset", sleep = 0.1, in_sequence = False): + + """ + Initialize the rate controller. + :param rate_limit: The no. of operations allowed per unit of time. + :param seconds: The time period in seconds in which the tokens get reset or added. + :param mode: To select what happens when the period is over. 'reset' mode means that the remaining tokens from + the previous period are discarded and the counter is set back to the rate limit, and 'add' mode means that + new tokens are added on top of the exiting ones that were unused in the previous period. + :param sleep: The delay to add before checking back to see if tokens are available. Keep it longer for longer + periods (which is adjusted by the 'seconds' parameter). + :param in_sequence: Whether, or not, you want to maintain the sequence in which the requests were made + (experimental). + """ + + if rate_limit is not None: + rate_limit = int(rate_limit) + self.__semaphore = asyncio.Semaphore(1 if in_sequence else rate_limit) + mode = mode.lower() + if mode not in [self.MODE_RESET, self.MODE_ADD]: mode = self.MODE_RESET + self.__mode = mode + self.__rate_limit = rate_limit + self.__token_count = rate_limit + self.__last_token_generation_time = time.time() + self.__sleep = sleep + self.__seconds = seconds + + async def has_turn(self): + + """ + Just to check if a token is available. + Should be used only if you don't want to wait for turn in case it is unavailable. + :return: True if the token is available, False if not. + """ + + # If the user doesn't want any rate-limits, we return immediately. + # Otherwise, we check if the user has any tokens available: + if self.__rate_limit is None: return True + if self.__token_count > 0: return True + else: return False + + async def get_turn(self, timeout = None): + + """ + To wait till either you get your turn or the wait gets timed-out. + :param timeout: The time (in seconds) to wait to get a turn before exiting with failure. + :return: True if a turn was received, else False if timed-out. + """ + + # If the user doesn't want any rate-limits, we return immediately: + if self.__rate_limit is None: return + + # Start by making variables: + got_turn = False + wait_start = time.time() + + # Otherwise, we wait for the semaphore: + async with self.__semaphore: + + # Wait till you get your turn or the attempt gets timed-out: + while True: + + # If the timeout has been exceeded: + if ( + timeout is not None and + time.time() - wait_start >= timeout + ): break + + # If it is time to generate new tokens: + time_delta = time.time() - self.__last_token_generation_time + if time_delta >= self.__seconds: + if self.__mode == self.MODE_RESET: self.__token_count = self.__rate_limit + else: self.__token_count += int(self.__rate_limit * math.floor(time_delta / self.__seconds)) + self.__last_token_generation_time = time.time() + + # If a token is available (or not): + if self.__token_count > 0: + self.__token_count -= 1 + got_turn = True + break + else: await asyncio.sleep(self.__sleep) + + # Done here: + return got_turn + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import random + + async def single_test(rate_controller, count): + has_turn = await rate_controller.has_turn() + got_turn = await rate_controller.get_turn() + print(f"TURN: {count:.<5} {'Y' if has_turn else '-'} / {'Y' if got_turn else '-'} ({datetime.datetime.now()})") + if got_turn: await asyncio.sleep(random.random()) + + async def multi_test(max_count): + rate_controller = TokenBucket( + rate_limit = 10, + seconds = 1, + in_sequence = False, + sleep = 0.025, + mode = "reset", + timeout = 3 + ) + tasks = [single_test(rate_controller, count + 1) for count in range(max_count)] + await asyncio.gather(*tasks) + + + start_time = time.time() + asyncio.run(multi_test(100)) + print(f"FINISHED IN {time.time() - start_time} SECONDS!") diff --git a/security/__init__.py b/security/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/security/__pycache__/__init__.cpython-310.pyc b/security/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..20af223 Binary files /dev/null and b/security/__pycache__/__init__.cpython-310.pyc differ diff --git a/security/__pycache__/sanitizers.cpython-310.pyc b/security/__pycache__/sanitizers.cpython-310.pyc new file mode 100644 index 0000000..7193fe2 Binary files /dev/null and b/security/__pycache__/sanitizers.cpython-310.pyc differ diff --git a/security/hash.py b/security/hash.py new file mode 100644 index 0000000..b3fc582 --- /dev/null +++ b/security/hash.py @@ -0,0 +1,245 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 26th Aug., 2024 + + OBJECTIVE: + + To provide an easy way to hash inputs. + + REFERENCES: + + 1) Book: Full Stack Python Security - Dennis Byrne + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# For hashing: +import hashlib +import hmac +import secrets +from bcrypt import hashpw, gensalt + +# To work with buffers: +import io + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class Hasher: + + def __init__( + self, + algorithm = hashlib.sha256, + key = None + ): + + """ + hashes data or a message or a file. Uses HMAC if 'key' is specified, else performs simple hashing. + :param algorithm: The algorithm to use. SHA256 by default. + :param key: Specify this as either a string or as bytes to use HMAC. Leave as null for simple hashing. + """ + + # Initialize the hasher: + self.__hasher = None, + self.__algorithm = algorithm + self.__hmac_key = key + if self.__hmac_key is not None: + self.__hmac_key = self.__hmac_key.encode("utf-8") if isinstance(self.__hmac_key, str) else self.__hmac_key + self.reset() + + @staticmethod + def generate_key(byte_count = 32, url_safe = False): + + """ + A mechanism to generate key/salt values. + NOTE: For a proper salt for passwords, I recommend using "generate_salt" method. It's far better. + :param byte_count: The number of bytes to have in the key. The hex output (as string) will have 2x the + characters. + :param url_safe: Set to True if you need the generated output to be a part of a URL. + :return: The generate key/salt. + """ + + return secrets.token_urlsafe(byte_count) if url_safe else secrets.token_hex(byte_count) + + @staticmethod + def generate_salt(): + + """ + Generate a salt to use while hashing things like passwords. + :return: The salt as bytes. + """ + + return gensalt() + + @staticmethod + def hash_password(password, salt): + + """ + Hashes a password with the given salt. + :param password: The password to hash, either as a string or as bytes. + :param salt: The salt to hash the password with, either as a string or as bytes. + :return: The hashed string. + """ + + return hashpw( + password = password.encode("utf-8") if isinstance(password, str) else password, + salt = salt.encode("utf-8") if isinstance(salt, str) else salt + ) + + @staticmethod + def compare_hashes(hash_0, hash_1): + + """ + Compares two hashes to see if they match. + Comparison is done in constant time to avoid timing-based side-channel attacks. + :param hash_0: One of the hashes to compare. + :param hash_1: The other hash to compare. + :return: True if they match, else False. + """ + + return hmac.compare_digest(hash_0, hash_1) + + def reset(self): + + """ + Resets the hasher by removing all the data that was fed into it. + :return: None. + """ + + if self.__hmac_key is not None: + self.__hasher = hmac.new( + key = self.__hmac_key, + digestmod = self.__algorithm + ) + else: self.__hasher = self.__algorithm() + + def update(self, data): + + """ + Adds data to the hash to update it. + :param data: The data to be hashed. + :return: None. + """ + + data = data.encode("utf-8") if isinstance(data, str) else data + self.__hasher.update(data) + + def digest(self): + + """ + Returns the hexadecimal representation of the hash as a string. + :return: The hexadecimal representation of the hash as a string + """ + + return self.__hasher.digest() + + def hexdigest(self): + + """ + Returns the hexadecimal representation of the hash as a string. + :return: The hexadecimal representation of the hash as a string + """ + + return self.__hasher.hexdigest() + + def hash_message(self, message, as_hex = True): + + """ + Hashes one message and returns the result, and then resets the instance. + :param message: The data you want to hash. + :param as_hex: Invokes 'hexdigest' if True, else 'digest'. + :return: The hash of the message in either hexadecimal string form or binary form. + """ + + self.update(message) + hash_result = self.hexdigest() if as_hex else self.digest() + self.reset() + return hash_result + + def hash_file(self, file, chunk_size = 4096, as_hex = True): + + """ + Hashes one file and returns the result, and then resets the instance. + :param file: The file you want to hash either as a path or as some buffer (like io.BytesIO). + :param chunk_size: The size of data (in bytes) that you would like to pick at one time. + :param as_hex: Invokes 'hexdigest' if True, else 'digest'. + :return: The hash of the file in either hexadecimal string form or binary form. + """ + + # In case the file was given as a io.BytesIO buffer: + if isinstance(file, io.BytesIO): + file.seek(0) + while True: + chunk = file.read(chunk_size) + if not chunk: break + self.update(chunk) + + # In case the file was given as a path: + else: + with open(file, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + self.update(chunk) + + # Now we capture the results, reset the instance, and return the result: + hash_result = self.hexdigest() if as_hex else self.digest() + self.reset() + return hash_result + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + print( + Hasher.hash_password( + password = "mic test, mic test, 123", + salt = Hasher.generate_salt() + ) + ) diff --git a/security/otp.py b/security/otp.py new file mode 100644 index 0000000..49edcd2 --- /dev/null +++ b/security/otp.py @@ -0,0 +1,156 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 10th Sept., 2024 + + OBJECTIVE: + + To provide a way to generate and verify OTPs. + + REFERENCES: + + 01. https://pyauth.github.io/pyotp/# + 02. https://en.wikipedia.org/wiki/Google_Authenticator + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with OTPs: +import pyotp +import base64 +import hashlib + +# To work with date and time: +import time +import datetime + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class HashedOTP: + + def __init__( + self, + secret + ): + + """ + Used to generate and verify HMAC-based OTPs. + :param secret: The key to use to generate and verify OTPs. + """ + + # Note down the input variables: + self.__secret = secret + self.__otp_client = pyotp.HOTP(secret) + + @staticmethod + def generate_secret(message = None): + + """ + Generate a secret key to then use to generate and verify the OTPs. + You may override the random generator by giving a "message" of any length. + :param message: A custom value to convert into a key. Avoid using this for better security, but this can be used + to generate keys based on user identifiers. THERE IS NO RANDOMNESS IF YOU USE THIS FEATURE. IT IS FOR + CONVENIENCE ONLY. NOT RECOMMENDED. + :return: The key (as a string) that can be used to generate and verify the OTPs. + """ + + # If the user wants to generate a key from a custom input: + if message: + + # Ensure we have a bytes object: + if not isinstance(message, (str, bytes)): message = str(message) + if isinstance(message, str): message = message.encode("utf-8") + + # Hash the bytes object: + sha256_hash = hashlib.sha256() + sha256_hash.update(message) + hashed_key = sha256_hash.digest() + + # Convert to base-32: + return base64.b32encode(hashed_key).decode("utf-8") + + # If the user wants a totally random key: + else: return pyotp.random_base32() + + def generate_otp(self, count: int): + + """ + Generates the OTP at a particular step. + :param count: The step at which the OTP needs to be generated. + :return: The OTP string (6 digits). + """ + + return str(self.__otp_client.at(count)) + + def verify_otp(self, otp, count: int): + + """ + Verifies the claimed OTP. + :param otp: The OTP as claimed by the end user. + :param count: The step at which the OTP needs to be verified. + :return: True if the OTP is valid, else False. + """ + + return self.__otp_client.verify(otp, counter = count) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/security/sanitizers.py b/security/sanitizers.py new file mode 100644 index 0000000..26dfe5b --- /dev/null +++ b/security/sanitizers.py @@ -0,0 +1,135 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 25th Jul., 2024 + + OBJECTIVE: + + To provide a set of data cleaning functions for inputs like phone numbers, emails, etc. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# My utils: +from utils_v2.string import regex + +# For random strings and tokens: +import string +import random + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def file_name(input_string: str): + + """ + Cleans up the string to allow it to safely become a filename. + :param input_string: The string that you want to make safe for using as a filename. + :return: The string that can safely be used as a filename. + """ + + return regex.replace( + text = input_string.replace("\n", " "), + pattern = r"[^a-zA-Z0-9 \-_\.]", + substitute_text = "" + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def for_mongo(input_document): + + """ + Sanitizes and disarms any JSON-like input that could be used for NoSQL-injection attacks. + :param input_document: The list or dict to be sanitized. + :return: The sanitized list or dict. + """ + + # A special function that disarms any input string by dealing with special characters + # that Mongo may consider to be instructions: + def disarm(input_string): + input_string = regex.replace( + text = input_string, + pattern = r"[^a-zA-Z0-9,_\-\.\\\/:;' ]", + substitute_text = "" + ) + return input_string + + # Initially we assign the value of the input to the output: + sanitized_document = input_document + + # Handle the case where the input is an array: + if isinstance(input_document, list): + sanitized_document = [for_mongo(document) for document in input_document] + + # Handle the case when the input is a document: + elif isinstance(input_document, dict): + sanitized_document = {} + for k, v in input_document.items(): + sanitized_document[disarm(k)] = v if type(v) not in [list, dict] else for_mongo(v) + + # Done here: + return sanitized_document + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/serialization/__init__.py b/serialization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/serialization/json_serializer.py b/serialization/json_serializer.py new file mode 100644 index 0000000..87595e0 --- /dev/null +++ b/serialization/json_serializer.py @@ -0,0 +1,129 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Sunday 1st Sept. 2024. + + OBJECTIVE: + + To provide a way to convert any input data to serialized bytes, and back. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with files: +from utils_v2.string import json + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class JSONSerializer: + + def __init__(self): + + """ + Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and + python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour. + """ + + pass + + @staticmethod + def serialize(data, encoding = "utf-8"): + + """ + Serializes the data that is given to it. + The input has to be JSON-compatible. + :param data: The data to serialize. + :param encoding: The encoding to use. + :return: The bytes representing the data. + """ + + # If the data is not already a JSON string, parse it. Then return it as bytes: + data = data if isinstance(data, str) else json.to_string(data, no_space = True) + return data.encode(encoding) + + @staticmethod + def deserialize(data, encoding = "utf-8"): + + """ + Deserializes the bytes that are given to it. + The input has to be JSON-compatible. + :param data: The bytes to deserialize. + :param encoding: The encoding to use. + :return: The data from the bytes that described it. + """ + + data = data.decode(encoding) + return json.from_string(data) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/serialization/universal_serializer.py b/serialization/universal_serializer.py new file mode 100644 index 0000000..23818c8 --- /dev/null +++ b/serialization/universal_serializer.py @@ -0,0 +1,265 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Sunday 1st Sept. 2024. + + OBJECTIVE: + + To provide a way to convert any input data to serialized bytes, and back. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with files: +from utils_v2.string import json + +# To work with tabulated data: +import pandas as pd + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class UniversalSerializer: + + def __init__(self): + + """ + Use this when you are working with varied datatypes. You can add custom data-converters also using the + 'add_converters' method. Otherwise, most default pythonic datatypes are supported out of the box. Note that this + is NOT recommended because of how large the serialized messages become. Try using 'JSONSerializer' when you know + you will be working specifically with JSOn-compatible inputs. + """ + + # These are the converters to use when serializing data: + self.__forward_converters = { + "set": lambda x: list(x), + "tuple": lambda x: list(x), + "complex": lambda x: {"r": x.real, "i": x.imag}, + "DataFrame": lambda x: x.to_dict() + } + + # These are the converters to use when deserializing data: + self.__reverse_converters = { + "set": lambda x: set(x), + "tuple": lambda x: tuple(x), + "complex": lambda x: complex(x["r"], x["i"]), + "DataFrame": lambda x: pd.DataFrame.from_dict(x) + } + + def add_converters( + self, + type_name, + forward_converter_func, + reverse_converter_func + ): + + """ + Add custom datatype converters. + RULES: + 01. Each of the converter functions must take in exactly on argument and return one output of native python + type. This is very important. + 02. Each forward and reverse converters must give symmetric results. + :param type_name: The name of the datatype. HINT: type(obj).__name__ + :param forward_converter_func: The function to handle conversion to bytes. Use when serializing. + :param reverse_converter_func: The function to handle conversion from bytes. Used when deserializing. + :return: None. + """ + + self.__forward_converters[type_name] = lambda x: forward_converter_func(x) + self.__reverse_converters[type_name] = lambda x: reverse_converter_func(x) + + def __describe(self, data): + + """ + Notes down the input datatypes of everything. + Does everything upto conversion to byes. + :param data: The data to process. + :return: The description of the datatypes and values of what was given. + """ + + # Note down the type of data that was sent as the input: + data_type = type(data).__name__ + + # Handle iterables: + if isinstance(data, list): data = [self.__describe(item) for item in data] + elif isinstance(data, set): data = [self.__describe(item) for item in data] + elif isinstance(data, tuple): data = [self.__describe(item) for item in data] + elif isinstance(data, dict): data = [ + { + "k": self.__describe(k), + "v": self.__describe(v) + } for k, v in data.items() + ] + + # Convert here, and return: + conv = self.__forward_converters.get(data_type) + if conv is not None: data = conv(data) + return {"d": data, "t": data_type} + + def serialize(self, data, encoding = "utf-8"): + + """ + Serializes the data that is given to it. + :param data: The data to serialize. + :param encoding: The encoding to use. + :return: The bytes representing the data. + """ + + data = self.__describe(data) + data = json.to_string(data, no_space = True) + return data.encode(encoding) + + def __interpret(self, data): + + """ + Interprets the types of data that were serialized originally. + :param data: The data in the serialized form. + :return: Data where the appropriate datatypes have been applied. + """ + + # Handle iterables: + if data["t"] == "list": data = [self.__interpret(item) for item in data["d"]] + elif data["t"] == "set": data = set([self.__interpret(item) for item in data["d"]]) + elif data["t"] == "tuple": data = tuple([self.__interpret(item) for item in data["d"]]) + elif data["t"] == "dict": data = { + self.__interpret(item["k"]): self.__interpret(item["v"]) + for item in data["d"] + } + + # Handle custom types: + else: + conv = self.__reverse_converters.get(data["t"]) + data = data["d"] + if conv is not None: data = conv(data) + + # Done here + return data + + def deserialize(self, data, encoding = "utf-8"): + + """ + Deserializes the bytes that are given to it. + :param data: The bytes to deserialize. + :param encoding: The encoding to use. + :return: The data from the bytes that described it. + """ + + data = data.decode(encoding) + data = json.from_string(data) + return self.__interpret(data) + + +# --------------------------------------------------------------------------------------------------------------------- + + +class JSONSerializer: + + def __init__(self): + + """ + Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and + python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour. + """ + + pass + + @staticmethod + def serialize(data, encoding = "utf-8"): + + """ + Serializes the data that is given to it. + The input has to be JSON-compatible. + :param data: The data to serialize. + :param encoding: The encoding to use. + :return: The bytes representing the data. + """ + + # If the data is not already a JSON string, parse it. Then return it as bytes: + data = data if isinstance(data, str) else json.to_string(data, no_space = True) + return data.encode(encoding) + + @staticmethod + def deserialize(data, encoding = "utf-8"): + + """ + Deserializes the bytes that are given to it. + The input has to be JSON-compatible. + :param data: The bytes to deserialize. + :param encoding: The encoding to use. + :return: The data from the bytes that described it. + """ + + data = data.decode(encoding) + return json.from_string(data) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/sms/__init__.py b/sms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sms/async_nimbus.py b/sms/async_nimbus.py new file mode 100644 index 0000000..1d4b926 --- /dev/null +++ b/sms/async_nimbus.py @@ -0,0 +1,252 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 9th Sept., 2024 + + OBJECTIVE: + + To be able to send SMSs from Nimbus's API and manage the templates and other things from one place. + + REFERENCES: + + 01. https://github.com/innovativevijay/SmsHitApiSample + 02. https://nimbusit.net/appforms/apimanual.php + + DOWNLOADS: + + N/A + + WEB-PORTAL: + + 01. http://nimbusit.net/ + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To make API Calls: +import httpx + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class AsyncNimbusSMS: + + MESSAGE_TYPE_REGULAR = 0 + MESSAGE_TYPE_UNICODE = 1 + + def __init__( + self, + entity_id, + sender_id, + user_id, + api_key, + debug = True, + debug_prefix = "Nimbus SMS | " + ): + + """ + Sets up an instance of the SMS sender through Nimbus IT. + :param entity_id: The entity id as registered with DLT. + :param sender_id: The 6-char code like "HDFCBK", "NSESMS", "ZRODHA" that you see in your SMS inbox. + :param user_id: The 6-digit id that Nimbus has assigned to you. + :param api_key: The API key generated through Nimbus's portal. + :param debug: Whether, or not, you would like to show debugging messages (can be changed on the fly). + :param debug_prefix: The prefix text to show with the debug string. + """ + + # Create the debugging tools: + self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self.__printer.disable() + + # Create an HTTP client to work with: + limits = httpx.Limits( + max_connections = 5, + max_keepalive_connections = 5, + keepalive_expiry = 3600 + ) + self.__http_client = httpx.AsyncClient(limits = limits, timeout = 120) + + # Capture the input config: + self.__entity_id = entity_id + self.__sender_id = sender_id + self.__user_id = user_id, + self.__api_key = api_key + + async def get_balance(self): + + """ + Checks the balance in the Nimbus wallet. + :return: The balance (float) if the request was successful, else None. + """ + + balance = None + + try: + + # Call the API: + response = await self.__http_client.get( + url = r"http://nimbusit.net/api/balance", + params = {"user": self.__user_id, "authkey": self.__api_key} + ) + + # The response of a successful API call looks like "BALANCE:599". We need just the number: + if response.status_code in [200]: balance = float(response.content.decode().split(":")[-1].strip()) + + except Exception as exception: + self.__printer(exception) + + return balance + + async def send_sms( + self, + template_id, + recipient_number, + message, + message_type = MESSAGE_TYPE_REGULAR + ): + + """ + Sends one SMS through Nimbus IT's system. The text of the message must match the template that had been + submitted. A mismatch may cause the message to fail at best, and raise troubles in the real-world with + government bodies at worst. Be careful. + :param template_id: The id of the SMS template as registered on Nimbus's portal. + :param recipient_number: The phone number of the recipient. You can send an array of numbers, too, BUT IT IS + STRONGLY RECOMMENDED TO NOT DO THAT TO AVOID BEING BLOCKED BY DLT. + :param message: The message to send to the recipient. Should match the template that is being sent. + :param message_type: Choose between 'AsyncNimbusSMS.MESSAGE_TYPE_REGULAR' (default) and + 'AsyncNimbusSMS.MESSAGE_TYPE_UNICODE' based on the type of characters being sent. Both are class variables. + :return: The dict of all the details of the message that was sent including whether, or not, it was successfully + sent. Other details depend on the service provider (Nimbus IT in this case). + """ + + # Construct the basic structure of the response of this method: + summary = { + "success": False, + "info": None, + "sender": self.__sender_id, + "recipient": recipient_number, + "message": message, + "length": len(message), + "template_id": template_id, + "raw": None + } + + try: + + # Pre-process the recipient's number: + if not isinstance(recipient_number, (list, set, tuple)): recipient_number = [recipient_number] + + # Call the API: + response = await self.__http_client.get( + url = r"http://nimbusit.net/api/pushsms", + params = { + "user": self.__user_id, + "authkey": self.__api_key, + "sender": self.__sender_id, + "mobile": ",".join([str(num) for num in recipient_number]), + "text": message, + "entityid": self.__entity_id, + "templateid": template_id, + "type": message_type + } + ) + + # For a successful API call: + if response.status_code == 200: + response_json = response.json() + summary["success"] = True if response_json.get("STATUS", "ERROR").lower() in ["ok"] else False + summary["info"] = response_json.get("RESPONSE", {}).get("INFO") + summary["raw"] = { + "http_code": response.status_code, + "response": response_json, + } + + # For any other code that indicates some form of failure: + else: summary["raw"] = { + "http_code": response.status_code, + "response": response.content.decode() + } + + except Exception as exception: + self.__printer(exception) + + return summary + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + + async def main(): + + sender = AsyncNimbusSMS( + entity_id = "", + sender_id = "", + user_id = "", + api_key = "" + ) + + response = await sender.send_sms( + template_id = "", + recipient_number = "", + message = "" + ) + print("SMS API RESPONSE:", response) + + my_balance = await sender.get_balance() + print("REMAINING BALANCE:", my_balance) + + + asyncio.run(main()) diff --git a/string/__init__.py b/string/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/string/__pycache__/__init__.cpython-310.pyc b/string/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..adc32f1 Binary files /dev/null and b/string/__pycache__/__init__.cpython-310.pyc differ diff --git a/string/__pycache__/json.cpython-310.pyc b/string/__pycache__/json.cpython-310.pyc new file mode 100644 index 0000000..5b01bd8 Binary files /dev/null and b/string/__pycache__/json.cpython-310.pyc differ diff --git a/string/__pycache__/regex.cpython-310.pyc b/string/__pycache__/regex.cpython-310.pyc new file mode 100644 index 0000000..ad9ed17 Binary files /dev/null and b/string/__pycache__/regex.cpython-310.pyc differ diff --git a/string/fuzzy.py b/string/fuzzy.py new file mode 100644 index 0000000..34c08d4 --- /dev/null +++ b/string/fuzzy.py @@ -0,0 +1,168 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 18th May, 2024 + + OBJECTIVE: + + To provide a quick set of functions to work with fuzzy logic. + + REFERENCES: + + 1) https://www.w3schools.com/python/python_json.asp + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To apply fuzzy logic: +from thefuzz import fuzz, process + +# To work with tabulated data: +import pandas as pd + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_best_match( + target, + choices, + threshold = 0.70, + partial = False +): + + if partial: scorer = fuzz.partial_token_sort_ratio + else: scorer = fuzz.ratio + + result = process.extractOne( + target, + choices, + score_cutoff = threshold * 100, + scorer = scorer + ) + + try: return result[0] + except: return None + + +# --------------------------------------------------------------------------------------------------------------------- + + +def rank(target, choices, partial = True): + + if partial: scorer = fuzz.partial_token_sort_ratio + else: scorer = fuzz.ratio + + result = process.extract( + target, + choices, + limit = len(choices), + scorer = scorer + ) + + result = pd.DataFrame(result, columns = ["choice", "closeness"]) + result["closeness"] = result["closeness"] / 100.0 + + return result + + +# --------------------------------------------------------------------------------------------------------------------- + + +def match(targets, choices, threshold = 0.7, partial = False, allow_null = False): + + all_matches_df = None + all_matches = {target: None for target in targets} + something_is_null = False + + for target in targets: + match_df = rank(target, choices, partial = partial) + match_df["target"] = target + if all_matches_df is None: all_matches_df = match_df + else: all_matches_df = pd.concat([all_matches_df, match_df]) + + all_matches_df = all_matches_df.sort_values(by = ["closeness"], ascending = False).reset_index(drop = True) + + for target in targets: + target_df = all_matches_df[all_matches_df["target"] == target].reset_index(drop = True) + if target_df.empty: continue + if target_df.at[0, "closeness"] >= threshold: + choice = target_df.at[0, "choice"] + all_matches[target] = choice + all_matches_df = all_matches_df[all_matches_df["choice"] != choice] + else: + all_matches[target] = None + something_is_null = True + + # print(all_matches) + if something_is_null and not allow_null: return None + else: return all_matches + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import async_json_utils + + awb_numbers = [ + "SF1111BIC", + "SF2222BIC", + "SF3333BIC", + "SF4444BIC", + ] + + chat_text = "SF1112BIC" + + # print(chat_text == names[0]) + best_match = get_best_match(chat_text, awb_numbers, threshold = 0.60, partial = False) + print(f"Best match for '{chat_text}' is '{best_match}'") diff --git a/string/help.py b/string/help.py new file mode 100644 index 0000000..85dcc97 --- /dev/null +++ b/string/help.py @@ -0,0 +1,177 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 24th Aug., 2024 + + OBJECTIVE: + + To provide an overview of any function or class in a string. + The generated overview can then either be shown on the terminal, or transmitted over some other medium for + collaborative work. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# System-level activities: +import io +import inspect + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_help_for_class(cls, skip_methods = None): + + """ + Returns the help documentation to use this class. + :param cls: The class whose help string is desired. + :param skip_methods: A list of methods to NOT include in the help text. + :return: This help documentation. + """ + + separator = "\n\n" + ("=" * 120) + "\n\n" + if skip_methods is None: skip_methods = [] + elif not isinstance(skip_methods, list): skip_methods = [skip_methods] + + # Get class name and docstring: + class_name = cls.__name__ + docstring = inspect.getdoc(cls) or "" + help_string = "HELP FOR:\n\n" + help_string += class_name + "\n\n" + help_string += "This document has upto 120 chars per line.\n" + help_string += "Best viewed with monospaced font :)" + help_string += docstring + separator + + # Get all methods and their docstrings. + # Then note the documentation of the methods while ignoring the blacklisted ones: + members = inspect.getmembers(cls, predicate = inspect.isfunction) + func_help = [] + for name, method in members: + + # Ignore if asked, or extract the details: + if name in skip_methods or name.startswith(f"_{class_name}__"): continue + else: func_help.append(get_help_for_function(method)) + + # Put all the things together: + func_help = separator.join(func_help) + help_string += func_help + + # Done here: + return help_string + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_help_for_function(func): + + """ + Get the help string for one function. + It could be a standalone function, or a method of a class. + :param func: The function (or method) whose help string is needed. + :return: The help string of the function. + """ + + # Get the name and documentation: + func_name = func.__name__ + async_indicator = " (async)" if inspect.iscoroutinefunction(func) else "" + func_doc = inspect.getdoc(func) or "" + + # Create the decorated header: + func_decorator = "-" * (len(func_name) + 2) + func_head = "." + func_decorator + f".\n| {func_name} |{async_indicator}\n`" + func_decorator + "`\n\n" + + # Add the 'args' and 'kwargs': + func_args = [] + for name, param in inspect.signature(func).parameters.items(): + default = param.default + if isinstance(default, str): default = f"\"{default}\"" + if default == inspect.Parameter.empty: func_args.append(f"{name}") + else: func_args.append(f"{name}: {type(default).__name__} = {default}") + if len(func_args) > 0: func_args = f"{func_name} (\n\t" + "\n\t".join(func_args) + "\n):\n\n" + else: func_args = f"{func_name} ():\n\n" + + # Get the params and return value part from the doc: + params_start = func_doc.find(":param") + return_start = func_doc.find(":return") + func_params = "\n" + func_doc[params_start:return_start] if params_start >= 0 else "" + func_return = "\n" + func_doc[return_start:] if return_start >= 0 else "" + + # Isolate the documentation part: + if params_start >= 0: func_doc = func_doc[:params_start] + elif return_start >= 0: func_doc = func_doc[:return_start] + + # Done here: + return func_head + func_args + func_doc + func_params + func_return + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_help(entity, skip_methods = None): + + """ + Get the help documentation for anything from its docstring. + :param entity: The entity you want to get help for. + :param skip_methods: A list of methods to ignore if inspecting a class. Not valid for standalone functions. + :return: The help string. + """ + + if inspect.isclass(entity): return get_help_for_class(entity, skip_methods = skip_methods) + else: return get_help_for_function(entity) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/string/json.py b/string/json.py new file mode 100644 index 0000000..67d39f7 --- /dev/null +++ b/string/json.py @@ -0,0 +1,202 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Create: Saturday, 18th May, 2022 + Update: Thursday, 22nd Aug. 2024 + + OBJECTIVE: + + To provide an easy way to work with '.json' data and files. + + REFERENCES: + + 1) https://www.w3schools.com/python/python_json.asp + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# To work with the JSON standard: +import json + +# To work with files: +from utils_v2.system import files + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def from_string(json_data): + + """ + Decodes a JSON string to a pythonic variable like a dict. + :param json_data: The JSON string to decode. + :return: The decoded pythonic variable. + """ + + python_data = json.loads(json_data) + return python_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_string( + python_data, + indent = 4, + default = None, + separators = None, + no_space = False +): + + """ + Converts the given pythonic data to a JSON string. + :param python_data: The input data like a dict. + :param indent: The tab-width for pretty presentation. + :param default: The function to use on something that cannot be directly parsed into a JSON string. + :param separators: Custom separators to use. + :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not + good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT + 'default'. + :return: The JSON string representation of the input pythonic data. + """ + + if no_space: + json_data = json.dumps( + python_data, + default = default, + separators = (',', ':') + ) + + else: + json_data = json.dumps( + python_data, + indent = indent, + default = default, + separators = separators + ) + + return json_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +def from_file(file): + + """ + Reads a JSON file and returns it as a pythonic variable like a dict. + :param file: The path to the file on the disk or a file held in RAM as a BytesIO object. + :return: The decoded pythonic variable. + """ + + if isinstance(file, io.BytesIO): + file.seek(0) + json_data = file.getvalue() + else: json_data = files.read_file(file) + python_data = from_string(json_data) + return python_data + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_file( + file, + python_data, + indent = 4, + default = None, + separators = None, + no_space = False +): + + """ + + :param file: Either a path to a file on disk, or a buffer in RAM in the form of a BytesIO object. + :param python_data: The pythonic data to be converted to the JSON string. + :param indent: The tab-width for pretty presentation. + :param default: The function to use on something that cannot be directly parsed into a JSON string. + :param separators: Custom separators to use. + :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not + good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT + 'default'. + :return: True/False if a path was given, else the same BytesIO object with the written JSON data. + """ + + json_data = to_string( + python_data, + indent = indent, + default = default, + separators = separators, + no_space = no_space + ) + + if isinstance(file, io.BytesIO): + file.write(json_data.encode("utf-8")) + file.seek(0) + return file + + else: + try: + files.write_file(file, json_data, mode = "w") + return True + except: return False + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/string/regex.py b/string/regex.py new file mode 100644 index 0000000..3f01892 --- /dev/null +++ b/string/regex.py @@ -0,0 +1,263 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Sunday, 28th Apr., 2024 + + OBJECTIVE: + + To provide a convenient way to perform RegEx operations like finding patterns and substituting them. + + REFERENCES: + + 1) https://www.w3schools.com/python/python_regex.asp + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# To work with RegEx: +import re + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Common RegEx patterns: +REGEX_EMAIL_ID = r"[\d\w_.+]*@[\d\w_]*.[\d\w]{2,}" +REGEX_PASSWORD = r"^(?=.*[a-z])(?=.*[A-Z])(?=.*[\d])(?=.*[!@#$%^&*()_+{}\[\]:;<>,.?~\\\/-]).{8,}$" +REGEX_NAME = r"^[\d\w .\-]{1,30}$" +REGEX_USERNAME = r"^[\d\w_]{8,25}$" +REGEX_CONTACT_NUMBER = r"\+?\d{0,3}\s*$?\d{3}$?[-.\s]?\d{3}[-.\s]?\d{4}" +REGEX_DATE = r"\b(?:\d{4}-\d{2}-\d{2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},?\s+\d{4}|\d{1,2}\/\d{1,2}\/\d{4}|\d{1,2}-\d{1,2}-\d{2}|\d{1,2}(?:st|nd|rd|th)\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?),?\s+\d{4})\b" +REGEX_GSTIN = r"[0-9]{2}[A-Z]{5}[0-9]{4}[A-Z]{1}[1-9A-Z]{1}Z[0-9A-Z]{1}" +REGEX_PAN = r"[A-Z]{5}[0-9]{4}[A-Z]{1}" +REGEX_IPV4 = (r"[0-9]{1,3}\." * 3) + r"[0-9]{1,3}" +REGEX_IPV6 = (r"[0-9a-fA-F]{1,4}:" * 7) + r"[0-9a-fA-F]{1,4}" +REGEX_IFSC = r"[A-Z]{4}0[A-Z0-9]{6}" +REGEX_UPI = r"[a-zA-Z0-9\.\-]{2,256}@[a-zA-Z][a-zA-Z]{2,64}" +REGEX_MAC_ADDRESS = r"([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})|([0-9a-fA-F]{4}\\.[0-9a-fA-F]{4}\\.[0-9a-fA-F]{4})" +REGEX_METRIC_WEIGHT = r"[\d\.]+[ ]?[k]?g" + + +# RegEx chars (append them to the patterns if needed): +REGEX_START = "^" +REGEX_END = "$" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def find(text, pattern, case_sensitive = True, dot_all = False): + + """ + Returns a list of substrings that match the given RegEx pattern in the input text. + :param text: The text in which the pattern needs to be found. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: An array (list) of substring that match the pattern. Can be an empty list as well. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and clean the results: + matches = [match if type(match) is str else match[1] for match in re.findall(pattern, text, flags = flags)] + matches = [match for match in matches if len(match) > 0] + + # Return the results: + return matches + + +# --------------------------------------------------------------------------------------------------------------------- + + +def replace(text, pattern, substitute_text, case_sensitive = True, dot_all = False): + + """ + Replaces any substring in the text that matches the RegEx pattern. + :param text: The text in which the substitutions need to be made. + :param pattern: The RegEx pattern that needs to be substituted. + :param substitute_text: The text that will replace the matches that were found. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: The text with the substitutions. If no matches are found, the original string is returned. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + return re.sub(pattern, substitute_text, text, flags = flags) + + +# --------------------------------------------------------------------------------------------------------------------- + + +def search(text, pattern, case_sensitive = True, dot_all = False): + + """ + Checks if the given RegEx pattern occurs ANYWHERE in the text that was provided. + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: True if the pattern matches, else False. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + if re.search(pattern, text, flags = flags): return True + else: return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def match(text, pattern, case_sensitive = True, dot_all = False): + + """ + Checks if the given text matches the RegEx pattern that was provided. The check is made only at the start of the + input string. + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: True if the pattern matches, else False. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + if re.match(pattern, text, flags = flags): return True + else: return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def split(text, pattern, case_sensitive = True, dot_all = False): + + """ + Splits an input string based on the pattern that is being matched. + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: True if the pattern matches, else False. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + substrings = re.split(pattern, text, flags = flags) + if len(substrings) > 0 and substrings[0] == "": substrings.pop(0) + return substrings + + +# --------------------------------------------------------------------------------------------------------------------- + + +def to_json(text, pattern, case_sensitive = True, dot_all = False): + + """ + Gives out a dict from the extracted features in a string. It is based on the concept of Named Groups. + Consider the following example (assuming the search is case-insensitive): + TEXT: "UPI/309258561479/14:17:35/UPI/omsainurses@okhdfc" + PATTERN: "upi/.*/(?P.*)/.*/(?P.*)" + RESULT: {'time': '14:17:35', 'ref': 'omsainurses@okhdfc'} + :param text: The text that needs to be matched against the pattern. + :param pattern: The RegEx pattern to look for. + :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. + :param dot_all: Allow all characters to be matched in ".". + :return: A dict with all the extracted features. + """ + + # Prepare the flags: + flags = 0 + if not case_sensitive: flags |= re.IGNORECASE + if dot_all: flags |= re.DOTALL + + # Perform the RegEx operation, and return the results: + matches = re.search(pattern, text, flags = flags) + regex_json = matches.groupdict() if matches else {} + return regex_json + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import json_utils + + for narr in [ + # r"UPI/309258561479/14:17:35/UPI/omsainurses@okhdf cb", + # r"UPI/309258561479/14:17:35/omsainurses@okhdf cb", + # r"NEFT-N095232403538009-RELIGARE BROKING LIMITED MAI", + r"Product listing - My product - 75g - Super combo pack", + r"Product listing - My product - 750 g", + r"0.5 kgs mini pack" + ]: + + result = to_json( + narr, + r"upi/.*/(?P.*)/.*/(?P.*)", + case_sensitive = False + ) + print(result) diff --git a/system/__init__.py b/system/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/system/__pycache__/__init__.cpython-310.pyc b/system/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..dac9ac6 Binary files /dev/null and b/system/__pycache__/__init__.cpython-310.pyc differ diff --git a/system/__pycache__/files.cpython-310.pyc b/system/__pycache__/files.cpython-310.pyc new file mode 100644 index 0000000..3755755 Binary files /dev/null and b/system/__pycache__/files.cpython-310.pyc differ diff --git a/system/files.py b/system/files.py new file mode 100644 index 0000000..1dd719c --- /dev/null +++ b/system/files.py @@ -0,0 +1,308 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Friday, 26th April, 2024 + + OBJECTIVE: + + To provide an easy way to work with files and directories in an synchronous way. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# Other system-level dependencies: +import os +import sys +import pathlib +import platform + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +def get_cwd(): + + if platform.system().lower().find("windows") > -1: return os.getcwd() + else: return os.path.split(os.path.realpath(__file__))[0] + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_parent_directory(path): + + return pathlib.Path(path).parent.absolute() + + +# --------------------------------------------------------------------------------------------------------------------- + + +def make_directory(path): + + if not os.path.exists(path): + try: + os.makedirs(path) + except Exception as excp: + print("FILE UTILS EXCEPTION (make dir):", excp) + return False + + if os.path.exists(path): + return True + + else: + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def get_directory_for_file_path(file_path): + + if os.path.exists(file_path): + path_components = os.path.split(file_path) + file_name = path_components[-1] + if file_name.find(".") > -1: file_path = os.path.join("", *path_components[:-1]) + return file_path + + +# --------------------------------------------------------------------------------------------------------------------- + + +def write_file(file_path, file_data, mode = "w"): + + try: + + file = open(file_path, mode) + file.write(file_data) + file.close() + return True + + except Exception as excp: + print("FILE UTILS EXCEPTION (write file):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def append_file(file_path, file_data): + + return write_file(file_path, file_data, mode = "a") + + +# --------------------------------------------------------------------------------------------------------------------- + + +def read_file(file_path, mode = "r", encoding = "utf8"): + + try: + file = open(file_path, mode, encoding = encoding) + file_contents = file.read() + file.close() + return file_contents + + except Exception as excp: + print("FILE UTILS EXCEPTION (read file):", excp) + return None + + +# --------------------------------------------------------------------------------------------------------------------- + + +def rename_file(current, new): + + try: + if os.path.isfile(current): + os.rename(current, new) + return True + return False + + except Exception as excp: + print("FILE UTILS EXCEPTION (rename file):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def rename_directory(current, new): + + try: + if os.path.isdir(current): + os.rename(current, new) + return True + return False + + except Exception as excp: + print("FILE UTILS EXCEPTION (rename dir):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def delete_file(file_path): + + try: + if os.path.isfile(file_path): + os.remove(file_path) + return True + return False + + except Exception as excp: + print("FILE UTILS EXCEPTION (delete file):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def delete_directory(directory_path): + + try: + if os.path.exists(directory_path): + for root, directories, files in os.walk(directory_path, topdown = False): + for file in files: + os.remove(os.path.join(root, file)) + for directory in directories: + os.rmdir(os.path.join(root, directory)) + os.rmdir(directory_path) + return True + return False + + except Exception as excp: + print("FILE UTILS EXCEPTION (delete dir):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def move_file(file_path, new_dir): + + try: + destination = os.path.join(new_dir, os.path.split(file_path)[-1]) + os.replace(file_path, destination) + return True + + except Exception as excp: + print("FILE UTILS EXCEPTION (move file):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def copy_file(file_path, new_dir): + + try: + destination = os.path.join(new_dir, os.path.split(file_path)[-1]) + handle_src = open(file_path, mode = "r") + handle_dst = open(destination, mode = "w") + stat_src = os.stat(file_path) + n_bytes = stat_src.st_size + fd_src = handle_src.fileno() + fd_dst = handle_dst.fileno() + os.sendfile(fd_dst, fd_src, 0, n_bytes) + return True + + except Exception as excp: + print("FILE UTILS EXCEPTION (copy file):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def list_files(directory_path, full_path = False): + + try: + path = pathlib.Path(directory_path) + files = [entry.name for entry in path.iterdir() if entry.is_file()] + if full_path: files = [os.path.join(directory_path, file) for file in files] + return files + + except Exception as excp: + print("FILE UTILS EXCEPTION (list files):", excp) + return False + + +# --------------------------------------------------------------------------------------------------------------------- + + +def list_dirs( + directory_path, + full_path = False, + raise_exception = False +): + + try: + path = pathlib.Path(directory_path) + files = [entry.name for entry in path.iterdir() if entry.is_dir()] + if full_path: files = [os.path.join(directory_path, file) for file in files] + return files + + except Exception as excp: + if raise_exception: raise + print("FILE UTILS EXCEPTION (list dirs):", excp) + return [] + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass