diff --git a/api/cred_data/blueprint.py b/api/cred_data/blueprint.py index 994c160..b1caa11 100644 --- a/api/cred_data/blueprint.py +++ b/api/cred_data/blueprint.py @@ -57,7 +57,8 @@ from utils_v2.api.async_quart import ( only_whitelisted_ips, limit_rate, validate_input, - handle_cancelled_request + handle_cancelled_request, + make_ordered_json ) # For asynchronous activities: @@ -239,17 +240,12 @@ async def get_data( ) # Successfully retrieved: - response = await make_response(json.to_string(data_json, no_space = True)) - response.headers['Content-Type'] = "application/json" - return response - - print("DATA:", dict(data_json)) - return json.to_string(data_json, no_space = True) - return ResponseModel( + json_data, http_code = ResponseModel( status_code = StatusCodes.OK, data = data_json["content"], message = data_json["desc"] - ) + ).for_quart() + return await make_ordered_json(json_data, http_code) # --------------------------------------------------------------------------------------------------------------------- diff --git a/utils_v2/.gitignore b/utils_v2/.gitignore deleted file mode 100644 index ebb370b..0000000 --- a/utils_v2/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -/.venv/ -/.idea/ -**/__pycache__/ -__pycache__/ - -*.pem -*.pyc -*.pyd \ No newline at end of file diff --git a/utils_v2/__init__.py b/utils_v2/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/ai/__init__.py b/utils_v2/ai/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/ai/image_classification/__init__.py b/utils_v2/ai/image_classification/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/ai/image_classification/async_blur.py b/utils_v2/ai/image_classification/async_blur.py deleted file mode 100644 index f69af42..0000000 --- a/utils_v2/ai/image_classification/async_blur.py +++ /dev/null @@ -1,202 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Monday, 2nd Sept., 2024 - - OBJECTIVE: - - To provide an easy way to assess images for blurriness. - Tried and implemented using HuggingFace behaviour_models. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# For using the AI model: -import torch -from transformers import pipeline - -# To download images from URLs: -import requests - -# To read images: -from PIL import Image - -# Common: -from shared import variables - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AssessImageBlur: - - def __init__( - self, - model, - blur_label = "Blur", - clarity_label = "Normal", - blur_threshold = 0.25, - clarity_threshold = 0.65 - ): - - """ - This class assesses the input image and tells if it is blurry or clear. - :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is - stored. - :param blur_threshold: The max allowed blurriness (0 to 1 range). - :param clarity_threshold: The minimum needed clarity (0 to 1 range). - """ - - # Note down the config: - self.__model = model - self.__device = "gpu" if torch.cuda.is_available() else "cpu" - self.__blur_label = blur_label - self.__clarity_label = clarity_label - self.__blur_threshold = blur_threshold - self.__clarity_threshold = clarity_threshold - - # Initialize the classifier: - self.__classifier = pipeline( - task = "image-classification", - model = self.__model, - device = self.__device - ) - - @staticmethod - async def read_image(source): - - """ - Reads an image in whichever format it is provided and returns it as a PIL object. - :param source: The image as either a path or a URL or a io.BytesIO object. - :return: The image opened as a PIL object. - """ - - # If the input image is already a PIL image: - if isinstance(source, Image.Image): return source - - # If a buffer is provided: - elif isinstance(source, io.BytesIO): - source.seek(0) - return Image.open(source) - - # If a string is provided (local path or URL): - elif isinstance(source, str): - if source.startswith("http://") or source.startswith("https://"): - response = await variables.http_client.get(source) - return Image.open(io.BytesIO(response.content)) - else: return Image.open(source) - - async def classify(self, image): - - """ - To get the prediction of the model from the given input image. - :param image: The image (as a PIL object or file path or a URL). - :return: The dictionary of classes with their respective predictions. - """ - - image = await self.read_image(image) - predictions = self.__classifier(image) - classes = {p["label"]: p["score"] for p in predictions} - return classes - - async def is_ok( - self, - image, - blur_threshold = None, - clarity_threshold = None - ): - - """ - Checks if the image given to it can be used, or should be rejected. - :param image: The image (as a PIL object or file path or a URL). - :param blur_threshold: A custom threshold to test against. If not provided, the default will be taken that was - provided when the instance was created. - :param clarity_threshold: A custom threshold to test against. If not provided, the default will be taken that - was provided when the instance was created. - :return: True if the image is okay, else False. - """ - - image = await self.read_image(image) - blur_threshold = blur_threshold or self.__blur_threshold - clarity_threshold = clarity_threshold or self.__clarity_threshold - classes = await self.classify(image) - if ( - classes[self.__blur_label] <= blur_threshold and - classes[self.__clarity_label] >= clarity_threshold - ): return True - return False - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - - - async def main(): - image = r"/home/developer/Downloads/low-res-check.png" - assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh") - usable = await assessor.is_ok(image) - classes = await assessor.classify(image) - print("IS OKAY:", usable) - print("CLASSES:", classes) - - asyncio.run(main()) diff --git a/utils_v2/ai/image_classification/async_nsfw.py b/utils_v2/ai/image_classification/async_nsfw.py deleted file mode 100644 index 446c34a..0000000 --- a/utils_v2/ai/image_classification/async_nsfw.py +++ /dev/null @@ -1,188 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Monday, 2nd Sept., 2024 - - OBJECTIVE: - - To provide an easy way to assess images for adult content. - Tried and implemented using HuggingFace behaviour_models. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# For using the AI model: -import torch -from transformers import pipeline - -# To download images from URLs: -import requests - -# To read images: -from PIL import Image - -# Common: -from shared import variables - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AssessImageNSFW: - - def __init__( - self, - model, - nsfw_label = "nsfw", - nsfw_threshold = 0.25, - ): - - """ - This class assesses the input image and tells if it is blurry or clear. - :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is - stored. - :param nsfw_threshold: The max allowed blurriness (0 to 1 range). - """ - - # Note down the config: - self.__model = model - self.__device = "gpu" if torch.cuda.is_available() else "cpu" - self.__nsfw_label = nsfw_label - self.__nsfw_threshold = nsfw_threshold - - # Initialize the classifier: - self.__classifier = pipeline( - task = "image-classification", - model = self.__model, - device = self.__device - ) - - @staticmethod - async def read_image(source): - - """ - Reads an image in whichever format it is provided and returns it as a PIL object. - :param source: The image as either a path or a URL or a io.BytesIO object. - :return: The image opened as a PIL object. - """ - - # If the input image is already a PIL image: - if isinstance(source, Image.Image): return source - - # If a buffer is provided: - elif isinstance(source, io.BytesIO): - source.seek(0) - return Image.open(source) - - # If a string is provided (local path or URL): - elif isinstance(source, str): - if source.startswith("http://") or source.startswith("https://"): - response = await variables.http_client.get(source) - return Image.open(io.BytesIO(response.content)) - else: return Image.open(source) - - async def classify(self, image): - - """ - To get the prediction of the model from the given input image. - :param image: The image (as a PIL object or file path or a URL). - :return: The dictionary of classes with their respective predictions. - """ - - image = await self.read_image(image) - predictions = self.__classifier(image) - classes = {p["label"]: p["score"] for p in predictions} - return classes - - async def is_ok( - self, - image, - nsfw_threshold = None - ): - - """ - Checks if the image given to it can be used, or should be rejected. - :param image: The image (as a PIL object or file path or a URL). - :param nsfw_threshold: A custom threshold to test against. If not provided, the default will be taken that was - provided when the instance was created. - :return: True if the image is okay, else False. - """ - - image = await self.read_image(image) - nsfw_threshold = nsfw_threshold or self.__nsfw_threshold - classes = await self.classify(image) - return True if classes[self.__nsfw_label] <= nsfw_threshold else False - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - - - async def main(): - - assessor = AssessImageNSFW(model = r"/path/to/model") - usable = await assessor.is_ok(r"/path/to/image/img.jpg") - classes = await assessor.classify(r"https://...") - - - asyncio.run(main()) diff --git a/utils_v2/ai/image_segmentation/__init__.py b/utils_v2/ai/image_segmentation/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py b/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py deleted file mode 100644 index fb6d6c7..0000000 --- a/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py +++ /dev/null @@ -1,223 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Saturday, 14th Sept., 2024 - - OBJECTIVE: - - To provide an easy way to get masks from dichotomous image segmentation. - - This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait". - You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This - means that the model should have only two classes like "foreground", and "background". The specified model was - trained for implementing portrait mode style blurring of backgrounds. - - The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not - support drop-in replacement for other behaviour_models, please be aware about this. - - REFERENCES: - - 01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait - 02. https://github.com/ZhengPeng7/BiRefNet - - DOWNLOADS: - - 01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait/tree/main - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For system-level activities: -import io - -# To work with PIL images: -from PIL import Image - -# To make asynchronous HTTP calls: -from shared.variables import http_client - -# For using the AI model: -import torch -from torchvision import transforms -from transformers import AutoModelForImageSegmentation - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class DichotomousSegmenter: - - def __init__( - self, - model = r"zhengpeng7/BiRefNet-portrait" - ): - - """ - This class uses Dichotomous Image Segmentation to produce a mask of what is in the foreground (or what is of - interest in a given image). - :param model: The path of the model. - """ - - # Check if GPU is available for faster predictions: - self.__cuda_is_available = True if torch.cuda.is_available() else False - - # Initialize the model: - self.__model = AutoModelForImageSegmentation.from_pretrained( - pretrained_model_name_or_path = model, - trust_remote_code = True - ) - torch.set_float32_matmul_precision(["high", "highest"][0]) - self.__model.to("cuda" if self.__cuda_is_available else "cpu") - - @staticmethod - async def read_image(source): - - """ - Reads an image in whichever format it is provided and returns it as a PIL object. - :param source: The image as either a path or a URL or a io.BytesIO object. - :return: The image opened as a PIL object. - """ - - # If the input image is already a PIL image: - if isinstance(source, Image.Image): return source - - # If a buffer is provided: - elif isinstance(source, io.BytesIO): - source.seek(0) - return Image.open(source) - - # If a string is provided (local path or URL): - elif isinstance(source, str): - if source.startswith("http://") or source.startswith("https://"): - response = await http_client.get(source) - return Image.open(io.BytesIO(response.content)) - else: return Image.open(source) - - async def get_mask(self, image): - - """ - Process the image to figure out the mask. In some cases (like the image of a sunset) you will have no white - pixels in the mask. White pixels represent the areas that the AI considered to be the foreground (area of - interest) and the black parts are the background. - :param image: The image data as either a PIL object, or a path to a file on the local disk, or a URL. - :return: The mask as a PIL object. - """ - - # Prepare the image transformer: - image_size = (1024, 1024) - transform_image = transforms.Compose([ - transforms.Resize(image_size), - transforms.ToTensor(), - transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) - ]) - - # Read and transform the image: - image = await self.read_image(image) - input_image = transform_image(image).unsqueeze(0).to("cuda" if self.__cuda_is_available else "cpu") - - # Make the prediction: - with torch.no_grad(): - if self.__cuda_is_available: mask = self.__model(input_image)[-1].sigmoid().cuda() - else: mask = self.__model(input_image)[-1].sigmoid().cpu() - - # Create a PIL object of the mask data: - mask = mask[0].squeeze() - mask = transforms.ToPILImage()(mask) - mask = mask.resize(image.size) - - # Done here: - return mask - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - import numpy as np - - async def main(): - - model = DichotomousSegmenter() - - images = [ - # r"/home/developer/Downloads/kate.jpg", - # r"/home/developer/Downloads/IMG-20240730-WA0001.jpg", - # r"/home/developer/Downloads/pexels-pixabay-57416.jpg", - # r"/home/developer/Downloads/summer_clothes.jpg", - # r"/home/developer/Downloads/low_res_cat - upscaled.png", - # r"/home/developer/Downloads/blurry_traffic.jpg", - r"/home/developer/Downloads/card_0.jpg", - r"/home/developer/Downloads/card_1.jpg", - r"/home/developer/Downloads/card_2.jpg", - r"/home/developer/Downloads/card_3.jpg", - r"/home/developer/Downloads/sushmita_card.jpg", - r"/home/developer/Downloads/niranjan_card.jpg", - r"/home/developer/Downloads/niranjan_card_2.jpg", - ] - - for image in images: - print(image.split("/")[-1]) - image = await DichotomousSegmenter.read_image(image) - mask = await model.get_mask(image) - image.show() - mask.show() - print("MASK!") - array = np.array(mask) - print("Dimensions of the array:", array.shape) - masked_count = np.sum(array > 125) - print(masked_count, type(masked_count), np.size, type(np.size)) - percentage = masked_count / np.prod(array.shape) - print("Area:", round(percentage * 100, 4)) - # image.putalpha(mask) - # image.show() - # mask.show() - - - asyncio.run(main()) diff --git a/utils_v2/ai/object_detection/__init__.py b/utils_v2/ai/object_detection/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/ai/object_detection/yolo.py b/utils_v2/ai/object_detection/yolo.py deleted file mode 100644 index a435ca6..0000000 --- a/utils_v2/ai/object_detection/yolo.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 4th Jul, 2024 - - OBJECTIVE: - - To provide a class to detect objects in images using YOLO behaviour_models. - - REFERENCES: - - 1) Code Examples: https://docs.ultralytics.com/usage/python/ - 2) Models: https://docs.ultralytics.com/models/yolov8/#supported-tasks-and-modes - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import os - -# To use YOLO architecture: -from ultralytics import YOLO - -# To work with images: -from PIL import Image - -# utils: -from utils_v2.string import json - -# Common: -from shared import constants - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -class YoloDetect: - - def __init__( - self, - model_file, - debug = True, - debug_prefix = "YOLO | " - ): - - # Load the model: - self._model = YOLO(model = model_file) - - # Initialize the dbugging tool: - self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self._printer.disable() - - def disable_debug(self): - self._printer.disable() - - def enable_debug(self): - self._printer.enable() - - @staticmethod - def _process_result(raw_result): - - # Extract the various types of results available from the inference: - speed = {k: (v / 100.0) for k, v in raw_result.speed.items()} - class_mapping = raw_result.names - probabilities = raw_result.probs - boxes = raw_result.boxes - masks = raw_result.masks - - # Construct a default response: - response_json = { - "speed": speed, - "classes": class_mapping, - "boxes": None - } - - # Object-Detection results: - formatted_boxes = [] - for box in boxes: - detected_class = int(box.cls.numpy()[0]) - x1, y1, x2, y2 = box.xyxy[0] - formatted_boxes.append( - { - "class": detected_class, - "className": class_mapping[detected_class], - "confidence": float(box.conf.numpy()[0]), - "x1": int(x1.numpy()), - "y1": int(y1.numpy()), - "x2": int(x2.numpy()), - "y2": int(y2.numpy()) - } - ) - response_json["boxes"] = formatted_boxes - - # Done here: - return response_json - - def predict(self, image, show = False, verbose = False): - - # Process the input image with the given task: - results = self._model( - source = image, - show = show, - verbose = verbose - ) - - # Based on the task, interpret the results: - results_json = self._process_result(results[0]) - - # Done here: - return results_json - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt") - model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt" - # sample_image_path = r"/home/developer/Downloads/2_cats.jpg" - sample_image_path = r"/home/developer/Downloads/flattened_image.jpg" - my_yolo = YoloDetect(model_file = model_file_path) - results = my_yolo.predict(image = Image.open(sample_image_path)) - print("FINAL RESULTS:", json.to_string(results)) - diff --git a/utils_v2/ai/tts/__init__.py b/utils_v2/ai/tts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/ai/tts/easy_tts.py b/utils_v2/ai/tts/easy_tts.py deleted file mode 100644 index fc4bb9b..0000000 --- a/utils_v2/ai/tts/easy_tts.py +++ /dev/null @@ -1,175 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 26th Sept., 2024 - - OBJECTIVE: - - To provide a fast way to make audio from TTS engines. - - REFERENCES: - - 01. Usage: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#python-api - 02. Installation: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#linux-and-macos-install - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# To use the AI: -from melo.api import TTS -import numpy as np -import soundfile - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class EasyTTS: - - __sampling_rate = 44_100 - - def __init__( - self, - language = "EN", - speed = 1.0 - ): - - self.__audio = np.zeros(1) - self.__language = language - self.__speed = speed - self.__model = TTS(language = language, device = "auto") - self.__speakers = self.__model.hps.data.spk2id - - def list_speakers(self): - return list(self.__speakers.keys()) - - def speak( - self, - text, - speaker - ): - - this_audio = self.__model.tts_to_file( - text, - self.__speakers[speaker], - speed = self.__speed, - quiet = True - ) - - self.__audio = np.concatenate((self.__audio, this_audio)) - - def pause(self, seconds): - - self.__audio = np.concatenate(( - self.__audio, - np.zeros(int(self.__sampling_rate * seconds)) - )) - - def to_wav(self, path = None): - - if path is None: - audio_buffer = io.BytesIO() - soundfile.write(audio_buffer, self.__audio, self.__sampling_rate) - return audio_buffer - - else: soundfile.write(path, self.__audio, self.__sampling_rate) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - speaker = "EN-BR" - tts_maker = EasyTTS(language = "EN", speed = 0.9) - - tts_maker.speak( - text = """ - Imagine delighting your doctors with a personalized calendar, - crafted from their own cherished memories. - """, - speaker = speaker - ) - tts_maker.pause(seconds = 0.3) - tts_maker.speak( - text = """ - Every day, as they turn the page, - they’ll not only relive those special moments but also remember you, - the one who made it happen. - """, - speaker = speaker - ) - tts_maker.pause(seconds = 0.75) - tts_maker.speak( - text = "STEP 1:", - speaker = speaker - ) - tts_maker.pause(seconds = 0.3) - tts_maker.speak( - text = "Start by identifying the doctors you’d like to engage with, and add them to our app.", - speaker = speaker - ) - tts_maker.pause(seconds = 0.3) - tts_maker.speak( - text = "No rush, you can add their photographs later as well.", - speaker = speaker - ) - tts_maker.pause(seconds = 0.3) - tts_maker.speak( - text = "With this, your engagement funnel is created.", - speaker = speaker - ) - - tts_maker.to_wav(r"/home/developer/Downloads/voiceover.wav") diff --git a/utils_v2/ai/tts/test.wav b/utils_v2/ai/tts/test.wav deleted file mode 100644 index d38cfc0..0000000 Binary files a/utils_v2/ai/tts/test.wav and /dev/null differ diff --git a/utils_v2/api/__init__.py b/utils_v2/api/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/api/__pycache__/__init__.cpython-310.pyc b/utils_v2/api/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index d03a806..0000000 Binary files a/utils_v2/api/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/__pycache__/async_quart.cpython-310.pyc b/utils_v2/api/__pycache__/async_quart.cpython-310.pyc deleted file mode 100644 index 2f3a904..0000000 Binary files a/utils_v2/api/__pycache__/async_quart.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/__pycache__/codes.cpython-310.pyc b/utils_v2/api/__pycache__/codes.cpython-310.pyc deleted file mode 100644 index 9189fd0..0000000 Binary files a/utils_v2/api/__pycache__/codes.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/__pycache__/log.cpython-310.pyc b/utils_v2/api/__pycache__/log.cpython-310.pyc deleted file mode 100644 index 39955f2..0000000 Binary files a/utils_v2/api/__pycache__/log.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc b/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc deleted file mode 100644 index 724908a..0000000 Binary files a/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/__pycache__/response.cpython-310.pyc b/utils_v2/api/__pycache__/response.cpython-310.pyc deleted file mode 100644 index e62afd0..0000000 Binary files a/utils_v2/api/__pycache__/response.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/async_quart.py b/utils_v2/api/async_quart.py deleted file mode 100644 index e0f2973..0000000 --- a/utils_v2/api/async_quart.py +++ /dev/null @@ -1,1161 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 2nd Aug., 2024 - - OBJECTIVE: - - To provide an easy way to perform repetitive tasks in quart. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - - NOTES: - - 01. PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING ANY OF THESE - DECORATORS. - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To work with Quart: -from quart import request, current_app, g - -# To make decorators: -from functools import wraps - -# My utils: -from utils_v2.string import json -from utils_v2.date_time import date_time -from utils_v2.security import sanitizers -from utils_v2.api.codes import StatusCodes, HttpCodes -from utils_v2.api.log import APILogModel -from utils_v2.api.response import ResponseModel -from utils_v2.logging.context import AsyncLoggerContext, AsyncMongoLogger - -# To work with date and time: -import time -import datetime - -# System-level activities: -import io -import os - -# To work with datatypes: -from types import NoneType -import pandas as pd - -# For Pydantic data-behaviour_models: -import pydantic - -# For hashing and shortening the hash: -import hashlib -import base64 - -# To make things human-readable: -import humanize - -# For debugging: -import traceback -import random -import string - -# For asynchronous activities: -import asyncio - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# Chars to choose from for random strings: -ALPHANUMERIC_CHARS = string.ascii_letters + string.digits - -# To capture system information: -PROCESS_ID = os.getppid() -PARENT_PROCESS_ID = os.getppid() - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** EXCEPTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -class AuthDetailsIncompleteException(Exception): - def __str__(self): - return "incomplete auth details" - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -async def data_from_request(inbound_request): - - """ - Adaptively extract the params from incoming request in whichever way it was provided. - :param inbound_request: The request that came in. - :return: A dict (could be empty) of the data/params that came in with the request. - """ - - inbound_data = {} - - # Extract data from the params in the URL: - from_args = inbound_request.args.to_dict() - if isinstance(from_args, dict): - for k, v in from_args.items(): - inbound_data[k] = v - - # Extract data from the raw JSON data: - from_json = await request.get_json() - if isinstance(from_json, dict): - for k, v in from_json.items(): - inbound_data[k] = v - - # Extract inputs from the form body: - from_form = await inbound_request.form - from_form = from_form.to_dict() - if isinstance(from_form, dict): - for k, v in from_form.items(): - inbound_data[k] = v - - return inbound_data - - -# --------------------------------------------------------------------------------------------------------------------- - - -async def file_from_request(inbound_request, file_key): - - """ - Extracts ONE file from the incoming request's form-data. - WARNING: NOT RECOMMENDED FOR LARGE FILES. STRICTLY USE FOR SMALL FILES THAT WON'T CRASH THE SCRIPT. - :param inbound_request: The incoming request. - :param file_key: The key of the file that you want to extract. - :return: A tuple of the file's name and data. - """ - - file_name = None - file_data = None - files = await inbound_request.files - - if file_key in files: - file_name = files[file_key].filename - file_data = io.BytesIO(files[file_key].read()) - - return file_name, file_data - - -# --------------------------------------------------------------------------------------------------------------------- - - -async def files_from_request(inbound_request: request): - - """ - Extracts ALL files from the incoming request's form-data. - WARNING: NOT RECOMMENDED FOR LARGE FILES. STRICTLY USE FOR SMALL FILES THAT WON'T CRASH THE SCRIPT. - :param inbound_request: The incoming request. - :return: A dict describing the file's name, data, and size. - """ - - files = await inbound_request.files - - inbound_files = {} - for file_key in files: - file_data = io.BytesIO(files[file_key].read()) - file_size = file_data.seek(0, 2) - file_data.seek(0) - inbound_files[file_key] = { - "name": files[file_key].filename, - "data": file_data, - "size": file_size, - "type": files[file_key].content_type - } - - return inbound_files - - -# --------------------------------------------------------------------------------------------------------------------- - - -async def headers_from_request( - inbound_request, - mandatory_keys: list = None -): - - """ - Extract custom headers and some extra info. from the incoming request. - Raises an exception if any mandatory key is missing. - IMPORTANT: CUSTOMIZE THIS FOR THE NEEDS OF YOUR PROJECT. - :param inbound_request: The request that came in. - :param mandatory_keys: The keys that you need to have in the auth. - :return: The extracted auth details. - """ - - # Start by extracting whatever complies with the format of "X-{Header-Name}": - head_json = {k: v for k, v in request.headers.items() if k.startswith("X-")} - - # Now note down things that you want to keep from incoming requests: - head_json["Remote-IP"] = inbound_request.remote_addr - head_json["Host"] = inbound_request.headers.get("Host") - head_json["Origin"] = inbound_request.headers.get("Origin") - head_json["User-Agent"] = inbound_request.headers.get("User-Agent") - head_json["Content-Type"] = inbound_request.headers.get("Content-Type") - head_json["Content-Length"] = inbound_request.headers.get("Content-Length") - - # Raise an exception if any of the mandatory auth details were missing: - if mandatory_keys is not None: - available_keys = head_json.keys() - for mandatory_key in mandatory_keys: - if mandatory_key not in available_keys: raise AuthDetailsIncompleteException - - # Done here: - return head_json - - -# --------------------------------------------------------------------------------------------------------------------- - - -def cause_exception(): - - """ - Call this from any function when you want to raise an exception. - Example use case would be when receiving data from an API call and that field is not supposed to be null. - :return: None. - """ - - return 100/0 - - -# --------------------------------------------------------------------------------------------------------------------- - - -def describe_exception(exc): - - """ - Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire - traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it - to the dev team over some service like WhatsApp/Telegram. - :param exc: The exception that occurred. - :return: The dict that explains the exception. - """ - - exc_desc = { - "type": type(exc).__name__, - "msg": str(exc), - "tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)] - } - - return exc_desc - - -# --------------------------------------------------------------------------------------------------------------------- - - -def messages_from_pydantic_exception(exception, as_str = True, sep = ", "): - - """ - Creates a list of readable error messages from Pydantic's validation failure. - :param exception: Pydantic's ValidationError - :param as_str: Set to True to receive all messages as one string, False to receive an array of strings. - :param sep: The separator to use when joining multiple messages as one string. - :return: A list of messages of all the things that went wrong. - """ - - # Make a variable to hold all individual messages: - messages = [] - - # Interpret all the problems: - for error in exception.errors(): - loc = " --> ".join([str(item) for item in error["loc"]]) - msg = error.get("msg") - if error["type"] == "missing": messages.append(f"missing input: '{loc}'") - elif error["type"] == "model_type": messages.append(f"invalid input: '{loc}'") - elif error["type"] == "bool_parsing": messages.append(f"invalid bool: '{loc}'") - elif error["type"] == "string_type": messages.append(f"invalid string: '{loc}'") - elif error["type"] == "float_parsing": messages.append(f"invalid float: '{loc}'") - elif error["type"] == "int_parsing": messages.append(f"invalid integer: '{loc}'") - elif error["type"] == "extra_forbidden": messages.append(f"extra input: '{loc}'") - elif error["type"] == "value_error": messages.append(f"validation failed: '{loc}' ({msg})") - else: messages.append(f"invalid value: '{loc}'") - - # Return a response as per the preference of the user: - if as_str: return sep.join(messages) - else: return messages - - -# --------------------------------------------------------------------------------------------------------------------- - - -def summarize_variable( - value, - str_limit = 100, - expand: bool | int = False, - sensitive_keys: list[str] = None -): - - """ - To summarize an input value to capture the essence without hoarding to much data. - :param value: Anything that you want to summarize. - :param str_limit: The max. no. of chars of a string to retain. - :param expand: Set to True for full expansion, False for no expansion, and an integer for a specific level of - expansion. Applicable on iterables and dicts. The smaller this number, the more concise the summary will be, - and vice versa. - :param sensitive_keys: The list of keys (of a dict) to obscure when summarizing. - :return: The summarized version of the input. - """ - - # Check the sensitive keys: - if sensitive_keys is None: sensitive_keys = [] - - # Handle datatypes that you don't want to modify: - if isinstance(value, (int, float, bool, NoneType)): pass - - # When the value is a list or similar iterable: - elif isinstance(value, (list, tuple, set)): - if expand: - if not isinstance(expand, bool): expand -= 1 - value = [AsyncLoggerContext.summarize( - v, - expand = expand, - sensitive_keys = sensitive_keys - ) for v in value] - else: value = f"array of {len(value)} item(s)" - - # If the value is a dict: - elif isinstance(value, dict): - if expand: - if not isinstance(expand, bool): expand -= 1 - value = { - k: AsyncLoggerContext.summarize( - v, - expand = expand, - sensitive_keys = sensitive_keys - ) if k not in sensitive_keys else "********" - for k, v in value.items() - } - else: value = f"object of {len(value.keys())} field(s) [{', '.join(value.keys())}]" - - # When a dataframe is passed: - elif isinstance(value, pd.DataFrame): - cols = value.columns.to_list() - value = f"table with {len(cols)} col(s) [{', '.join(cols)}] and {len(value)} row(s)" - str_limit = 999 - - # If the input is some form of non-standard object: - else: value = str(value) - - # Handle strings: - if isinstance(value, str): - if len(value) > str_limit: value = value[:str_limit] + "..." - - # Done here: - return value - - -# --------------------------------------------------------------------------------------------------------------------- - - -def set_api_version(api_version): - - """ - Use this decorator to automatically note down the API version no. and propagate it throughout the downstream - decorators. Use this as the entry point if possible. - :param api_version: The version code to assign to the API. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # set the version information in the variable. - # This makes it available to the downstream decorators too! - kwargs["api_version"] = api_version - - # we are ready to call the function that we are wrapping: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - if isinstance(response, ResponseModel): response.api_version = api_version - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def read_input( - read_headers = True, - read_data = True, - read_files = True, - sanitize_headers = True, - sanitize_data = True -): - - """ - Use this decorator to read the inputs from the incoming request and sanitize them. Sanitization makes the inputs - safe against certain threats like injections attacks. If you expect to take in inputs that you want to use to run - database commands, you could disable them manually. - PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING THIS DECORATOR. - :param read_headers: Whether, or not, to read the headers of the request. - :param read_data: Whether, or not, to read the data of the request. - :param read_files: Whether, or not, to read the files of the request. - :param sanitize_headers: Whether, or not, you would like to sanitize the params coming in through the headers. - :param sanitize_data: Whether, or not, you would like to sanitize the params coming in through the body or query. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # Get the headers: - if read_headers: - kwargs["inbound_headers"] = await headers_from_request(request) - if sanitize_headers: kwargs["inbound_headers"] = sanitizers.for_mongo(kwargs["inbound_headers"]) - # else: kwargs["inbound_headers"] = None - - # Get the data: - if read_data: - kwargs["inbound_data"] = await data_from_request(request) - if sanitize_data: kwargs["inbound_data"] = sanitizers.for_mongo(kwargs["inbound_data"]) - # else: kwargs["inbound_data"] = None - - # Get small files from the request: - if read_files: kwargs["inbound_files"] = await files_from_request(request) - # else: kwargs["inbound_files"] = None - - # # We also make a provision for capturing an identifier - # # for the logs that we make through a sister decorator: - # kwargs["log_id"] = "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(8)) - # kwargs["log_chain"] = kwargs.get("inbound_headers", {}).get("X-Log-Chain") or kwargs["log_id"] - - # Now that we have unpacked the incoming data, - # we are ready to run the function that we are wrapping: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def validate_input( - mandatory_header_keys = None, - mandatory_data_keys = None, - mandatory_file_keys = None, - header_validator = None, - data_validator = None -): - - """ - USE THIS ONLY AFTER YOU HAVE USED 'read_input'. This decorator will help you run validation on the inputs that were - extracted from the request. The mandatory keys will be checked first and the validations will be run after that. If - your validator already checks for keys, you may skip mentioning mandatory keys. DO NOTE THAT YOUR VALIDATOR - FUNCTIONS MUST RAISE AN EXCEPTION FOR THIS DECORATOR TO WORK. - :param mandatory_header_keys: The keys in 'inbound_headers' that are absolutely necessary. - :param mandatory_data_keys: The keys in 'inbound_data' that are absolutely necessary. - :param mandatory_file_keys: The keys in 'inbound_files' that are absolutely necessary. - :param header_validator: The function to use to validate the 'inbound_headers'. - :param data_validator: The function to use to validate 'inbound_data'. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # We first validate the mandatory header keys. - # Having a null value in this case is NOT allowed: - if mandatory_header_keys is not None: - for mandatory_key in mandatory_header_keys: - if kwargs["inbound_headers"].get(mandatory_key) is None: - return ResponseModel( - status_code = StatusCodes.HEADERS_INCOMPLETE, - message = f"missing: '{mandatory_key}'" - ) - - # Return with failure if any of the mandatory JSON details are missing. - # Having a null value is allowed, but is should be sent by the user on intention. - if mandatory_data_keys is not None: - for mandatory_key in mandatory_data_keys: - try: kwargs["inbound_data"][mandatory_key] - except: return ResponseModel( - status_code = StatusCodes.DATA_INCOMPLETE, - message = f"missing: '{mandatory_key}'" - ) - - # Return with failure if any of the mandatory file-keys details are missing: - if mandatory_file_keys is not None: - provided_file_keys = kwargs["inbound_files"].keys() - for mandatory_key in mandatory_file_keys: - if mandatory_key not in provided_file_keys: - return ResponseModel( - status_code = StatusCodes.FILE_MISSING, - message = f"missing: '{mandatory_key}'" - ) - - # Next we validate the headers: - if header_validator is not None: - - # Try validate the data: - try: kwargs["inbound_headers"] = header_validator(kwargs["inbound_headers"]) - - # In case some needed field is missing: - except KeyError as exception: - return ResponseModel( - status_code = StatusCodes.DATA_VALIDATION_FAILURE, - message = "missing: " + str(exception), - http_code = HttpCodes.BAD_REQUEST, - ) - - # In case some pydantic data model fails validation: - except pydantic.ValidationError as exception: - return ResponseModel( - status_code = StatusCodes.DATA_VALIDATION_FAILURE, - message = messages_from_pydantic_exception(exception), - http_code = HttpCodes.BAD_REQUEST - ) - - # In case some other exception was raised: - except Exception as exception: - return ResponseModel( - status_code = StatusCodes.DATA_VALIDATION_FAILURE, - message = str(exception), - http_code = HttpCodes.BAD_REQUEST - ) - - # Finally, we validate the incoming data: - if data_validator is not None: - - # Try validate the data: - try: kwargs["inbound_data"] = data_validator(kwargs["inbound_data"]) - - # In case some needed field is missing: - except KeyError as exception: - return ResponseModel( - status_code = StatusCodes.DATA_VALIDATION_FAILURE, - message = "missing: " + str(exception), - http_code = HttpCodes.BAD_REQUEST - ) - - # In case some pydantic data model fails validation: - except pydantic.ValidationError as exception: - return ResponseModel( - status_code = StatusCodes.DATA_VALIDATION_FAILURE, - message = messages_from_pydantic_exception(exception), - http_code = HttpCodes.BAD_REQUEST - ) - - # In case some other exception was raised: - except Exception as exception: - return ResponseModel( - status_code = StatusCodes.DATA_VALIDATION_FAILURE, - message = str(exception), - http_code = HttpCodes.BAD_REQUEST - ) - - # Now that we have unpacked the incoming data, - # we are ready to run the function that we are wrapping: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def log_request_to_mongo( - attr_name, - collection: str = "logs", - api_version: str = None, - project: str = None, - log_type: str = None, - operation: str = None, - log_input: bool | int = True, - log_output: bool | int = True, - sensitive_keys: list = None -): - - """ - USE THIS ONLY AFTER YOU HAVE USED 'read_input'. This decorator will log the whole process of the API call to - MongoDB. The variable that holds the instance of 'AsyncMongo' needs to be accessible in the scope of 'current_app'. - PLEASE USE "ResponseModel" AS THE RETURNED VALUE OF THE API ENDPOINT IF YOU ARE USING THIS DECORATOR. - :param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the - scope of 'current_app'. - :param collection: The name of the collection to write the log into. - :param api_version: The version code of the API endpoint that is being logged. - :param project: The name of the project that the endpoint was built for. - :param log_type: A hint to identify what the log was for. - :param operation: A hint to identify what was action was being performed. - :param log_input: Set to True to capture everything that went into the function, False to capture the least - info, and set it to an integer to capture a certain depth of the input (applicable on iterables and dicts. - :param log_output: The same as 'log_input', but applicable to the response from the function. - :param sensitive_keys: The list of keys to not log. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Create ids for getting logs: - kwargs["log_id"] = "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(8)) - kwargs["log_chain"] = kwargs.get("inbound_headers", {}).get("X-Log-Chain") or kwargs["log_id"] - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # Set the api version as needed: - kwargs["api_version"] = kwargs.get("api_version", api_version) - - # Make variables and extract available info.: - exception = None - response = None - request_ts = date_time.get_current_utc_date_time() - start_ts = time.perf_counter() - cpu_start_ts = time.process_time() - - # Execute the function that is being wrapped: - try: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - except Exception as exc: exception = exc - - # Ensure that the response is not null: - response = response if response is not None else ResponseModel( - status_code = StatusCodes.UNKNOWN_ERROR, - message = "null response for request" - ) - - # Add params to the response. - # THIS IS ONLY APPLICABLE WHEN THE TYPE OF THE RESPONSE IS 'ResponseModel': - if isinstance(response, ResponseModel): - response.api_version = kwargs.get("api_version") - response.log_id = kwargs.get("log_id") - - # Extract the response to log: - response_to_log = "not logged" - http_code_to_log = 200 - if log_output: - if isinstance(response, ResponseModel): response_to_log, http_code_to_log = response.for_quart() - elif isinstance(response, tuple): response_to_log, http_code_to_log = response - else: response_to_log, http_code_to_log = str(response), 200 - response_to_log = summarize_variable( - response_to_log, - expand = log_output, - sensitive_keys = sensitive_keys - ) - - # Try to get the information about the request. - # There will be no data in any of these if the decorator was used to catch start-up and shut-down events. - request_method = None - request_url = None - request_route = None - try: - request_method = f"{request.method}" - request_url = f"{request.url}" - request_route = str(request.url_rule.rule) - except: pass - - # Redact the sensitive keys: - if sensitive_keys: - for k in sensitive_keys: - for var in ["inbound_headers", "inbound_data"]: - try: kwargs[var][k] = len(str(kwargs[var][k])) * "*" - except: pass - - # Construct the log: - api_log = APILogModel( - pid = PROCESS_ID, - ppid = PARENT_PROCESS_ID, - project = project, - log = log_type, - operation = operation, - apiVer = kwargs.get("api_version"), - logId = kwargs.get("log_id"), - logChain = kwargs.get("log_chain", kwargs.get("inbound_headers", {}).get("X-Log-Chain")), - method = request_method, - url = request_url, - route = request_route, - ts = request_ts, - tat = time.perf_counter() - start_ts, - cpuTime = time.process_time() - cpu_start_ts, - headers = kwargs.get("inbound_headers"), - data = summarize_variable( - kwargs.get("inbound_data"), - expand = log_input, - sensitive_keys = sensitive_keys - ), - files = { - k: { - "name": v["name"], - "size": v["size"] - } for k, v in kwargs.get("inbound_files", {}).items() - }, - exception = None if exception is None else describe_exception(exception), - response = response_to_log, - httpCode = http_code_to_log - ) - - # Write the log: - app_attr = getattr(current_app, attr_name) - inserted_id = await app_attr.insert_one( - collection = collection, - document = api_log.model_dump() - ) - - # Return the response from the wrapped function. - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def log_chain_to_mongo(attr_name): - - """ - Use this to run everything in the context of the logging decorator. Everything that is decorated with the - custom decorator will be logged if this is used. - :param attr_name: The name of the db connection (an instance of 'AsyncMongo' or 'AsyncMongoStorage') - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # Fetch the attribute and get it to log the whole chain: - async with AsyncLoggerContext.logging_context( - logger = AsyncMongoLogger(getattr(current_app, attr_name)), - log_chain = kwargs.get("log_chain") - ): - response = await func(*args, **kwargs) - - # Done here: - kwargs["decorator_count"] -= 1 - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def should_not_be_under_maintenance(attr_name): - - """ - Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create - a boolean variable within the scope of the 'current_app' for this to work. An alternate to this is to set the value - in an environment variable named 'IS_UNDER_MAINTENANCE' to a string value of either 'True' or 'False' for - multi-worker deployments. - :param attr_name: The name of the boolean variable that will hold the information about the app being under - maintenance. If its value is True at the time of checking, the incoming request will be rejected. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # Get the attribute and check if it indicates that the app is under maintenance, - # call the wrapped function if not under maintenance: - app_attr = getattr(current_app, attr_name) - env_attr = True if os.environ.get("IS_UNDER_MAINTENANCE", "False").lower() == "true" else False - if app_attr or env_attr: - response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() - else: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def only_whitelisted_ips(attr_name): - - """ - Use this decorator to reject any requests coming from unauthorized IPs. The list of IP addresses to allow must be - in a list that is accessible in the context of 'current_app'. - :param attr_name: The name of the boolean variable that will hold the information about the app being under - maintenance. If its value is True at the time of checking, the incoming request will be rejected. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # Get the attribute and check if the request's IP is in the permitted list: - app_attr = getattr(current_app, attr_name) - if request.remote_addr not in app_attr: - return ResponseModel( - status_code = StatusCodes.AUTHORIZATION_FAILED, - message = "bad ip", - http_code = HttpCodes.UNAUTHORIZED - ).for_quart() - - # Now that we have checked that the IP is permitted, - # we are ready to run the function that we are wrapping: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def limit_rate( - attr_name, - rate_limit: int = 5, - seconds: float = 1.0, - message = None, - header_keys: list = None, - data_keys: list = None, - allow_if_exception = False, - count_for_http_codes = None -): - - """ - Use this decorator to apply rate-limiting to incoming requests. - SUGGESTION: WHEN STACKING UP MANY RATE LIMITS, PUT THE SMALLEST TIME PERIOD ON TOP AND LARGEST TIME PERIOD AT THE - BOTTOM. THIS ENSURES PROPER FUNCTIONALITY. - WARNING: TIMING STARTS WHEN THE FIRST PERMITTED REQUEST GOES THROUGH. THIS MEANS THAT, IF YOU HAVE A PER-DAY LIMIT, - AND YOU START MAKING REQUESTS AT 11:00 PM AND EXHAUST YOUR LIMIT AT 11:59 PM, YOUR LIMIT WILL BE REPLENISHED AT - 11:00 PM OF THE NEXT DAY, NOT AT 12:00 AM. - :param attr_name: The name of the variable that holds the instance of 'AsyncRedisCache'. Should be available in the - context of 'current_app'. - :param rate_limit: The number of requests per unit time. - :param seconds: The time period in which the rate limit is to be applied. - :param message: The custom message to respond with. - :param header_keys: The keys in the header to consider when apply rate limits (like 'sessionToken'). - :param data_keys: The keys in the header to consider when apply rate limits (like 'sessionToken'). - :param allow_if_exception: In case Redis is unresponsive, would you prefer allowing the request to pass through or - would you prefer the request getting blocked. - :param count_for_http_codes: If this is provided, the counter will be incremented only if the response code was one - of these values. If not provided, all requests will be counted. This can be used in cases when you want to count - only when the request was successfully served. - :return: The decorator factory. - """ - - # Param-cleaning: - rate_limit = max(1, rate_limit) - if header_keys is None: header_keys = [] - if data_keys is None: data_keys = [] - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # Note down the combination of values requested and the limits prescribed: - params = {"route": str(request.url_rule), "header": {}, "data": {}, "limit": rate_limit, "seconds": seconds} - for key in header_keys: params["header"][key] = kwargs["inbound_headers"].get(key) - for key in data_keys: params["data"][key] = kwargs["inbound_data"].get(key) - - # Now make a unique key from this combination: - params_json = json.to_string(params, no_space = True) - sha256_hash = hashlib.sha256() - sha256_hash.update(params_json.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # We first get the value of the counter: - app_attr = getattr(current_app, attr_name) - counter_value = await app_attr.count(base64_key, value = 1, expiry = seconds) - - # If any exception occurred in getting the count, - # and exceptions haven't been allowed: - if counter_value is None and not allow_if_exception: - response = ResponseModel( - status_code = StatusCodes.RATE_LIMIT_EXCEEDED, - message = "Please contact admin (E)" - ) - - # If the rate-limit has already been crossed, - # or when the counter was not fetched but exceptions are allowed: - elif (counter_value or 0) > rate_limit: - response = ResponseModel( - status_code = StatusCodes.RATE_LIMIT_EXCEEDED, - message = message or ", ".join([ - f"rate limit: {rate_limit} in {humanize.naturaldelta(datetime.timedelta(seconds = seconds))}", - f"this is your {humanize.ordinal(counter_value)} request in the given period" - ]) - ) - - # If the rate-limit hasn't been crossed: - else: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # If we have been told to count only for specific status codes, - # and if the HTTP code of this response is not in the list of codes, we reduce the counter by one: - if count_for_http_codes: - http_code = 200 # ... Default assumption. - if isinstance(response, (list, tuple, set)): http_code = response[1] - elif isinstance(response, ResponseModel): http_code = response.http_code.value - if http_code not in count_for_http_codes: - await app_attr.count(base64_key, value = -1, expiry = seconds) - - # Return the response from the function call: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): - - """ - Use this decorator to handle prematurely terminated requests. If your clean-up function needs access to variables, - consider using 'g' to hold data in the scope of the request. - :param cleanup_func: The function to call when the cancelled request needs graceful handling. - :param cleanup_coro: The coroutine to call when the cancelled request needs graceful handling. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - try: - - # we are ready to run the function that we are wrapping: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - # In case the client closes the connection pre-maturely: - except asyncio.CancelledError as exception: - kwargs["decorator_count"] -= 1 - if hasattr(current_app, "printer"): getattr(current_app, "printer")(exception) - if cleanup_func is not None: cleanup_func() - if cleanup_coro is not None: await cleanup_coro() - return ResponseModel( - api_version = kwargs.get("api_version"), - status_code = StatusCodes.CLIENT_CLOSED_REQUEST - ).for_quart() - - # We propagate any other kind of exception: - except Exception as exception: - kwargs["decorator_count"] -= 1 - raise - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def handle_failed_request(cleanup_func = None, cleanup_coro = None): - - """ - Use this decorator to handle requests that fail due to exceptions. If your clean-up function needs access to - variables, consider using 'g' to hold data in the scope of the request. THIS DECORATOR IS NOT MEANT TO SUPRESS - EXCEPTIONS. IT IS, INSTEAD, MEANT TO PERFORM CLEAN-UP AND PROPAGATE THE EXCEPTION. - :param cleanup_func: The function to call when the cancelled request needs graceful handling. - :param cleanup_coro: The coroutine to call when the cancelled request needs graceful handling. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - try: - - # we are ready to run the function that we are wrapping: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Done here: - if ( - kwargs["decorator_count"] == 0 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - # In case the client closes the connection pre-maturely: - except Exception as exception: - kwargs["decorator_count"] -= 1 - if hasattr(current_app, "printer"): getattr(current_app, "printer")(exception) - if cleanup_func is not None: cleanup_func() - if cleanup_coro is not None: await cleanup_coro() - raise exception - - return wrapper - - return decorator - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/api/codes.py b/utils_v2/api/codes.py deleted file mode 100644 index 5773cc9..0000000 --- a/utils_v2/api/codes.py +++ /dev/null @@ -1,164 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 12th Sept., 2024 - - OBJECTIVE: - - To maintain all status codes in one place. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -from enum import Enum, unique - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -@unique -class HttpCodes(Enum): - - """ - Commonly used standard HTTP status codes. - Can be sent after an API Call is processed. - Refer to: https://http.dev/status - NOTE: THIS LIST IS NOT EXHAUSTIVE! - """ - - # 1XX - Informational: - CONTINUE = 100 # .............. https://http.dev/102 - SWITCHING_PROTOCOLS = 101 # ... https://http.dev/101 - PROCESSING = 102 # ............ https://http.dev/102 - EARLY_HINTS = 103 # ........... https://http.dev/103 - - # 2XX - Success: - SUCCESS = 200 # .................. https://http.dev/200 - CREATED = 201 # .................. https://http.dev/201 - ACCEPTED = 202 # ................. https://http.dev/202 - NON_AUTHORITATIVE_INFO = 203 # ... https://http.dev/203 - NO_CONTENT = 204 # ............... https://http.dev/204 - RESET_CONTENT = 205 # ............ https://http.dev/205 - PARTIAL_CONTENT = 206 # .......... https://http.dev/206 - MULTI_STATUS = 207 # ............. https://http.dev/207 - ALREADY_REPORTED = 208 # ......... https://http.dev/208 - THIS_IS_FINE = 218 # ............. https://http.dev/218 - IM_USED = 226 # .................. https://http.dev/226 - - # 3XX - Redirection: - MULTIPLE_CHOICES = 300 # ..... https://http.dev/300 - MOVED_PERMANENTLY = 301 # .... https://http.dev/301 - MOVED_TEMPORARILY = 302 # .... https://http.dev/302 - SEE_OTHER = 303 # ............ https://http.dev/303 - NOT_MODIFIED = 304 # ......... https://http.dev/304 - USE_PROXY = 305 # ............ https://http.dev/305 - SWITCH_PROXY = 306 # ......... https://http.dev/306 - TEMPORARY_REDIRECT = 307 # ... https://http.dev/307 - PERMANENT_REDIRECT = 308 # ... https://http.dev/308 - - # 4XX - Client Errors: - BAD_REQUEST = 400 # ..................... https://http.dev/401 - UNAUTHORIZED = 401 # .................... https://http.dev/401 - PAYMENT_REQUIRED = 402 # ................ https://http.dev/402 - FORBIDDEN = 403 # ....................... https://http.dev/403 - NOT_FOUND = 404 # ....................... https://http.dev/404 - METHOD_NOT_ALLOWED = 405 # .............. https://http.dev/405 - NOT_ACCEPTABLE = 406 # .................. https://http.dev/406 - PROXY_AUTH_REQUIRED = 407 # ............. https://http.dev/407 - REQUEST_TIMEOUT = 408 # ................. https://http.dev/408 - CONFLICT = 409 # ........................ https://http.dev/409 - GONE = 410 # ............................ https://http.dev/410 - LENGTH_REQUIRED = 411 # ................. https://http.dev/411 - PRECONDITION_FAILED = 412 # ............. https://http.dev/412 - PAYLOAD_TOO_LARGE = 413 # ............... https://http.dev/413 - URI_TOO_LONG = 414 # .................... https://http.dev/414 - UNSUPPORTED_MEDIA_TYPE = 415 # .......... https://http.dev/415 - PAGE_EXPIRED = 419 # .................... https://http.dev/419 - TOO_MANY_REQUESTS = 429 # ............... https://http.dev/429 - UNAVAILABLE_FOR_LEGAL_REASONS = 451 # ... https://http.dev/451 - INVALID_TOKEN = 498 # ................... https://http.dev/498 - CLIENT_CLOSED_REQUEST = 499 # ........... https://http.dev/499 - - # 5XX - Server Errors: - INTERNAL_SERVER_ERROR = 500 # ........... https://http.dev/500 - NOT_IMPLEMENTED = 501 # ................. https://http.dev/501 - BAD_GATEWAY = 502 # ..................... https://http.dev/502 - SERVICE_UNAVAILABLE = 503 # ............. https://http.dev/503 - GATEWAY_TIMEOUT = 504 # ................. https://http.dev/504 - HTTP_VERSION_NOT_SUPPORTED = 505 # ...... https://http.dev/505 - VARIANT_ALSO_NEGOTIATES = 506 # ......... https://http.dev/506 - INSUFFICIENT_STORAGE = 507 # ............ https://http.dev/507 - LOOP_DETECTED = 508 # ................... https://http.dev/508 - BANDWIDTH_LIMIT_EXCEEDED = 509 # ........ https://http.dev/509 - WEB_SERVER_DOWN = 521 # ................. https://http.dev/521 - ORIGIN_IS_UNREACHABLE = 523 # ........... https://http.dev/523 - SERVICE_IS_OVERLOADED = 529 # ........... https://http.dev/529 - NETWORK_READ_TIMEOUT_ERROR = 598 # ...... https://http.dev/598 - NETWORK_CONNECT_TIMEOUT_ERROR = 599 # ... https://http.dev/599 - - -# --------------------------------------------------------------------------------------------------------------------- - - -@unique -class StatusCodes(Enum): - - """ - To be used internally within the context of your service. Customize these to match your service. - The format is: (SUCCESS_INDICATOR, INTERNAL_NUMERIC_CODE, HTTP_CODE) - Example: (True, 1, 200) - """ - - # Legacy Codes: - OK = (True, 1, HttpCodes.SUCCESS.value) - FAILED = (False, 0, HttpCodes.INTERNAL_SERVER_ERROR.value) - PARTIAL_SUCCESS = (True, 2, HttpCodes.PARTIAL_CONTENT.value) - PARTIAL_FAILURE = (False, 3, HttpCodes.PARTIAL_CONTENT.value) - - # Authentication Codes: - LOGGED_IN_SUCCESSFULLY = (True, 200, HttpCodes.SUCCESS.value) - LOGIN_FAILED = (False, 201, HttpCodes.UNAUTHORIZED.value) - INVALID_SESSION_TOKEN = (False, 202, HttpCodes.UNAUTHORIZED.value) - AUTHENTICATION_DETAILS_INCOMPLETE = (False, 203, HttpCodes.BAD_REQUEST.value) - - # Authorization Codes: - AUTHORIZED_SUCCESSFULLY = (True, 300, HttpCodes.SUCCESS.value) - NOT_ALLOWED = (False, 301, HttpCodes.FORBIDDEN.value) - AUTHORIZATION_DETAILS_INCOMPLETE = (False, 302, HttpCodes.BAD_REQUEST.value) - AUTHORIZATION_FAILED = (False, 303, HttpCodes.UNAUTHORIZED.value) - - # General failures: - DOWN_FOR_MAINTENANCE = (False, 800, HttpCodes.SERVICE_UNAVAILABLE.value) - UNKNOWN_ERROR = (False, 801, HttpCodes.INTERNAL_SERVER_ERROR.value) - DATA_INCOMPLETE = (False, 802, HttpCodes.BAD_REQUEST.value) - HEADERS_INCOMPLETE = (False, 803, HttpCodes.BAD_REQUEST.value) - FILES_MISSING = (False, 804, HttpCodes.BAD_REQUEST.value) - CLIENT_CLOSED_REQUEST = (False, 805, HttpCodes.CLIENT_CLOSED_REQUEST.value) - - # Validation failure: - DATA_VALIDATION_FAILURE = (False, 900, HttpCodes.BAD_REQUEST.value) - RATE_LIMIT_EXCEEDED = (False, 901, HttpCodes.TOO_MANY_REQUESTS.value) diff --git a/utils_v2/api/log.py b/utils_v2/api/log.py deleted file mode 100644 index 8c8c13e..0000000 --- a/utils_v2/api/log.py +++ /dev/null @@ -1,138 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 12th Sept., 2024 - - OBJECTIVE: - - To have a structure to the response sent from the API calls. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# System-level activities: -import distro -import socket -import platform - -# For data-modelling: -from pydantic import BaseModel, Field -from typing import Any, Optional, List, Literal - -# To work with date and time: -import datetime - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# Info for logging that will stay constant during runtime: -SERVER_HOSTNAME = str(socket.gethostname()) -PLATFORM_INFO = platform.uname() -HOST_OS = str(distro.name(True)) -HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class APILogModel(BaseModel): - - # To identify the machine the code is running on. - # DO NOT MODIFY THESE: - hostname: str = SERVER_HOSTNAME - os: str = HOST_OS - cpu: str = HOST_CPU - # Can modify these: - pid: Optional[Any] = None - ppid: Optional[Any] = None - - # To identify the project and actions: - project: Optional[str] = None - log: str - operation: str - apiVer: Optional[str] = None - logId: Optional[str] = None - logChain: Optional[str] = None - - # Timing metrics: - ts: datetime.datetime - tat: float - cpuTime: float - - # To understand the request that came in: - method: Optional[str] = None - url: Optional[str] = None - route: Optional[str] = None - headers: Optional[Any] = None - data: Optional[Any] = None - files: Optional[Any] = None - - # To understand the output that went out: - exception: Optional[Any] = None - response: Optional[Any] = None - httpCode: Optional[int] = None - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - my_log = APILogModel( - log = "internal" - ) - - print(my_log) diff --git a/utils_v2/api/response.py b/utils_v2/api/response.py deleted file mode 100644 index 02e3d72..0000000 --- a/utils_v2/api/response.py +++ /dev/null @@ -1,133 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 12th Sept., 2024 - - OBJECTIVE: - - To have a structure to the response sent from the API calls. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# For data-modelling: -from pydantic import BaseModel -from typing import Any, Optional, List - -# My utils: -from utils_v2.api.codes import StatusCodes, HttpCodes - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class ResponseModel(BaseModel): - - """ - A model for how the response should be when developing API endpoints. - """ - - # The fields that you want in your response: - status_code: StatusCodes - message: Optional[str | List] = None - data: Optional[Any] = None - seconds: Optional[float | int] = None - log_id: Optional[str] = None - http_code: Optional[HttpCodes] = None - api_version: Optional[str] = None - - def for_quart(self): - - """ - Call this when you are using either Flask or Quart as your framework. - :return: The output as expected by Flask and Quart. - """ - - # Construct the basic structure: - response_dict = { - "status": 1 if self.status_code.value[0] else 0, - "code": self.status_code.value[1], - "message": self.message or self.status_code.name.replace("_", " ").lower(), - "data": self.data, - "apiVer": self.api_version - } - - # Now add the additional fields: - if self.seconds is not None: response_dict["seconds"] = self.seconds - if self.log_id is not None: response_dict["logId"] = self.log_id - - # Figure out the HTTP code: - response_http_code = self.http_code.value if self.http_code is not None else self.status_code.value[2] - - # Done here: - return response_dict, response_http_code - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - my_response = ResponseModel( - status_code = StatusCodes.RATE_LIMIT_EXCEEDED - ) - my_response.log_id = "abc123" - - print(my_response.for_quart()) diff --git a/utils_v2/cache/__init__.py b/utils_v2/cache/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/cache/__pycache__/__init__.cpython-310.pyc b/utils_v2/cache/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 39ed6b7..0000000 Binary files a/utils_v2/cache/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc b/utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc deleted file mode 100644 index 6a0d7d1..0000000 Binary files a/utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/cache/async_redis_cache.py b/utils_v2/cache/async_redis_cache.py deleted file mode 100644 index a0ab8c0..0000000 --- a/utils_v2/cache/async_redis_cache.py +++ /dev/null @@ -1,497 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 11th April, 2024 - - OBJECTIVE: - - To provide an easy way to work with '.json' files in an async manner. - - REFERENCES: - - 1) https://www.w3schools.com/python/python_json.asp - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To use redis: -import redis.asyncio as redis - -# Other utils: -from utils_v2.string import json - -# For debugging: -from icecream import IceCreamDebugger - -# To make a decorator: -from functools import wraps - -# For hashing and shortening the hash: -import hashlib -import base64 - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** WRAPPERS *** -# ***** **** -# ***************************************************************************************************************** - - -def cache_it(cache = None, expiry = 120): - - """ - This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. - If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the - cache instead of going through the whole function again. - :param cache: The instance of 'AsyncRedisCache'. - :param expiry: The time in seconds after which the cached data must be cleared. - :return: The decorator that automatically caches your data. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # We first use the name of the function and the inputs given to it to generate a key for Redis with the - # simple hashing and shortening by way of base64 strings: - inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) - sha256_hash = hashlib.sha256() - sha256_hash.update(inputs_given.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # Now we check if we have the value in cache: - try: response = await cache.get(base64_key, raise_exception = True) - - # If the key doesn't exist, we pass through the function and store the results. - except: - response = await func(*args, **kwargs) - await cache.set(key = base64_key, value = response, expiry = expiry) - - # Return the response from the wrapped function. - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def cache_class_methods(attr_name, expiry = 120): - - """ - This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. - If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the - cache instead of going through the whole function again. - :param attr_name: The name of the variable that has an instance of "AsyncRedisCache". - :param expiry: The time in seconds after which the cached data must be cleared. - :return: The decorator that automatically caches your data. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(self, *args, **kwargs): - - # Get the cache object first: - cache_obj = getattr(self, attr_name) - - # We first use the name of the function and the inputs given to it to generate a key for Redis with the - # simple hashing and shortening by way of base64 strings: - inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) - sha256_hash = hashlib.sha256() - sha256_hash.update(inputs_given.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # Now we check if we have the value in cache: - try: response = await cache_obj.get(base64_key, raise_exception = True) - - # If the key doesn't exist, we pass through the function and store the results. - except: - response = await func(self, *args, **kwargs) - await cache_obj.set(key = base64_key, value = response, expiry = expiry) - - # Return the response from the wrapped function. - return response - - return wrapper - - return decorator - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncRedisCache: - - # Handle datatypes when setting values to cache: - __set_converters = { - "set": lambda x: list(x), - "tuple": lambda x: list(x), - "complex": lambda x: {"r": x.real, "i": x.imag} - } - - # Handle datatypes when getting from cache: - __get_converters = { - "set": lambda x: set(x), - "tuple": lambda x: tuple(x), - "complex": lambda x: complex(x["r"], x["i"]) - } - - def __init__( - self, - connection_string, - ping_counter = 1_000, - debug = False, - debug_prefix = "R-Cache | " - ): - - """ - Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools, - strings, dicts, lists, sets, and tuples :) - :param connection_string: The connection URL for connecting to Redis. - :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection. - :param debug: Whether, or not, you want to show debugging messages from the start. - :param debug_prefix: The prefix text to show with the debugging messages. - """ - - # Note down the configuration: - self.__client = None - self.__ping_counter = ping_counter - self.__requests_since_last_ping = 0 - self.__connection_string = connection_string - - # For debugging: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - def enable_debug(self): - self.__printer.enable() - - def disable_debug(self): - self.__printer.disable() - - @staticmethod - def make_key(*args, **kwargs): - - """ - Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the - inputs stay the same, the output stays the same. - :param args: Any number of args that you would like to use to generate the key. - :param kwargs: Any number of kwargs that you would like to use to generate the key. - :return: A string that can be used as a key to store values on Redis. - """ - - # Take everything into one plain text string: - plain_text = "".join(str(a) for a in args) - plain_text += json.to_string(kwargs, no_space = True) - - # Hash the plain text value, and create a key from it: - sha256_hash = hashlib.sha256() - sha256_hash.update(plain_text.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # Done here: - return base64_key - - def add_converter( - self, - type_name, - set_converter_func, - get_converter_func, - ): - - """ - To add custom data converters to handle new values. - RULE: Each of the converter functions must take in exactly on argument. - :param type_name: The name of the type of the object to handle. HINT: type(obj).__name__. - :param set_converter_func: A converter function that converts from the custom datatype to a datatype that Redis - can work with. Try converting to an bool, int, float, str, list or dict. - :param get_converter_func: The inverse of the set converter. This function will be used to convert from a - datatype that Redis can work with to the custom datatype. - :return: None. - """ - - self.__set_converters[type_name] = lambda x: set_converter_func(x) - self.__get_converters[type_name] = lambda x: get_converter_func(x) - - async def connect(self): - - """ - To make an asynchronous connection request to the Redis server to establish a connection. - :return: True if connected, else False. - """ - - try: - self.__client = redis.from_url( - self.__connection_string - ) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def disconnect(self): - - """ - Close the connection to the Redis server. - :return: True if successful, else False. - """ - - if self.__client is not None: - try: await self.__client.close() - except Exception as exception: - self.__printer(exception) - return False - return True - return True - - async def ensure_connection(self): - - """ - To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected. - :return: True if connected, else False. - """ - - # If we are not connected, we try to establish a connection: - if self.__client is None: return await self.connect() - - # Else we check if we are connected. If not, we try to connect. - # But we check only once in a while. In the meantime, we assume that we are connected. - elif self.__requests_since_last_ping > self.__ping_counter: - try: - await self.__client.ping() - self.__requests_since_last_ping = 0 - return True - except Exception as exception: - self.__printer(exception) - return await self.connect() - else: - self.__requests_since_last_ping += 1 - return True - - async def set(self, key, value, expiry: float = 120, raise_exception = False): - - """ - Saves some value to the cache. - :param key: The key with which the data will be stored and retrieved. - :param value: The value to store. - :param expiry: The time in seconds after which the data will expire. Must be a positive number. - :param raise_exception: If you want to raise an exception if the process fails. - :return: True if cached, else False. - """ - - await self.ensure_connection() - - try: - - # We handle conversion for unsupported Pythonic datatypes: - original_type = type(value).__name__ - converter = self.__set_converters.get(original_type) - if converter is not None: value = converter(value) - - # Here we actually try to store the data: - response = await self.__client.setex( - key, - int(expiry), - json.to_string({ - "value": value, - "type": original_type - }, indent = 0, no_space = False) - ) - return response - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return False - - async def get(self, key, raise_exception = False, on_fail = None): - - """ - Retrieve the cached value. - :param key: The key with which the data was saved. - :param raise_exception: If you want to raise an exception if the process fails. - :param on_fail: What to return if the process fails due to an exception. - :return: The retrieved data or null if not found. - """ - - await self.ensure_connection() - - try: - data = await self.__client.get(key) - data = data.decode("utf-8") - data = json.from_string(data) - converter = self.__get_converters.get(data["type"]) - if converter is not None: return converter(data["value"]) - return data["value"] - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return on_fail - - async def delete(self, key, raise_exception = False): - - """ - Prematurely delete the value from the cache before it expires. - :param key: The key with which the data was saved. - :param raise_exception: If you want to raise an exception if the process fails. - :return: True if deleted, else False. - """ - - await self.ensure_connection() - - try: - response = await self.__client.delete(key) - return True if response else False - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return False - - async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False): - - """ - To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the - counter will be created and the value will be incremented from 0. - :param key: The name of the counter. - :param value: The amount to increment the value by. Send negative values to count backwards. - :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created. - This value has to be an integer. If a float is passed, the value will be rounded off. - :param raise_exception: If you want to raise an exception if the process fails. - :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed. - """ - - await self.ensure_connection() - - try: - - already_existed = await self.__client.exists(key) - new_value = await self.__client.incrby(key, value) - if expiry and not already_existed: await self.__client.expire(key, int(expiry)) - return new_value - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return None - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - - def set_complex(x): - return {"r": x.real, "i": x.imag} - - def get_complex(x): - return complex(x["r"], x["i"]) - - async def main(): - - my_cache = AsyncRedisCache( - connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0", - ping_counter = 100, - debug = False - ) - - my_cache.add_converter( - type_name = type(2j).__name__, - set_converter_func = set_complex, - get_converter_func = get_complex - ) - - value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624") - print("GET:", value) - print("TYP:", type(value), end = "\n\n") - - # success = await my_cache.set( - # key = "name", - # value = {"first": "John", "last": "Doe"}, - # expiry = 10 - # ) - # print("SET:", success, end = "\n\n") - # - # value = await my_cache.get(key = "name") - # print("GET:", value) - # print("TYP:", type(value), end = "\n\n") - # - # success = await my_cache.delete(key = "name") - # print("DEL:", success, end = "\n\n") - # - # value = await my_cache.get(key = "cnt") - # print("GET:", value) - # print("TYP:", type(value), end = "\n\n") - # - # await my_cache.delete(key = "cnt") - # for _ in range(50): - # await asyncio.sleep(1.0) - # counter = await my_cache.count(key = "cnt", value = 1, expiry = 10) - # print("COUNTER:", counter) - - asyncio.run(main()) diff --git a/utils_v2/cache/async_redis_cache_v2.py b/utils_v2/cache/async_redis_cache_v2.py deleted file mode 100644 index 7c82797..0000000 --- a/utils_v2/cache/async_redis_cache_v2.py +++ /dev/null @@ -1,458 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Saturday, 26th Oct., 2024 - - OBJECTIVE: - - To provide an easy way to cache data for fast access. This version has the change that it can handle custom - serializers by way of dependency injection. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To use redis: -import redis.asyncio as redis - -# Other utils: -from utils_v2.string import json -from utils_v2.serialization.pickle_serializer import PickleSerializer - -# For debugging: -from icecream import IceCreamDebugger - -# To make a decorator: -from functools import wraps - -# For hashing and shortening the hash: -import hashlib -import base64 - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** WRAPPERS *** -# ***** **** -# ***************************************************************************************************************** - - -def cache_it(cache = None, expiry = 120): - - """ - This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. - If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the - cache instead of going through the whole function again. - :param cache: The instance of 'AsyncRedisCache'. - :param expiry: The time in seconds after which the cached data must be cleared. - :return: The decorator that automatically caches your data. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # We first use the name of the function and the inputs given to it to generate a key for Redis with the - # simple hashing and shortening by way of base64 strings: - inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) - sha256_hash = hashlib.sha256() - sha256_hash.update(inputs_given.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # Now we check if we have the value in cache: - try: response = await cache.get(base64_key, raise_exception = True) - - # If the key doesn't exist, we pass through the function and store the results. - except: - response = await func(*args, **kwargs) - await cache.set(key = base64_key, value = response, expiry = expiry) - - # Return the response from the wrapped function. - return response - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------------------------------------------------- - - -def cache_class_methods(attr_name, expiry = 120): - - """ - This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. - If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the - cache instead of going through the whole function again. - :param attr_name: The name of the variable that has an instance of "AsyncRedisCache". - :param expiry: The time in seconds after which the cached data must be cleared. - :return: The decorator that automatically caches your data. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(self, *args, **kwargs): - - # Get the cache object first: - cache_obj = getattr(self, attr_name) - - # We first use the name of the function and the inputs given to it to generate a key for Redis with the - # simple hashing and shortening by way of base64 strings: - inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) - sha256_hash = hashlib.sha256() - sha256_hash.update(inputs_given.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # Now we check if we have the value in cache: - try: response = await cache_obj.get(base64_key, raise_exception = True) - - # If the key doesn't exist, we pass through the function and store the results. - except: - response = await func(self, *args, **kwargs) - await cache_obj.set(key = base64_key, value = response, expiry = expiry) - - # Return the response from the wrapped function. - return response - - return wrapper - - return decorator - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncRedisCache: - - def __init__( - self, - connection_string, - serializer = None, - ping_counter = 1_000, - debug = False, - debug_prefix = "R-Cache | " - ): - - """ - Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools, - strings, dicts, lists, sets, and tuples :) - :param connection_string: The connection URL for connecting to Redis. - :param serializer: The serializer to use. - :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection. - :param debug: Whether, or not, you want to show debugging messages from the start. - :param debug_prefix: The prefix text to show with the debugging messages. - """ - - # Note down the configuration: - self.__client = None - self.__serializer = serializer or PickleSerializer() - self.__ping_counter = ping_counter - self.__requests_since_last_ping = 0 - self.__connection_string = connection_string - - # For debugging: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - def enable_debug(self): - self.__printer.enable() - - def disable_debug(self): - self.__printer.disable() - - @staticmethod - def make_key(*args, **kwargs): - - """ - Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the - inputs stay the same, the output stays the same. - :param args: Any number of args that you would like to use to generate the key. - :param kwargs: Any number of kwargs that you would like to use to generate the key. - :return: A string that can be used as a key to store values on Redis. - """ - - # Take everything into one plain text string: - plain_text = "".join(str(a) for a in args) - plain_text += json.to_string(kwargs, no_space = True) - - # Hash the plain text value, and create a key from it: - sha256_hash = hashlib.sha256() - sha256_hash.update(plain_text.encode("utf-8")) - hashed_key = sha256_hash.digest() - base64_key = base64.b64encode(hashed_key).decode("utf-8") - - # Done here: - return base64_key - - async def connect(self): - - """ - To make an asynchronous connection request to the Redis server to establish a connection. - :return: True if connected, else False. - """ - - try: - self.__client = redis.from_url( - self.__connection_string - ) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def disconnect(self): - - """ - Close the connection to the Redis server. - :return: True if successful, else False. - """ - - if self.__client is not None: - try: await self.__client.close() - except Exception as exception: - self.__printer(exception) - return False - return True - return True - - async def ensure_connection(self): - - """ - To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected. - :return: True if connected, else False. - """ - - # If we are not connected, we try to establish a connection: - if self.__client is None: return await self.connect() - - # Else we check if we are connected. If not, we try to connect. - # But we check only once in a while. In the meantime, we assume that we are connected. - elif self.__requests_since_last_ping > self.__ping_counter: - try: - await self.__client.ping() - self.__requests_since_last_ping = 0 - return True - except Exception as exception: - self.__printer(exception) - return await self.connect() - else: - self.__requests_since_last_ping += 1 - return True - - async def set(self, key, value, expiry: float = None, raise_exception = False): - - """ - Saves some value to the cache. If an expiry is specified, the data will be deleted after that many seconds. - :param key: The key with which the data will be stored and retrieved. - :param value: The value to store. - :param expiry: The time in seconds after which the data will expire. Must be a positive number. - :param raise_exception: If you want to raise an exception if the process fails. - :return: True if cached, else False. - """ - - await self.ensure_connection() - - try: - - # Here we actually try to set the data: - data = self.__serializer.serialize(value) - if expiry: response = await self.__client.setex(key, int(expiry), data) - else: response = await self.__client.set(key, data) - return response - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return False - - async def get(self, key, raise_exception = False, on_fail = None): - - """ - Retrieve the cached value. - :param key: The key with which the data was saved. - :param raise_exception: If you want to raise an exception if the process fails. - :param on_fail: What to return if the process fails due to an exception. - :return: The retrieved data or null if not found. - """ - - await self.ensure_connection() - - try: - - # Here we try to fetch the data: - data = await self.__client.get(key) - data = self.__serializer.deserialize(data) - return data - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return on_fail - - async def delete(self, key, raise_exception = False): - - """ - Prematurely delete the value from the cache before it expires. - :param key: The key with which the data was saved. - :param raise_exception: If you want to raise an exception if the process fails. - :return: True if deleted, else False. - """ - - await self.ensure_connection() - - try: - - # Try to manually delete the key before expiry: - response = await self.__client.delete(key) - return True if response else False - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return False - - async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False): - - """ - To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the - counter will be created and the value will be incremented from 0. - :param key: The name of the counter. - :param value: The amount to increment the value by. Send negative values to count backwards. - :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created. - This value has to be an integer. If a float is passed, the value will be rounded off. - :param raise_exception: If you want to raise an exception if the process fails. - :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed. - """ - - await self.ensure_connection() - - try: - - already_existed = await self.__client.exists(key) - new_value = await self.__client.incrby(key, value) - if expiry and not already_existed: await self.__client.expire(key, int(expiry)) - return new_value - - except Exception as exception: - self.__printer(exception) - if raise_exception: raise - else: return None - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - - def set_complex(x): - return {"r": x.real, "i": x.imag} - - def get_complex(x): - return complex(x["r"], x["i"]) - - async def main(): - - my_cache = AsyncRedisCache( - connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0", - ping_counter = 100, - debug = False - ) - - my_cache.add_converter( - type_name = type(2j).__name__, - set_converter_func = set_complex, - get_converter_func = get_complex - ) - - value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624") - print("GET:", value) - print("TYP:", type(value), end = "\n\n") - - # success = await my_cache.set( - # key = "name", - # value = {"first": "John", "last": "Doe"}, - # expiry = 10 - # ) - # print("SET:", success, end = "\n\n") - # - # value = await my_cache.get(key = "name") - # print("GET:", value) - # print("TYP:", type(value), end = "\n\n") - # - # success = await my_cache.delete(key = "name") - # print("DEL:", success, end = "\n\n") - # - # value = await my_cache.get(key = "cnt") - # print("GET:", value) - # print("TYP:", type(value), end = "\n\n") - # - # await my_cache.delete(key = "cnt") - # for _ in range(50): - # await asyncio.sleep(1.0) - # counter = await my_cache.count(key = "cnt", value = 1, expiry = 10) - # print("COUNTER:", counter) - - asyncio.run(main()) diff --git a/utils_v2/database/__init__.py b/utils_v2/database/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/database/__pycache__/__init__.cpython-310.pyc b/utils_v2/database/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 906606a..0000000 Binary files a/utils_v2/database/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc b/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc deleted file mode 100644 index c0b66d1..0000000 Binary files a/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc b/utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc deleted file mode 100644 index eda566e..0000000 Binary files a/utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/database/async_firebase.py b/utils_v2/database/async_firebase.py deleted file mode 100644 index a8065a1..0000000 --- a/utils_v2/database/async_firebase.py +++ /dev/null @@ -1,504 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - Bhushan Thakkar - - DATE: - - Friday, 19th April, 2024 - - OBJECTIVE: - - To provide an easy interface to work with Firebase. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -# --- -import sys -sys.path.append(".") -sys.path.append("..") - -# For async operations: -# --- -import asyncio - -# For system-level activities: -# --- -import os - -# My async utils: -# --- -import async_json_utils - -# Firebase: -# --- -import firebase_admin -import firebase_admin.firestore_async -import firebase_admin.auth -import firebase_admin.db - -# For debugging and logging: -# --- -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncFirebase: - - def __init__( - self, - credentials_json_path, - app_name, - max_connections = 5, - debug = True - ): - - # Debugging print: - # --- - self.__printer = IceCreamDebugger(prefix = f"FBase ({app_name[:8]}) | ", includeContext = True) - if not debug: self.__printer.disable() - alert = f"Firebase session starting with max. {max_connections} connections." - self.__printer(alert) - - # Initialize the instance: - # --- - self.__semaphore = asyncio.Semaphore(max_connections) - credentials = firebase_admin.credentials.Certificate(credentials_json_path) - self.__firebase_app = firebase_admin.initialize_app(credentials, name = app_name) - self.__firestore = firebase_admin.firestore_async.client(self.__firebase_app) - - def __del__(self): - firebase_admin.delete_app(self.__firebase_app) - self.__firestore.close() - alert = "Firebase session ended." - self.__printer(alert) - - def __user_to_json(self, firebase_user): - - user_json = { - "uid": firebase_user.uid, - "email": firebase_user.email, - "emailVerified": firebase_user.email_verified, - "displayName": firebase_user.display_name, - "phoneNo": firebase_user.phone_number, - "photoUrl": firebase_user.photo_url, - "customClaims": firebase_user.custom_claims, - "disabled": firebase_user.disabled, - "providerId": firebase_user.provider_id, - "providerData": firebase_user.provider_data, - "tenantId": firebase_user.tenant_id - } - - return user_json - - async def create_custom_token(self, uid): - - async with self.__semaphore: - - try: - custom_token = firebase_admin.auth.create_custom_token( - uid = uid, - app = self.__firebase_app - ).decode("utf-8") - return custom_token - - except Exception as exception: - self.__printer(exception) - return None - - async def authenticate_token(self, token): - - """ - To authenticate the given session token. - :param token: The session token generated by Firebase on a successful sign-in. - :return: Either the retrieved user information or a blank dictionary. - """ - - try: - user_info = firebase_admin.auth.verify_id_token(token, app = self.__firebase_app) - return user_info - except Exception as exception: - self.__printer(exception) - return {} - - async def create_user( - self, - uid, - display_name = None, - password = None, - email = None, - phone_number = None - ): - - """ - Create a new user. - :param uid: The id to identify the user by. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - firebase_admin.auth.create_user( - uid = uid, - app = self.__firebase_app - ) - return True - - except firebase_admin.auth.UidAlreadyExistsError as excp: - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def delete_user(self, uid): - - async with self.__semaphore: - - try: - firebase_admin.auth.delete_user( - uid = uid, - app = self.__firebase_app - ) - return True - - except firebase_admin.auth.UserNotFoundError as excp: - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def get_user(self, uid): - - async with self.__semaphore: - - try: - user = firebase_admin.auth.get_user( - uid = uid, - app = self.__firebase_app - ) - return self.__user_to_json(user) - - except Exception as exception: - self.__printer(exception) - return None - - async def list_users(self, users_per_page, page_token = None): - - """ - To get a list of users. Firebase has a limit of 1000 per call of this API. So we use page-tokens to fetch next - pages of users. - :param users_per_page: How many users you want to list in this API call (Max. 1,000). - :param page_token: To be used in case of pagination. - :return: The list of users and the page token to be used for the next call. Will be None in case of failure. - """ - - async with self.__semaphore: - - try: - users = firebase_admin.auth.list_users( - page_token = page_token, - max_results = min(users_per_page, 1000), - app = self.__firebase_app - ) - next_page_token = users.next_page_token if users.has_next_page else None - users = users.users - users_json = [self.__user_to_json(user) for user in users] - return users_json, next_page_token - - except Exception as exception: - self.__printer(exception) - return None, None - - async def create_document(self, collection_path, document_name, document_data = None): - - """ - To create a new document with the specified data in an existing collection. - :param collection_path: The path of the collection (can be a sub-collection). - :param document_name: The name of the document you want to create. - :param document_data: The data that you want to populate in the document. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.collection( - collection_path - ).add( - document_id = document_name, - document_data = document_data or {} - ) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def delete_document(self, path): - - """ - To delete a document. - :param path: The path of the document. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.document(path).delete() - return True - - except Exception as excp: - print("FIRESTORE DOCUMENT DELETION EXCEPTION:", excp) - return False - - async def set_document(self, path, data): - - """ - To overwrite the data in a document. - :param path: The path of the document. - :param data: The data you want to update as a dictionary. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.document(path).set(data) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def update_document(self, path, data): - - """ - To update the data in a document. Provide only the fields that you want to update. - :param path: The path of the document. - :param data: The data you want to update as a dictionary. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.document(path).update(data) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def get_document(self, path): - - """ - To get the data in a document. - :param path: The path of the document. - :return: The dictionary of data as found in the path specified or None if the operation failed. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.document(path).get() - return snapshot.to_dict() - - except Exception as exception: - self.__printer(exception) - return None - - async def get_document_fields(self, path, fields): - - """ - To get only specific fields (keys) in a document. Like how projections are used in Mongo. - :param path: The path of the document. - :param fields: The list of fields (keys) of the document that you want. - :return: The dictionary of data as found in the path specified. - """ - - async with self.__semaphore: - - try: - if type(fields) is not list: fields = [fields] - snapshot = await self.__firestore.document(path).get(fields) - return snapshot.to_dict() - - except Exception as exception: - self.__printer(exception) - return None - - async def create_collection(self, collection_name, document_name, document_data): - - """ - Creates a new collection in the root of the database. Note that Firebase doesn't allow creating new empty - collections, so we must add one first document in it. - :param collection_name: The name of the collection you want to create. - :param document_name: The name of the first document you want to put in the collection. - :param document_data: The data that you want to put in the first document of the new collection. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.collection( - collection_name - ).add( - document_id = document_name, - document_data = document_data - ) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def create_sub_collection( - self, - document_path, - sub_collection_name, - sub_document_name = None, - sub_document_data = None - ): - - """ - Firebase allows you to create collections inside documents. This method is built for that. Note that Firebase - doesn't allow creating new empty collections, so we must add one first document in it. - :param document_path: The path of the document in which you want to create a new collection. - :param sub_collection_name: The name of the collection you want to create. - :param sub_document_name: The name of the first document you want to put in the collection. - :param sub_document_data: The data that you want to put in the first document of the new collection. - :return: True or False based on the success of the operation. - """ - - async with self.__semaphore: - - try: - snapshot = await self.__firestore.document( - document_path - ).collection( - sub_collection_name - ).add( - document_id = sub_document_name, - document_data = sub_document_data or {} - ) - return True - - except Exception as exception: - self.__printer(exception) - return False - - async def get_collection(self, path): - - """ - Get a whole collection's data. - :param path: The path of the collection. - :return: The collection. - """ - - async with self.__semaphore: - - try: - snapshots = await self.__firestore.collection(path).get() - return {snapshot.id: snapshot.to_dict() for snapshot in snapshots} - - except Exception as exception: - self.__printer(exception) - return None - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import time - from shared import constants - - user_data = { - "firebaseUid": None, - "roles": "ClientAdmin", - "sessionToken": "siddhesh_20240419" - } - - async def test(): - my_firebase = AsyncFirebase( - f"{constants.PROJECT_DIRECTORY}/utils/cred/firebase_certs.json", - "myFire", - max_connections = 5 - ) - print("\n\n---\n\n") - results = await my_firebase.get_document("testCollection/testDoc") - print(async_json_utils.to_json_string(results)) - print("\n\n---\n\n") - results = await my_firebase.get_document_fields("testCollection/testDoc", ["sampleMap"]) - print(async_json_utils.to_json_string(results)) - # await my_firebase.create_sub_collection( - # "myDeepCollection/deepDocId", - # "subCollection3", - # "subDoc", - # {"sub_key": "sub_val"} - # ) - # await my_firebase.create_collection( - # "rootCollection", - # "subDoc", - # {"sub_key": "sub_val"} - # ) - # await my_firebase.delete_document("activeSessions/9d402f1502dfd55a4326fa7fc8e6cb7d") - # print(async_json_utils.to_json_string(await my_firebase.get_collection("myDeepCollection"))) - - start_time = time.time() - asyncio.run(test()) - print(f"FINISHED IN {time.time() - start_time} SECONDS.") diff --git a/utils_v2/database/async_mongo.py b/utils_v2/database/async_mongo.py deleted file mode 100644 index c052fac..0000000 --- a/utils_v2/database/async_mongo.py +++ /dev/null @@ -1,843 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 28th May, 2024 - - OBJECTIVE: - - To have one central place from where all async database connectivity happens. - - REFERENCES: - - 01. https://motor.readthedocs.io/en/stable/ - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For async behaviour: -import asyncio - -# For datetime handling: -import pytz -import datetime - -# MongoDB: -from motor.motor_asyncio import AsyncIOMotorClient -from bson.objectid import ObjectId -from bson.json_util import dumps, loads - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** EXCEPTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -class MongoFindException(Exception): - - def __init__(self, hint = None, origin = None): - self.__hint = hint - self.__origin = origin - - def __str__(self): - message = "mongo find operation failed" - if self.__hint is not None: message = f"{message} ({self.__hint})" - if self.__origin is not None: message = f"{self.__origin} --> {message}" - return message - - -# --------------------------------------------------------------------------------------------------------------------- - - -class MongoInsertException(Exception): - - def __init__(self, hint = None, origin = None): - self.__hint = hint - self.__origin = origin - - def __str__(self): - message = "mongo insert operation failed" - if self.__hint is not None: message = f"{message} ({self.__hint})" - if self.__origin is not None: message = f"{self.__origin} --> {message}" - return message - - -# --------------------------------------------------------------------------------------------------------------------- - - -class MongoUpdateException(Exception): - - def __init__(self, hint = None, origin = None): - self.__hint = hint - self.__origin = origin - - def __str__(self): - message = "mongo update operation failed" - if self.__hint is not None: message = f"{message} ({self.__hint})" - if self.__origin is not None: message = f"{self.__origin} --> {message}" - return message - - -# --------------------------------------------------------------------------------------------------------------------- - - -class MongoReplaceException(Exception): - - def __init__(self, hint = None, origin = None): - self.__hint = hint - self.__origin = origin - - def __str__(self): - message = "mongo replace operation failed" - if self.__hint is not None: message = f"{message} ({self.__hint})" - if self.__origin is not None: message = f"{self.__origin} --> {message}" - return message - - -# --------------------------------------------------------------------------------------------------------------------- - - -class MongoDeleteException(Exception): - - def __init__(self, hint = None, origin = None): - self.__hint = hint - self.__origin = origin - - def __str__(self): - message = "mongo delete operation failed" - if self.__hint is not None: message = f"{message} ({self.__hint})" - if self.__origin is not None: message = f"{self.__origin} --> {message}" - return message - - -# --------------------------------------------------------------------------------------------------------------------- - - -class MongoException(Exception): - - def __init__(self, hint = None, origin = None): - self.__hint = hint - self.__origin = origin - - def __str__(self): - message = "mongo operation failed" - if self.__hint is not None: message = f"{message} ({self.__hint})" - if self.__origin is not None: message = f"{self.__origin} --> {message}" - return message - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncMongo: - - __db = None - __db_name = None - __client = None - - def __init__( - self, - database_name = "myDb", - max_connections = 5, - debug = True, - debug_only_errors = True, - host = "localhost", - port = 27017, - connection_string = None - ): - - # Basic config: - self.__max_connections = max_connections - self.__db_name = database_name - self.__host = host, - self.__port = port - self.__connection_string = connection_string - - # For debugging: - self.__debug_only_errors = debug_only_errors - self.__printer = IceCreamDebugger(prefix = f"Mongo ({self.__db_name}) | ", includeContext = True) - if not debug: self.__printer.disable() - - # Rate/access control: - self.__exclusive_semaphore = asyncio.Semaphore(1) - - def enable_debug(self): - self.__printer.enable() - - def disable_debug(self): - self.__printer.disable() - - @staticmethod - def generate_id(): - - """ - Just generates an '_id' in MongoDB style. - :return: The '_id' in MongoDB style. - """ - - return str(ObjectId()) - - async def connect(self): - try: - if self.__connection_string is None: - self.__client = AsyncIOMotorClient( - self.__host, - self.__port, - maxPoolSize = self.__max_connections, - minPoolSize = self.__max_connections - ) - else: - self.__client = AsyncIOMotorClient( - self.__connection_string, - maxPoolSize = self.__max_connections, - minPoolSize = self.__max_connections - ) - self.__db = self.__client.get_database(self.__db_name) - if not self.__debug_only_errors: - server_info = await self.__client.server_info() - self.__printer(server_info) - except Exception as exception: self.__printer(exception) - - async def ensure_connection(self): - if self.__db is None: - async with self.__exclusive_semaphore: - await self.connect() - - @property - async def client(self): - await self.ensure_connection() - return self.__client - - @staticmethod - def dict_to_dot_notation(input_dict, parent_key = "", separator = "."): - items = [] - for k, v in input_dict.items(): - new_key = f"{parent_key}{separator}{k}" if parent_key else k - if isinstance(v, dict) and v: - items.extend(AsyncMongo.dict_to_dot_notation(v, new_key, separator = separator).items()) - else: - items.append((new_key, v)) - return dict(items) - - @staticmethod - def normalize_date_time(document): - - """ - MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it. - This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was - timezone naive, UTC timezone will be applied to it without changing the time value. - :param document: The document that you want to normalize the date-time in. - :return: The document with normalized datetime. - """ - - if isinstance(document, datetime.datetime): - utc_tz = pytz.timezone("UTC") - if document.tzinfo is None: document = utc_tz.localize(document) - else: document = document.astimezone(utc_tz) - - if type(document) is list: - document = [AsyncMongo.normalize_date_time(item) for item in document] - - if type(document) is dict: - document = { - AsyncMongo.normalize_date_time(k): AsyncMongo.normalize_date_time(v) - for k, v in document.items() - } - - return document - - @staticmethod - def __from_json_string(json_data): - return loads(json_data) - - @staticmethod - def __to_json_string(python_data, indent = 4, default = None): - return dumps(python_data, indent = indent, default = default) - - async def list_indexes( - self, - collection, - session = None, - raise_exception = False - ): - - """ - Lists out the indexes of a collection. - :param collection: The collection whose indexes you want to list out, - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The list of indexes or None if the action fails. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - indexes = None - - # Try to list the indexes: - try: - responses = await self.__db[collection].list_indexes(session = session).to_list(None) - indexes = [{key: value for key, value in response.items()} for response in responses] - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not indexes: raise MongoException(hint = "list indexes") - return indexes - - async def create_index( - self, - collection, - keys, - options = None, - session = None, - raise_exception = False - ): - - """ - Creates an index on a collection. - :param collection: The collection to create the index on. - :param keys: The keys (and sorting) to implement the index on. - :param options: Additional config. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the execution. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to make the insertion: - try: - options = options or {} - keys = [(k, v) for k, v in keys.items()] - response = await self.__db[collection].create_index(keys, session = session, **options) - if response: success = True - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not success: raise MongoException(hint = "create index") - return success - - async def insert_one( - self, - collection, - document, - session = None, - raise_exception = False - ): - - """ - Insert data into a collection. - :param collection: The collection you want to feed the data into. - :param document: The data to be stored. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The id of the inserted data, or null if the action fails. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - inserted_id = None - - # Try to make the insertion: - try: - response = await self.__db[collection].insert_one(document.copy(), session = session) - inserted_id = response.inserted_id - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not inserted_id: raise MongoInsertException(hint = f"{collection}") - return inserted_id - - async def insert_many( - self, - collection, - documents, - session = None, - raise_exception = False - ): - - """ - Insert a lot of data into a collection. - :param collection: The collection you want to feed the data into. - :param documents: The data to be stored. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The id of the inserted data, or null if the action fails. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - inserted_ids = [] - - # Try to make the insertion: - try: - response = await self.__db[collection].insert_many(documents, session = session) - inserted_ids = response.inserted_ids - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not inserted_ids: raise MongoInsertException(hint = f"{collection}") - return inserted_ids - - async def update_one( - self, - collection, - filter_json = None, - update_json = None, - upsert = False, - session = None, - raise_exception = False - ): - - """ - Update one document. - :param collection: The collection you want to update. - :param filter_json: The selection criteria to locate the document to update. - :param update_json: The values you want to update. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to make the insertion: - try: - response = await self.__db[collection].update_one( - filter_json, - update_json, - upsert = upsert, - session = session - ) - success = False if response.modified_count == 0 else True - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not success: raise MongoUpdateException(hint = f"{collection}") - return success - - async def update_many( - self, - collection, - filter_json = None, - update_json = None, - upsert = False, - session = None, - raise_exception = False - ): - - """ - Update many documents. - :param collection: The collection you want to update. - :param filter_json: The selection criteria to locate the document to update. - :param update_json: The values you want to update. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - update_count = 0 - - # Try to make the insertion: - try: - response = await self.__db[collection].update_many( - filter_json, - update_json, - upsert = upsert, - session = session - ) - update_count = response.modified_count - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not update_count: raise MongoUpdateException(hint = f"{collection}") - return update_count - - async def count( - self, - collection, - filter_json = None - ): - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # try to query the data: - try: - if filter_json is None: filter_json = {} - count = await self.__db[collection].count_documents(filter_json) - except Exception as exception: self.__printer(exception) - - # Done here: - return count - - async def bulk_write( - self, - collection, - requests, - session = None, - raise_exception = False - ): - - """ - To perform various individual operations in one go. You will have to import individual actions like "UpdateOne" - and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method. - :param collection: The collection you want to run the requests on. - :param requests: The array of requests (operations) to be performed. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # try to query the data: - try: - response = await self.__db[collection].bulk_write(requests, session = session) - count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not count: raise MongoException(hint = "bulk write") - return count - - async def find_many( - self, - collection, - filter_json, - projections = None, - skip = 0, - limit = None, - sort = None, - session = None, - as_json_string = False, - indent = 4, - raise_exception = False - ): - - """ - Finds one or more records that match the given conditions. - :param collection: The name of the collection to perform the search in. - :param filter_json: The filter criteria. - :param projections: What parts of the matching data you want to fetch. - :param skip: The no. of records to skip before picking next ones. Needed for pagination. - :param limit: The max. no. of records you want to fetch. - :param sort: The sorting rules to apply. - :param session: The session if you need to do this in a transaction. - :param as_json_string: Whether you want it as a JSON string or a Python dict/list. - :param indent: The indentation to use if you want it as a JSON string. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The array of matching records or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # Assume defaults: - if sort is None: sort = {"_id": -1} - if limit is None: limit = 10 - - # try to query the data: - try: - results = await self.__db[collection].find( - filter_json, - projections, - session = session - ).sort(sort).skip(skip).limit(limit).to_list(None) - if as_json_string: results = self.__to_json_string(results, indent = indent, default = str) - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not results: raise MongoFindException(hint = f"{collection}") - return results - - async def find_one( - self, - collection, - filter_json, - projections = None, - session = None, - as_json_string = False, - indent = 4, - raise_exception = False - ): - - """ - Finds one record that matches the given conditions. - :param collection: The name of the collection to perform the search in. - :param filter_json: The filter criteria. - :param projections: What parts of the matching data you want to fetch. - :param session: The session if you need to do this in a transaction. - :param as_json_string: Whether you want it as a JSON string or a Python dict/list. - :param indent: The indentation to use if you want it as a JSON string. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The array of matching records or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # try to query the data: - try: - results = await self.__db[collection].find_one(filter_json, projections, session = session) - if as_json_string: results = self.__to_json_string(results, indent = indent, default = str) - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not results: raise MongoFindException(hint = f"{collection}") - return results - - async def replace_one( - self, - collection, - filter_json, - replacement_json, - upsert = False, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param collection: The collection from which you want to delete many records. - :param filter_json: The filter criteria. - :param replacement_json: The data to put in place of the existing document. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # try to query the data: - try: - result = await self.__db[collection].replace_one( - filter_json, - replacement_json, - upsert = upsert, - session = session - ) - if result.modified_count or result.upserted_id: success = True - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not success: raise MongoReplaceException(hint = f"{collection}") - return success - - async def delete_one( - self, - collection, - filter_json, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param collection: The collection from which you want to delete many records. - :param filter_json: The filter criteria. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # try to query the data: - try: - result = await self.__db[collection].delete_one(filter_json, session = session) - count = result.deleted_count - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}") - return count - - async def delete_many( - self, - collection, - filter_json, - session = None, - raise_exception = False - ): - - """ - To delete many documents from a collection. - WARNING: sending {} in the filter would mean deleting ALL the documents. - :param collection: The collection from which you want to delete many records. - :param filter_json: The filter criteria. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # try to query the data: - try: - result = await self.__db[collection].delete_many(filter_json, session = session) - count = result.deleted_count - except Exception as exception: - self.__printer(exception) - - # Check results and return: - if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}") - return count - - async def aggregate( - self, - collection, - pipeline, - limit = None, - raise_exception = False - ): - - """ - Perform an advance query on the data. - :param collection: The collection to perform the query on. - :param pipeline: The pipeline of actions to take. Must be a list. - :param limit: The max. no. of records to retrieve. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The array of matching records or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # try to perform the aggregation action: - try: results = await self.__db[collection].aggregate(pipeline).to_list(limit) - except Exception as exception: self.__printer(exception) - - # Check results and return: - if raise_exception and not results: raise MongoException(hint = "aggregation") - return results - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/database/async_mongo_storage.py b/utils_v2/database/async_mongo_storage.py deleted file mode 100644 index 94df2d8..0000000 --- a/utils_v2/database/async_mongo_storage.py +++ /dev/null @@ -1,358 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 28th May, 2024 - - OBJECTIVE: - - To have one central place from where all async database connectivity happens. - - REFERENCES: - - 01. https://motor.readthedocs.io/en/stable/ - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -# --- -import sys -sys.path.append(".") -sys.path.append("..") - -# For system-level activity: -import io - -# For async behaviour: -import asyncio - -# MongoDB for File Storage: -from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket -from bson.objectid import ObjectId - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncMongoStorage: - - __db = None - __db_name = None - __client = None - __store = None - - def __init__( - self, - connection_string = None, - max_connections = 5, - host_name = "localhost", - port = 27017, - database_name = "fileStore", - debug = True, - debug_prefix = "GridFS (M) | ", - debug_only_errors = True - ): - - # Database Initialization: - self.__host_name = host_name - self.__port = port - self.__db_name = database_name - self.__connection_string = connection_string - self.__max_connections = max_connections - - # Debugging: - self.__debug_only_errors = debug_only_errors - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - @staticmethod - def generate_id(): - - """ - Just generates an '_id' in MongoDB style. - :return: The '_id' in MongoDB style. - """ - - return str(ObjectId()) - - async def connect(self): - - """ - Initialize the database connection. - :return: Nothing. - """ - - if self.__connection_string is None: - self.__client = AsyncIOMotorClient( - self.__host_name, - self.__port, - maxPoolSize = self.__max_connections, - minPoolSize = self.__max_connections - ) - else: - self.__client = AsyncIOMotorClient( - self.__connection_string, - maxPoolSize = self.__max_connections, - minPoolSize = self.__max_connections - ) - self.__db = self.__client.get_database(self.__db_name) - self.__store = AsyncIOMotorGridFSBucket(self.__db) - - @property - def fs(self): - - """ - To access the features that have not been wrapped in this reportlab directly. - This could include things like streaming files chunk-by-chunk. - :return: The file-store instance. - """ - - return self.__store - - async def write_from_memory(self, file_name, file_data, metadata_json = None): - - """ - Save a file (from RAM) to Mongo. Suitable for smaller files (a few MBs max.). - :param file_name: The name of the file. - :param file_data: The data of the file (held in RAM). - :param metadata_json: A JSON of metadata information that can later be used to search files (RECOMMENDED). - :return: The file's id as a string (if it gets saved) or None. - """ - - if self.__store is None: await self.connect() - - file_data.seek(0) - file_size = file_data.__sizeof__() - - file_id = None - try: file_id = await self.__store.upload_from_stream(file_name, file_data, metadata = metadata_json) - except Exception as exception: self.__printer(exception, file_name, file_size, file_id) - if not self.__debug_only_errors: self.__printer(file_name, file_size, file_id) - return str(file_id) - - async def read_to_memory(self, file_identifier, by_id = True): - - """ - To retrieve a file (in RAM) based on the provided identifier. - Suitable for smaller files (a few MBs max.). - :param file_identifier: Either the name or the "_id" of the file. - :param by_id: Set to True if you are fetching by the "_id" of the file. - :return: Either the file (in RAM) or None. - """ - - if self.__store is None: await self.connect() - - file_data = None - - try: - if by_id: grid_out = await self.__store.open_download_stream(ObjectId(file_identifier)) - else: grid_out = await self.__store.open_download_stream_by_name(file_identifier) - file_data = io.BytesIO(await grid_out.read()) - file_data.seek(0) - - except Exception as exception: - file_data = None - self.__printer(exception, file_identifier, by_id) - - if not self.__debug_only_errors: self.__printer(file_identifier, by_id) - return file_data - - async def delete_file_by_id(self, file_id): - - """ - Tries to delete one file by the id. - :param file_id: The id of the file in the database. - :return: True or False based on the success of the operation. - """ - - if self.__store is None: await self.connect() - - deleted = False - - try: - response = await self.__store.delete(file_id = ObjectId(file_id)) - deleted = True - except Exception as exception: - self.__printer(exception, file_id, deleted) - - return deleted - - def __format_metadata_json(self, metadata_json): - - """ - NOTE: ONLY USE WHEN SEARCHING FILES BY METADATA. - MongoDB expects dot-notation while searching for files by the metadata. We are making a function to search - files assuming that the conditions are to be applied to the metadata itself. So this function add the - dot-notation to the right places to conduct a successful search. - :param metadata_json: The JSON to format. - :return: The formatted JSON that has the right dot-notation. - """ - - formatted_metadata_json = {} - - for key, value in metadata_json.items(): - if not key.startswith("$"): key = f"metadata.{key}" - else: - if type(value) is dict: value = self.__format_metadata_json(value) - if type(value) is list: value = [self.__format_metadata_json(item) for item in value] - formatted_metadata_json[key] = value - - return formatted_metadata_json - - async def find_file_by_metadata(self, metadata_json, limit = None, skip = None, sort = None): - - """ - This method only lists the files that match the criteria mentioned in the metadata JSON. - :param metadata_json: The JSON that describes what you want to find. - :param limit: Max. no. of records to retrieve. - :param skip: No. of starting results to skip. Useful for pagination. - :param sort: The sorting conditions to follow. - :return: A list of (JSONs of) files that match the conditions. The list can be empty. - """ - - if self.__store is None: await self.connect() - - files_list = [] - - try: - limit = limit or 10 - skip = skip or 0 - sort = {"_id": -1} if not isinstance(sort, dict) else sort - formatted_metadata_json = self.__format_metadata_json(metadata_json) - return await self.__store.find( - formatted_metadata_json - ).sort(sort).skip(skip).limit(limit).to_list(None) - - except Exception as exception: self.__printer(exception, metadata_json, len(files_list)) - - if not self.__debug_only_errors: self.__printer(metadata_json, len(files_list)) - return files_list - - async def find_file_by_id(self, file_id): - - """ - This method allows you to get the file's info from the id of the file. - :param file_id: The id that was assigned by Mongo during upload. - :return: The file's info or None if the file doesn't exist. - """ - - if self.__store is None: await self.connect() - - file_info = None - - try: - formatted_metadata_json = {"_id": ObjectId(file_id)} - file_info = (await self.__store.find(formatted_metadata_json).to_list(1))[0] - - except Exception as exception: self.__printer(exception, file_id, file_info) - - return file_info - - async def get_file_name(self, file_id): - - """ - Returns the file name if the id of the file is known. - :param file_id: The id of the file as assigned by MongoDB when the file was stored. - :return: The file's name (if it exists), or None. - """ - - # Ensure that we are connected: - if self.__store is None: await self.connect() - - # Ensure that the input given is of 'ObjectId' type: - if type(file_id) is not ObjectId: file_id = ObjectId(str(file_id)) - - # Fetch and return the file name: - files_list = await self.__store.find( - {"_id": file_id}, - {"filename": True} - ).sort({"_id": -1}).limit(1).to_list(None) - try: file_name = files_list[0]["filename"] - except: file_name = None - return file_name - - async def aggregate( - self, - collection, - pipeline, - limit = None - ): - - """ - Perform an advance query on the data. - :param collection: The collection to perform the query on. - :param pipeline: The pipeline of actions to take. Must be a list. - :param limit: The max. no. of records to retrieve. - :return: The array of matching records or null if there was an exception. - """ - - if self.__store is None: await self.connect() - - results = None - try: results = await self.__db[collection].aggregate(pipeline).to_list(limit) - except Exception as exception: self.__printer(exception) - return results - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/database/async_mongo_v2.py b/utils_v2/database/async_mongo_v2.py deleted file mode 100644 index e5311c4..0000000 --- a/utils_v2/database/async_mongo_v2.py +++ /dev/null @@ -1,1736 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Original: Tuesday, 28th May, 2024 - Modified: Tuesday, 17th Sept., 2024 - - OBJECTIVE: - - To have one central place from where all async MongoDB activity happens. - - REFERENCES: - - 01. https://motor.readthedocs.io/en/stable/ - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For system-level activity: -import io - -# For async behaviour: -import asyncio - -# MongoDB for File Storage: -from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket -from bson.objectid import ObjectId -from bson.json_util import dumps, loads - -# My utils: -from utils_v2.string import json - -# For datetime handling: -import pytz -import datetime - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** EXCEPTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncMongoBase: - - def __init__( - self, - connection_string = None, - max_connections = 5, - host_name = "localhost", - port = 27017, - database_name = "myDb", - debug = True, - debug_prefix = "Mongo | ", - debug_only_errors = True - ): - - # Basic variables that will be needed later: - self._client = None - self._db = None - self._fs = None - - # Database Initialization: - self._host_name = host_name - self._port = port - self._db_name = database_name - self._connection_string = connection_string - self._max_connections = max_connections - - # Debugging: - self._debug_only_errors = debug_only_errors - self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self._printer.disable() - - def enable_debug(self): - self._printer.enable() - - def disable_debug(self): - self._printer.disable() - - async def connect(self): - - """ - Initialize the database connection. - :return: None. - """ - - self._printer("Connecting...") - - # If a connection string is given, - # give preference to it: - if self._connection_string is None: - self._client = AsyncIOMotorClient( - self._host_name, - self._port, - maxPoolSize = self._max_connections, - minPoolSize = self._max_connections - ) - - # In the absense of a connection string, - # try to connect through the other credentials: - else: - self._client = AsyncIOMotorClient( - self._connection_string, - maxPoolSize = self._max_connections, - minPoolSize = self._max_connections - ) - - # Debugging print: - if not self._debug_only_errors: - server_info = await self._client.server_info() - self._printer(server_info) - - # Now we connect to the database and the file-store: - self._db = self._client.get_database(self._db_name) - self._fs = AsyncIOMotorGridFSBucket(self._db) - - async def ensure_connection(self): - - """ - Call this at the start of every function to automatically connect to the database when the user of the library - forgets to explicitly connect to the database. - :return: None. - """ - - if self._client is None: await self.connect() - - @staticmethod - def generate_id(as_str = False): - - """ - Just generates an '_id' in MongoDB style. - :param as_str: Set to True to convert the generated id to a string; and False to receive it as an instance of - 'ObjectId'. This is useful when you need to pre-assign ids to files. - :return: The '_id' in MongoDB style. - """ - - generated_id = ObjectId() - return str(generated_id) if as_str else generated_id - - @property - async def client(self): - - """ - returns the client to perform activities that have not been explicitly wrapped in the class. - :return: The client's instance. - """ - - await self.ensure_connection() - return self._client - - @property - async def db(self): - - """ - returns the database connector to perform activities that have not been explicitly wrapped in the class. - :return: The database's connection instance. - """ - - await self.ensure_connection() - return self._fs - - @property - async def fs(self): - - """ - returns the file-store to perform activities that have not been explicitly wrapped in the class. - :return: The file-store instance. - """ - - await self.ensure_connection() - return self._fs - - @staticmethod - def from_json_string(json_data): - - """ - Converts from a JSON string to BSON. - :param json_data: The JSON string to convert to BSON. - :return: The BSON interpretation of the input JSON string. - """ - - return loads(json_data) - - @staticmethod - def to_json_string(data, indent = 4, default = None): - - """ - Converts from a BSON to JSON string. - :param data: The input BSON data. - :param indent: The no. of spaces to put into the string for pretty print. - :param default: The default function to apply to data that cannot be converted directly. - :return: The JSON string from the input data. - """ - - return dumps(data, indent = indent, default = default) - - @staticmethod - def dict_to_dot_notation(input_dict, pk = "", s = "."): - - """ - Converts an input dict to dot notation format. Can be used as a utility to perform searches. - :param input_dict: The dict that you want to convert to dot notation. - :param pk: Parent Key. DO NOT TOUCH (meant to be used during recursion). - :param s: Separator. DO NOT TOUCH (meant to be used during recursion). - :return: The dot notation representation of the input dict. - """ - - items = [] - for k, v in input_dict.items(): - new_key = f"{pk}{s}{k}" if pk else k - if isinstance(v, dict) and v: items.extend(AsyncMongoBase.dict_to_dot_notation(v, new_key, s = s).items()) - else: items.append((new_key, v)) - return dict(items) - - @staticmethod - def normalize_date_time(document): - - """ - MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it. - This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was - timezone naive, UTC timezone will be applied to it without changing the time value. - :param document: The document that you want to normalize the date-time in. - :return: The document with normalized datetime. - """ - - if isinstance(document, datetime.datetime): - utc_tz = pytz.timezone("UTC") - if document.tzinfo is None: document = utc_tz.localize(document) - else: document = document.astimezone(utc_tz) - - if type(document) is list: - document = [AsyncMongoBase.normalize_date_time(item) for item in document] - - if type(document) is dict: - document = { - AsyncMongoBase.normalize_date_time(k): AsyncMongoBase.normalize_date_time(v) - for k, v in document.items() - } - - return document - - @staticmethod - def read_to_ram(file_path): - - """ - Reads a file into a BytesIO object in RAM. - :param file_path: The path to the file on disk. - :return: The file in a BytesIO object. - """ - - with open(file_path, "rb") as file: file_data = file.read() - file_in_ram = io.BytesIO(file_data) - file_in_ram.seek(0) - return file_in_ram - - -# --------------------------------------------------------------------------------------------------------------------- - - -class AsyncMongo(AsyncMongoBase): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - async def list_indexes( - self, - collection, - session = None, - raise_exception = False - ): - - """ - Lists out the indexes of a collection. - :param collection: The collection whose indexes you want to list out, - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The list of indexes or None if the action fails. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - indexes = None - - # Try to list the indexes: - try: - responses = await self._db[collection].list_indexes(session = session).to_list(None) - indexes = [{key: value for key, value in response.items()} for response in responses] - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return indexes - - async def create_index( - self, - collection, - keys, - options = None, - session = None, - raise_exception = False - ): - - """ - Creates an index on a collection. - :param collection: The collection to create the index on. - :param keys: The keys (and sorting) to implement the index on. - :param options: Additional config. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the execution. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to make the insertion: - try: - options = options or {} - keys = [(k, v) for k, v in keys.items()] - response = await self._db[collection].create_index(keys, session = session, **options) - if response: success = True - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def count( - self, - collection, - filter, - session = None, - raise_exception = False - ): - - """ - Counts the no. of documents that match the given filter condition. - :param collection: The name of the collection to count in. - :param filter: The filter criteria that the documents must satisfy to be counted. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The count if the operation was performed successfully or None is something went wrong and the exception - was suppressed. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = None - - # try to query the data: - try: count = await self._db[collection].count_documents(filter, session = session) - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return count - - async def insert_one( - self, - collection, - document, - session = None, - raise_exception = False - ): - - """ - Insert data into a collection. - :param collection: The collection you want to feed the data into. - :param document: The data to be stored. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The id of the inserted data, or null if the action fails. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - inserted_id = None - - # Try to make the insertion: - try: - response = await self._db[collection].insert_one(document.copy(), session = session) - inserted_id = response.inserted_id - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return inserted_id - - async def insert_many( - self, - collection, - documents, - session = None, - raise_exception = False - ): - - """ - Insert a lot of data into a collection. - :param collection: The collection you want to feed the data into. - :param documents: The list of documents to be stored. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The id of the inserted data, or null if the action fails. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - inserted_ids = [] - - # Try to make the insertion: - try: - response = await self._db[collection].insert_many(documents, session = session) - inserted_ids = response.inserted_ids - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return inserted_ids - - async def find_one( - self, - collection, - filter, - projection = None, - session = None, - raise_exception = False - ): - - """ - Finds one record that matches the given conditions. - :param collection: The name of the collection to perform the search in. - :param filter: The filter criteria. - :param projection: What parts of the matching data you want to fetch. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The matching record or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # try to query the data: - try: results = await self._db[collection].find_one(filter, projection, session = session) - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Check results and return: - return results - - async def find_many( - self, - collection, - filter, - projection = None, - skip = 0, - limit = None, - sort = None, - session = None, - raise_exception = False - ): - - """ - Finds one or more records that match the given conditions. - :param collection: The name of the collection to perform the search in. - :param filter: The filter criteria. - :param projection: What parts of the matching data you want to fetch. - :param skip: The no. of records to skip before picking next ones. Needed for pagination. - :param limit: The max. no. of records you want to fetch. - :param sort: The sorting rules to apply. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The array of matching records or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # Assume defaults: - if sort is None: sort = {"_id": -1} - if limit is None: limit = 10 - - # try to query the data: - try: - results = await self._db[collection].find( - filter, - projection, - sort = sort, - skip = skip, - limit = limit, - session = session - ).to_list(None) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return results - - async def update_one( - self, - collection, - filter = None, - update = None, - upsert = False, - session = None, - raise_exception = False - ): - - """ - Update one document. - :param collection: The collection you want to update. - :param filter: The selection criteria to locate the document to update. - :param update: The values you want to update. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to make the update: - try: - response = await self._db[collection].update_one( - filter, - update, - upsert = upsert, - session = session - ) - success = True if response.modified_count > 0 or response.upserted_id else False - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def update_many( - self, - collection, - filter = None, - update = None, - upsert = False, - session = None, - raise_exception = False - ): - - """ - Update many documents. - :param collection: The collection you want to update. - :param filter: The selection criteria to locate the document to update. - :param update: The values you want to update. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - update_count = 0 - - # Try to make the insertion: - try: - response = await self._db[collection].update_many( - filter, - update, - upsert = upsert, - session = session - ) - update_count = response.modified_count + response.upserted_count - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return update_count - - async def find_one_and_update( - self, - collection, - filter = None, - update = None, - projection = None, - return_updated = False, - upsert = False, - session = None, - raise_exception = False - ): - - """ - Update one document. - :param collection: The collection you want to update. - :param filter: The selection criteria to locate the document to update. - :param update: The values you want to update. - :param projection: What parts of the matching data you want to fetch. - :param return_updated: To choose whether you want to retrieve the original document or the updated document. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - document = None - - # Try to make the update: - try: - document = await self._db[collection].find_one_and_update( - filter, - update, - projection = projection, - return_document = return_updated, - upsert = upsert, - session = session - ) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return document - - async def replace_one( - self, - collection, - filter, - replacement, - upsert = False, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param collection: The collection from which you want to delete many records. - :param filter: The filter criteria. - :param replacement: The data to put in place of the existing document. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # try to query the data: - try: - result = await self._db[collection].replace_one( - filter, - replacement, - upsert = upsert, - session = session - ) - if result.modified_count or result.upserted_id: success = True - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def find_one_and_replace( - self, - collection, - filter, - replacement, - projection = None, - return_replaced = False, - upsert = False, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param collection: The collection from which you want to delete many records. - :param filter: The filter criteria. - :param replacement: The data to put in place of the existing document. - :param projection: What parts of the matching data you want to fetch. - :param return_replaced: To choose whether you want to retrieve the original document or the updated document. - :param upsert: If you want to insert if the document doesn't already exist. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - document = None - - # try to query the data: - try: - document = await self._db[collection].find_one_and_replace( - filter, - replacement, - projection = projection, - return_document = return_replaced, - upsert = upsert, - session = session - ) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return document - - async def delete_one( - self, - collection, - filter, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param collection: The collection from which you want to delete many records. - :param filter: The filter criteria. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # try to query the data: - try: - result = await self._db[collection].delete_one(filter, session = session) - success = True if result.deleted_count else False - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def delete_many( - self, - collection, - filter, - session = None, - raise_exception = False - ): - - """ - To delete many documents from a collection. - WARNING: sending {} in the filter would mean deleting ALL the documents. - :param collection: The collection from which you want to delete many records. - :param filter: The filter criteria. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # try to query the data: - try: - result = await self._db[collection].delete_many(filter, session = session) - count = result.deleted_count - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return count - - async def find_one_and_delete( - self, - collection, - filter, - projection = None, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param collection: The collection from which you want to delete a record. - :param filter: The filter criteria. - :param projection: What parts of the matching data you want to fetch. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The document that matched your criteria or None. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - document = None - - # try to query the data: - try: - document = await self._db[collection].find_one_and_delete( - filter, - projection = projection, - session = session - ) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return document - - async def bulk_write( - self, - collection, - requests, - session = None, - raise_exception = False - ): - - """ - To perform various individual operations in one go. You will have to import individual actions like "UpdateOne" - and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method. - :param collection: The collection you want to run the requests on. - :param requests: The array of requests (operations) to be performed. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # try to query the data: - try: - response = await self._db[collection].bulk_write(requests, session = session) - count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return count - - async def aggregate( - self, - collection, - pipeline, - limit = None, - session = None, - raise_exception = False - ): - - """ - Perform an advance query on the data. - :param collection: The collection to perform the query on. - :param pipeline: The pipeline of actions to take. Must be a list. - :param limit: The max. no. of records to retrieve. preferably apply the limit from within the pipeline. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The array of matching records or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # try to perform the aggregation action: - try: results = await self._db[collection].aggregate(pipeline, session = session).to_list(limit) - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return results - - -# --------------------------------------------------------------------------------------------------------------------- - - -class AsyncMongoStorage(AsyncMongo): - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - async def easy_upload( - self, - source, - file_name: str, - file_metadata: dict = None, - file_id = None, - chunk_size: int = None, - session = None, - raise_exception = False - ): - - """ - Easily write one file to MongoDB's GridFS. Ideal for directly uploading small files without having to worry - about any internal mechanisms. - :param source: The actual file, supplied as either a path string or a file-like object, to be written to the - database. - :param file_name: The name of the file as it will be stored on (and retrieved from) GridFS. - :param file_metadata: Any metadata to later search the file by. - :param file_id: Any custom id to be given to the file. TRY STICKING TO THE ID GENERATED BY 'generate_id'. - :param chunk_size: The chunk size (in bytes) to use for storing this file. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The id of the inserted file or None if the upload failed. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - inserted_id = None - - # Try to write the file: - try: - - # Input pre-processing: - if isinstance(source, io.BytesIO): source.seek(0) - elif isinstance(source, str): source = open(source, mode = "rb") - - # If no file id was supplied: - if file_id is None: - inserted_id = await self._fs.upload_from_stream( - filename = file_name, - source = source, - metadata = file_metadata, - chunk_size_bytes = chunk_size, - session = session - ) - - # If a file id was supplied: - else: - await self._fs.upload_from_stream_with_id( - file_id = file_id, - filename = file_name, - source = source, - metadata = file_metadata, - chunk_size_bytes = chunk_size, - session = session - ) - inserted_id = file_id - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return inserted_id - - async def easy_download( - self, - destination, - file_id = None, - file_name = None, - session = None, - raise_exception = False - ): - - """ - Easily read one file from MongoDB's GridFS. Ideal for directly downloading small files without having to worry - about any internal mechanisms. - :param destination: The path on the local disk or a buffer in RAM to save the downloaded data to. - :param file_id: (RECOMMENDED) the id of the save file. - :param file_name: The name of the saved file. NOT RECOMMENDED because you could have many files with the same - name. The best way to tell files apart if from the id. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. The contents of the stored file are written - directly to the destination. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to read the file: - try: - - # Input pre-processing: - if isinstance(destination, str): destination = open(destination, mode = "wb") - - # If a file id is supplied (preferred way): - if file_id is not None: - await self._fs.download_to_stream( - destination = destination, - file_id = file_id, - session = session - ) - if isinstance(destination, io.BytesIO): destination.seek(0) - success = True - - # If a file name is supplied: - elif file_name is not None: - await self._fs.download_to_stream_by_name( - destination = destination, - filename = file_name, - session = session - ) - if isinstance(destination, io.BytesIO): destination.seek(0) - success = True - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def count_files( - self, - filter, - session = None, - raise_exception = False - ): - - """ - Counts the no. of documents that match the given filter condition. - :param filter: The filter criteria that the documents must satisfy to be counted. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The count if the operation was performed successfully or None is something went wrong and the exception - was suppressed. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = None - - # try to query the data: - try: count = await self._db["fs.files"].count_documents(filter, session = session) - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return count - - async def find_one_file( - self, - filter, - projection = None, - session = None, - raise_exception = False - ): - - """ - Finds one record that matches the given conditions. This does NOT return the file itself, it returns the record - that describes the file. - :param filter: The filter criteria. - :param projection: What parts of the matching data you want to fetch. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The matching record or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # try to query the data: - try: results = await self._db["fs.files"].find_one(filter, projection, session = session) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Check results and return: - return results - - async def find_many_files( - self, - filter, - projection = None, - skip = 0, - limit = None, - sort = None, - session = None, - raise_exception = False - ): - - """ - Finds one or more records that match the given conditions. This doesn't return any actual files directly, it - returns the records that describe the files. - :param filter: The filter criteria. - :param projection: What parts of the matching data you want to fetch. - :param skip: The no. of records to skip before picking next ones. Needed for pagination. - :param limit: The max. no. of records you want to fetch. - :param sort: The sorting rules to apply. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The array of matching records or null if there was an exception. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - results = None - - # Assume defaults: - if sort is None: sort = {"_id": -1} - if limit is None: limit = 10 - - # try to query the data: - try: - results = await self._db["fs.files"].find( - filter, - projection, - session = session - ).sort(sort).skip(skip).limit(limit).to_list(None) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return results - - async def update_metadata_for_one_file( - self, - filter: dict, - unset_data: dict = None, - set_data: dict = None, - session = None, - raise_exception = False - ): - - """ - Updates the metadata for one file. - :param filter: The conditions to filter the files by. - :param unset_data: The fields that you want to discard. - :param set_data: The fields that you want to add or update. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to make the update: - try: - - # Input pre-processing: - update = {} - if unset_data is not None: update["$unset"] = self.dict_to_dot_notation({"metadata": unset_data}) - if set_data is not None: update["$set"] = self.dict_to_dot_notation({"metadata": set_data}) - - # Actual update happens here: - if update: - response = await self._db["fs.files"].update_one( - filter, - update, - session = session - ) - success = False if response.modified_count == 0 else True - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def update_metadata_for_many_files( - self, - filter: dict, - unset_data: dict = None, - set_data: dict = None, - session = None, - raise_exception = False - ): - - """ - Updates the metadata for many files. Practically the same as 'update_metadata_for_one' except that the scope of - the modifications is far wider. - :param filter: The conditions to filter the files by. - :param unset_data: The fields that you want to discard. - :param set_data: The fields that you want to add or update. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = None - - # Try to make the update: - try: - - # Input pre-processing: - update = {} - if unset_data is not None: update["$unset"] = self.dict_to_dot_notation({"metadata": unset_data}) - if set_data is not None: update["$set"] = self.dict_to_dot_notation({"metadata": set_data}) - - # Actual update happens here: - if update: - response = await self._db["fs.files"].update_many( - filter, - update, - session = session - ) - count = response.modified_count - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return count - - async def replace_metadata_for_one_file( - self, - filter: dict, - replacement, - session = None, - raise_exception = False - ): - - """ - Updates the metadata for one file. - :param filter: The conditions to filter the files by. - :param replacement: The new metadata to put inplace of the old one. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to make the update: - try: - - # Actual update happens here: - result = await self._db["fs.files"].update_one( - filter, - {"$set": {"metadata": replacement}}, - upsert = False, - session = session - ) - if result.modified_count or result.upserted_id: success = True - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def replace_metadata_for_many_files( - self, - filter: dict, - replacement, - session = None, - raise_exception = False - ): - - """ - Updates the metadata for one file. - :param filter: The conditions to filter the files by. - :param replacement: The new metadata to put inplace of the old one. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = None - - # Try to make the update: - try: - - # Actual update happens here: - response = await self._db["fs.files"].update_many( - filter, - {"$set": {"metadata": replacement}}, - session = session - ) - count = response.modified_count - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return count - - async def delete_file_by_id( - self, - file_id, - session = None, - raise_exception = False - ): - - """ - Deletes one file by the file's id. Deleting files is different from deleting simple documents because, in files, - you also need to clear out the chunks (which are stored in a separate collection). The built-in mechanism of - Motor only provides support to delete by the file's id. - :param file_id: The id of the file that you want to delete. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Try to read the file: - try: - - # Delete the file: - await self._fs.delete(file_id = file_id, session = session) - success = True - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return success - - async def delete_one_file( - self, - filter, - session = None, - raise_exception = False - ): - - """ - To delete one document from a collection. - :param filter: The filter criteria. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: True or False based on the success of the operation. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - success = False - - # Find one file that matches the given filter: - file = await self.find_one( - collection = "fs.files", - filter = filter, - session = session, - raise_exception = raise_exception - ) - - # If we have a list of files to work with: - if file: success = await self.delete_file_by_id(file["_id"]) - - # Done here: - return success - - async def delete_many_files( - self, - filter, - session = None, - raise_exception = False - ): - - """ - To delete many documents from a collection. - WARNING: sending {} in the filter would mean deleting ALL the documents. - :param filter: The filter criteria. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The number of files deleted (can be zero) or None if something failed while searching the files. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - count = 0 - - # Find all the files that match the given filter: - files = await self.find_many( - filter = filter, - session = session, - raise_exception = raise_exception - ) - - # If we have a list of files to work with: - if files is not None: - tasks = [self.delete_file_by_id(file["_id"]) for file in files] - results = await asyncio.gather(*tasks) - count = sum(results) - - # Done here: - return count - - async def get_upload_stream( - self, - file_name: str, - file_metadata: dict = None, - file_id = None, - chunk_size: int = None, - session = None, - raise_exception = False - ): - - """ - Returns a GridIn object so that you can perform your own upload using the built-in writing methods. You must use - the 'write' method to write data to the file by passing it either a string of bytes or a file-like object. When - the file has been fully written, you must call the 'close' method to finish the operation. In case you need to - cancel the operation, you can call the 'abort' method to delete all the already written data and stop uploading - new data. - :param file_name: The name of the file as it will be stored on (and retrieved from) GridFS. - :param file_metadata: Any metadata to later search the file by. - :param file_id: Any custom id to be given to the file. TRY STICKING TO THE ID GENERATED BY 'generate_id'. - :param chunk_size: The chunk size (in bytes) to use for storing this file. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The upload stream that implements the 'write', 'close', and 'abort' methods, or None if something - failed. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - stream = None - - # Try to open the upload stream: - try: - - # If no file id was supplied: - if file_id is None: - stream = self._fs.open_upload_stream( - filename = file_name, - metadata = file_metadata, - chunk_size_bytes = chunk_size, - session = session - ) - - # If a file id was supplied: - else: - stream = self._fs.open_upload_stream_with_id( - file_id = file_id, - filename = file_name, - metadata = file_metadata, - chunk_size_bytes = chunk_size, - session = session - ) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return stream - - async def get_download_stream( - self, - file_id = None, - file_name = None, - session = None, - raise_exception = False - ): - - """ - Returns a GridOut object so that you can implement your own download logic using the built-in 'read' method. - Once the reading is done, use the 'close' method to release the resources used by the stream. - :param file_id: (RECOMMENDED) the id of the save file. - :param file_name: The name of the saved file. NOT RECOMMENDED because you could have many files with the same - name. The best way to tell files apart if from the id. - :param session: The session if you need to do this in a transaction. - :param raise_exception: Whether, or not, you want to raise an exception when something fails. - :return: The download stream that implements the 'read' and 'close' methods, or None if something failed. - """ - - # Ensure you are connected: - await self.ensure_connection() - - # Assume failure: - stream = None - - # Try to open a download stream: - try: - - # If a file id is supplied (preferred way): - if file_id is not None: - stream = await self._fs.open_download_stream( - file_id = file_id, - session = session - ) - - # If a file name is supplied: - elif file_name is not None: - stream = await self._fs.open_download_stream_by_name( - filename = file_name, - session = session - ) - - # When something goes wrong: - except Exception as exception: - self._printer(exception) - if raise_exception: raise - - # Done here: - return stream - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - from shared import constants - - async def main(): - - # Create an instance of the database connector: - my_db = AsyncMongo( - connection_string = constants.MONGO_DATA_CONNECTION_STRING, - database_name = constants.MONGO_DATA_DATABASE_NAME, - max_connections = 10, - debug = True - ) - - # Connect to the database: - await my_db.connect() - - # # Get the documents to migrate: - # documents = await my_db.find_many( - # collection = "scriptData", - # filter = {}, - # limit = 50, - # projection = {"_id": False} - # ) - # # print(json.to_string(documents, default = str)) - # - # # Adjust them: - # adjusted_documents = [] - # for document in documents: - # script_id = document.pop("scriptId") - # adjusted_document = { - # "scriptId": script_id, - # "desc": "no desc", - # "content": document - # } - # adjusted_documents.append(adjusted_document) - # print(json.to_string(adjusted_documents, default = str)) - # - # # Insert the adjusted ones to the new collection: - # response = await my_db.insert_many( - # collection = "_scriptData", - # documents = adjusted_documents - # ) - # print("RESPONSE:", response) - - - asyncio.run(main()) diff --git a/utils_v2/database/async_mysql_v2.py b/utils_v2/database/async_mysql_v2.py deleted file mode 100644 index f4cac6e..0000000 --- a/utils_v2/database/async_mysql_v2.py +++ /dev/null @@ -1,424 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 30th Aug., 2024 - - OBJECTIVE: - - To be able to access SQL-based databases from python in a simple way. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# MySQL Database: -import aiomysql -import decimal - -# For data-crunching: -import pandas as pd - -# For time-keeping: -import time - -# OS-level operations: -import os - -# My utils: -from utils_v2.string import json - -# For async activities: -import asyncio - -# For debugging: -from icecream import IceCreamDebugger -import traceback - -# To work with datatypes: -from typing import List - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncMySQL: - - def __init__( - self, - pool_size, - *args, - **kwargs - ): - - """ - A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as - JSON-like structures (list or dict). The format for the results was very specific to our use case for serving - Bicree's requirement. This may not serve your requirement at all. - :param pool_size: The number of connections to maintain n a pool. - :param args: Any arguments to pass. Not used. - :param kwargs: Pass the connection configuration from here. - """ - - # Set up the variables: - self.__args = args - self.__kwargs = kwargs - self.__min_pool_size = 10 - self.__max_pool_size = max(pool_size, self.__min_pool_size) - self.__pool = None - - # Set up the debugging tools: - self.__printer = IceCreamDebugger(prefix = "MySQL | ", includeContext = True) - - def __del__(self): - pass - - async def connect(self): - - """ - Establish a connection and create a pool of connections to call from. - :return: None. - """ - - try: - - self.__kwargs["db"] = self.__kwargs.pop("database") - self.__pool = await aiomysql.create_pool( - minsize = self.__min_pool_size, - maxsize = self.__max_pool_size, - loop = asyncio.get_event_loop(), - **self.__kwargs - ) - - except Exception as exception: - self.__printer(exception) - self.__pool = None - - async def ensure_connection(self): - - """ - Tries to ensure that a connection is present. - Can be called before every function to make sure that our pool is established. - :return: None. - """ - - if self.__pool is None: await self.connect() - - @staticmethod - def __parse_row(row): - - """ - Converts from the custom objects of 'aiomysql' to types that are supported by Python. - :param row: The row from the result. - :return: The parsed row which will have types that are closer to being native to Python.. - """ - - parsed_row = [] - for item in row: - if isinstance(item, decimal.Decimal): parsed_row.append(float(item)) - else: parsed_row.append(item) - return parsed_row - - async def fetch_all(self, cursor): - - # Make a variable to hold all the result sets. - # Needed for when the procedure responds with many "tables": - all_result_sets = [] - - # Iterate over all result sets, - # and process them one-by-one: - while True: - - # Process the current result set: - this_result_set = [] - result = await cursor.fetchall() - if not cursor.description: break - columns = [desc[0] for desc in cursor.description] - for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row)))) - all_result_sets.append(this_result_set) - - # Move to the next set, - # or break out of the loop if all done: - if not await cursor.nextset(): break - - # Done here: - return all_result_sets - - async def call_procedure(self, procedure_name, procedure_args): - - """ - To call stored procedures and retrieve all the responses. - :param procedure_name: The name of the stored procedure that must be called. - :param procedure_args: The args to be sent to the stored procedure. - :return: The raw result set as received from the database. - """ - - # Make sure we have a connection: - await self.ensure_connection() - - # Make a variable to hold all the result sets. - # Needed for when the procedure responds with many "tables": - all_result_sets = [] - - # Call the procedure and get the results: - async with self.__pool.acquire() as connection: - async with connection.cursor() as cursor: - await cursor.callproc(procedure_name, procedure_args) - all_result_sets = await self.fetch_all(cursor) - - # # Iterate over all result sets, - # # and process them one-by-one: - # while True: - # this_result_set = [] - # result = await cursor.fetchall() - # if not cursor.description: break - # columns = [desc[0] for desc in cursor.description] - # for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row)))) - # all_result_sets.append(this_result_set) - # await cursor.nextset() - - # Done here: - return all_result_sets - - async def call_procedure_and_get_json( - self, - procedure_name, - procedure_args, - retry_count = 1, - backoff_seconds = 0.5, - backoff_multiplier = 1.1, - return_exception = False - ): - - """ - The method to call when you need to call a stored procedure and retrieve the response as a JSON-like object. - This is custom formatting based on the structure created by Mr. bhushan Thakkar in late April (2024). - :param procedure_name: The name of the stored procedure that must be called. - :param procedure_args: The args to be sent to the stored procedure. - :param retry_count: The max. number of times to try in case one or more attempts fail. - :param backoff_seconds: The time to wait before making the next attempt if the retry count is more than 1. - :param backoff_multiplier: The factor that dictates how much to modify the time delay by when waiting to retry. - :param return_exception: Whether, or not, you would like to return the exception object if something goes wrong. - :return: The formatted response and the exception (if asked for). - """ - - # Note down the start time: - start_ts = time.time() - - # Try to get the data from the database: - results = [] - exception = None - for _ in range(retry_count): - try: results = await self.call_procedure( - procedure_name = procedure_name, - procedure_args = procedure_args, - ) - except Exception as exc: exception = exc - if exception is None: break - await asyncio.sleep(backoff_seconds) - backoff_seconds = backoff_seconds * backoff_multiplier - - # If the results are blank: - if len(results) == 0: - formatted_results = { - "status": 0, - "message": "Please contact admin (NE)" if exception is None else "Please contact admin (E)", - "seconds": time.time() - start_ts, - "data": {} - } - if return_exception: return formatted_results, exception - else: return formatted_results - - # Extract the very basic success or failure indicators: - formatted_results = { - "status": results[0][0]["status"], - "message": results[0][0].get("message", "ok"), - "seconds": 0.0, - "data": {} - } - - # Handle the remaining keys of the zeroth result set: - for key, value in results[0][0].items(): - if key not in formatted_results.keys(): - formatted_results["data"][key] = value - - # Format - for index in range(len(results)): - if index > 0: formatted_results["data"][f"rs{index-1}"] = results[index] - - # Note down the time taken: - formatted_results["seconds"] = time.time() - start_ts - - # Done here: - if return_exception: return formatted_results, exception - else: return formatted_results - - async def execute_one( - self, - query: str, - commit: bool = True, - return_exception: bool = False - ): - - """ - Runs one command / query in SQL. - :param query: The query / command to run. - :param commit: Whether, or not, you would like to commit the execution. - :param return_exception: Whether, or not, you would like to return the exception from this function. - :return: Either just the result or the result and the exception. - """ - - # Make sure we have a connection: - await self.ensure_connection() - - # Start by assuming failure: - rows_affected = None - results = None - excp = None - - try: - - # Get a connection and execute the command: - async with self.__pool.acquire() as connection: - async with connection.cursor() as cursor: - rows_affected = await cursor.execute(query) - results = await self.fetch_all(cursor) - if commit: await connection.commit() - - # SQL-specific errors: - except aiomysql.MySQLError as exception: - self.__printer("SQL Exception", exception) - excp = exception - - # Other errors: - except Exception as exception: - self.__printer("Other Exception", exception) - excp = exception - - # Done here: - if return_exception: return rows_affected, results, excp - else: return rows_affected, results - - async def execute_many( - self, - query: str, - data: List[tuple], - commit: bool = True, - return_exception: bool = False - ): - - """ - Runs many commands / queries in SQL. - Consider the following example: - QUERY: "INSERT INTO pincodeMaster (pincode, city, state) VALUES (%s, %s, %s);" - DATA: [ - ('110001', 'New Delhi', 'Delhi'), - ('500001', 'Hyderabad', 'Telangana'), - ('600001', 'Chennai', 'Tamil Nadu') - ] - :param query: The query / command to run. - :param data: The data to substitute into the query string. - :param commit: Whether, or not, you would like to commit the execution. - :param return_exception: Whether, or not, you would like to return the exception from this function. - :return: Either just the result or the result and the exception. - """ - - # Make sure we have a connection: - await self.ensure_connection() - - # Start by assuming failure: - rows_affected = None - results = None - excp = None - - try: - - # Get a connection and execute the command: - async with self.__pool.acquire() as connection: - async with connection.cursor() as cursor: - rows_affected = await cursor.executemany(query, data) - results = await self.fetch_all(cursor) - if commit: await connection.commit() - - # SQL-specific errors: - except aiomysql.MySQLError as exception: - self.__printer("SQL Exception", exception) - excp = exception - - # Other errors: - except Exception as exception: - self.__printer("Other Exception", exception) - excp = exception - - # Done here: - if return_exception: return rows_affected, results, excp - else: return rows_affected, results - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/dataframe/__init__.py b/utils_v2/dataframe/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/dataframe/splitter.py b/utils_v2/dataframe/splitter.py deleted file mode 100644 index 7725905..0000000 --- a/utils_v2/dataframe/splitter.py +++ /dev/null @@ -1,118 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 22nd Oct., 2024 - - OBJECTIVE: - - To provide an easy way to split dataframes into chunks and divide the workload nto more manageable batches. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - - USAGE EXAMPLE: - - for sub_df in DataFrameSplitter(df, chunk_size = 50): - - # Do your task here: - pass - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class DataFrameSplitter: - - def __init__(self, df, chunk_size): - - """ - Use this to process your dataframe in batches. Useful for when you need to send out alerts at intervals or need - to maintain checkpoints. - :param df: The dataframe to iterate over. - :param chunk_size: The max. no. of rows to pick at once. - """ - - self.df = df - self.row_count = df.shape[0] - self.chunk_size = chunk_size - self.offset = 0 - - def __iter__(self): - self.offset = 0 - return self - - def __next__(self): - if self.offset >= self.row_count: - raise StopIteration - end = self.offset + self.chunk_size - chunk = self.df.iloc[self.offset:end] - self.offset = end - return chunk - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/date_time/__init__.py b/utils_v2/date_time/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc b/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 9b4dd9f..0000000 Binary files a/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc b/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc deleted file mode 100644 index d6cdc35..0000000 Binary files a/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/date_time/date_time.py b/utils_v2/date_time/date_time.py deleted file mode 100644 index 6e57fd5..0000000 --- a/utils_v2/date_time/date_time.py +++ /dev/null @@ -1,271 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 21st jun, 2024 - - OBJECTIVE: - - To provide an easy way to work with time. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For date and time handling: -import pytz -from datetime import datetime, timedelta -import dateparser - -# To handle date-time objects from a Numpy array and Pandas Dataframe: -import numpy as np -import pandas as pd - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# Date-Time Formats: -DATE_TIME_FORMATS = ( - "%d/%m/%y", - "%d-%b-%y", - "%d-%m-%y", - "%d.%m.%y", - "%d/%m/%Y", - "%d-%b-%Y", - "%d-%m-%Y", - "%d.%m.%Y", - "%d/%b", - "%d%m%Y", - "%Y%m%d", - "%Y-%m-%d %H:%M:%S" -) - -# Useful Timezones: -TIMEZONE_UTC = pytz.timezone("UTC") -TIMEZONE_IST = pytz.timezone("Asia/Kolkata") -TIMEZONE_ET = pytz.timezone("America/New_York") -TIMEZONE_CT = pytz.timezone("America/Chicago") -TIMEZONE_MT = pytz.timezone("America/Denver") -TIMEZONE_PT = pytz.timezone("America/Los_Angeles") -TIMEZONE_JST = pytz.timezone("Asia/Tokyo") -TIMEZONE_CET = pytz.timezone("Europe/Paris") -TIMEZONE_GMT = pytz.timezone("GMT") -TIMEZONE_AEST = pytz.timezone("Australia/Sydney") -TIMEZONE_NZST = pytz.timezone("Pacific/Auckland") -TIMEZONE_CST = pytz.timezone("Asia/Shanghai") -TIMEZONE_KST = pytz.timezone("Asia/Seoul") -TIMEZONE_MSK = pytz.timezone("Europe/Moscow") -TIMEZONE_BRT = pytz.timezone("America/Sao_Paulo") -TIMEZONE_GST = pytz.timezone("Asia/Dubai") -TIMEZONE_SAST = pytz.timezone("Africa/Johannesburg") -TIMEZONE_AST = pytz.timezone("Asia/Riyadh") - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def translate_date_time_string( - datetime_string, - source_format = None, - destination_format = "%Y-%m-%dT%H:%M:%S" -): - - """ - To convert an input datetime string to a different format. - :param datetime_string: The datetime string to translate. - :param source_format: The current format of the string. If not provided, dateparser will be used. - :param destination_format: The format to convert to. - :return: The converted datetime string. - """ - - try: - if source_format is None: datetime_obj = dateparser.parse(datetime_string) - else: datetime_obj = datetime.strptime(datetime_string, source_format) - return datetime_obj.strftime(destination_format) - except Exception as exception: - return None - - -# --------------------------------------------------------------------------------------------------------------------- - - -def parse_date_time(input_value, timezone = None, date_formats = None): - - """ - To take any kind of input and interpret the datetime from it. - :param input_value: Either a string or an integer or some form of datetime representation. - :param timezone: The timezone to apply to the interpreted datetime. EXISTING TIMEZONE INFO WILL BE OVERWRITTEN. - :param date_formats: The string formats to consider when parsing a string input. - :return: The parsed datetime or null. - """ - - datetime_object = None - date_formats = date_formats or DATE_TIME_FORMATS - - # It could either be in seconds or milliseconds from epoch time's base date (January 1, 1970), - # or it could be days since Microsoft Excel's base date (December 31, 1899). - if isinstance(input_value, (int, float, np.number)) and not np.isnan(input_value): - if input_value > 9999999999.0: datetime_object = datetime.fromtimestamp(input_value / 1000.0) - if input_value > 999999.0: datetime_object = datetime.fromtimestamp(input_value) - else: datetime_object = datetime.fromtimestamp(input_value * 24 * 60 * 60.0) - timedelta(days = 25569) - - # The input can even be a pre-formatted date: - if isinstance(input_value, str): - datetime_object = dateparser.parse(input_value, date_formats = date_formats) - - # If the type is a datetime object, then return it as it is: - if isinstance(input_value, datetime): - datetime_object = input_value - - # If the type is the native datetime format of pandas: - if isinstance(input_value, pd._libs.tslibs.timestamps.Timestamp): - datetime_object = input_value.to_pydatetime() - - # Process the timezone: - if datetime_object is not None and timezone is not None: - datetime_object = as_if_timezone(datetime_object, timezone) - - # Done here: - return datetime_object - - -# --------------------------------------------------------------------------------------------------------------------- - - -def get_current_date_time(timezone = None, as_string = False): - - """ - Returns the current time as a datetime object. - :param timezone: The timezone to apply to the returned datetime. - :param as_string: Whether, or not, you want the output as a string. - :return: The datetime object/string representing the current time. - """ - - if timezone is not None and isinstance(timezone, str): timezone = pytz.timezone(timezone) - now = datetime.now(timezone) - return now.isoformat() if as_string else now - - -# --------------------------------------------------------------------------------------------------------------------- - - -def get_current_ist_date_time(as_string = False): - - """ - Gives out the current time in IST timezone. - :param as_string: Whether, or not, you want the output as a string. - :return: The datetime object or string representing the current time. - """ - - return get_current_date_time( - timezone = TIMEZONE_IST, - as_string = as_string - ) - - -# --------------------------------------------------------------------------------------------------------------------- - - -def get_current_utc_date_time(as_string = False): - - """ - Gives out the current time in UTC timezone. - :param as_string: Whether, or not, you want the output as a string. - :return: The datetime object or string representing the current time. - """ - - return get_current_date_time( - timezone = TIMEZONE_UTC, - as_string = as_string - ) - - -# --------------------------------------------------------------------------------------------------------------------- - - -def as_if_timezone(datetime_object, timezone): - - """ - Ignores existing timezone info and applies the intended timezone. - The time stays the same, only the timezone marker changes. - e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-09 00:00:00+00:00 - HINT: IT PRETENDS "AS IF" THE TIMEZONE WAS THE INPUT TIMEZONE. - :param datetime_object: The datetime object on which the timezone needs to be applied. - :param timezone: The timezone that needs to be applied. - :return: A timezone-aware datetime object. - """ - - tz_object = pytz.timezone(timezone) if isinstance(timezone, str) else timezone - return tz_object.localize(datetime_object.replace(tzinfo = None)) - - -# --------------------------------------------------------------------------------------------------------------------- - - -def to_timezone(datetime_object, timezone): - - """ - Converts from one timezone to another. The time is adjusted by computing the difference between the two timezones. - NOTE: THIS FUNCTION ASSUMES THE INPUT WAS IN UTC IF THE INPUT WAS TIMEZONE-NAIVE. - e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-08 18:30:00+00:00 - :param datetime_object: The datetime object on which the timezone needs to be applied. - :param timezone: The timezone that needs to be applied. - :return: A timezone-aware datetime object. - """ - - if isinstance(timezone, str): timezone = pytz.timezone(timezone) - if datetime_object.tzinfo is None: return datetime_object.replace(tzinfo = TIMEZONE_UTC) - return datetime_object.astimezone(timezone) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/image/__init__.py b/utils_v2/image/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/image/scanner/__init__.py b/utils_v2/image/scanner/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/image/scanner/scanner.py b/utils_v2/image/scanner/scanner.py deleted file mode 100644 index 28fe69b..0000000 --- a/utils_v2/image/scanner/scanner.py +++ /dev/null @@ -1,410 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 19th Sept., 2024 - - OBJECTIVE: - - To provide a way to easily scan documents like printouts and visiting cards. - - REFERENCES: - - 01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085 - 02. https://github.com/JaidedAI/EasyOCR - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys - -from diffusers.utils.import_utils import candidates - -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# To work with image processing: -import cv2 -import imutils -import numpy as np - -# To run OCR: -import easyocr - -# To download images from the web: -import requests - -# My utils: -from utils_v2.string import json -from utils_v2.ai.object_detection.yolo import YoloDetect - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class DocumentScanner: - - def __init__( - self, - layout_detection_yolo = None, - whitelisted_yolo_classes = None, - ocr_languages = None - ): - - self.__yolo = YoloDetect( - model_file = layout_detection_yolo, - debug = False - ) - self.__whitelisted_yolo_classes = whitelisted_yolo_classes or [] - self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"]) - pass - - @staticmethod - def open_image(image): - - """ - Opens an image from various sources. - :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. - :return: The image as a CV2 object. - """ - - # When the image is provided as a string, - # it could be either as a URL or a path to a local file: - if isinstance(image, str): - if image.startswith("https://") or image.startswith("http://"): - image = io.BytesIO(requests.get(image).content) - else: image = cv2.imread(image) - - # If the image is provided as a io.BytesIO buffer: - if isinstance(image, io.BytesIO): - image.seek(0) - image = np.asarray(bytearray(image.read()), dtype = np.uint8) - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - - # Done here: - return image - - @staticmethod - def polygon_area(points): - - """ - Computes the area occupied by a shape described by the array of points. - Example input: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]] - :return: The area of the shape. - """ - - n = len(points) - area = 0.0 - for i in range(n): - j = (i + 1) % n # next vertex - area += points[i][0] * points[j][1] - area -= points[j][0] * points[i][1] - return abs(area) / 2.0 - - @staticmethod - def show(image, title = "Preview", wait = True): - - """ - Just a quick wrapper to show the image in a window. - :param image: The image that you want to show. - :param title: The title of the window. - :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many - windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the - very last one. - :return: None. - """ - - cv2.imshow(title, image) - if wait: - cv2.waitKey() - cv2.destroyAllWindows() - - @staticmethod - def enhance(image): - - """ - Applies a form of contrast boost to make the edges more easily visible. - :param image: The OpenCV image that needs to be enhanced. - :return: The enhanced CV2 image. - """ - - return cv2.detailEnhance( - src = image, - sigma_s = 20, - sigma_r = 0.15 - ) - - @staticmethod - def get_edges(image): - - """ - Gets the edges in an image. - :param image: A CV2 image. - :return: The image with the edges detected. - """ - - # Denoise the image: - processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - processed_image = cv2.GaussianBlur( - processed_image, - ksize = (5, 5), - sigmaX = 0 - ) - - # Detect edges in the image: - processed_image = cv2.Canny( - processed_image, - threshold1 = 50, - threshold2 = 200 - ) - - # Close small gaps in the edges: - kernel = np.ones((5, 5), np.uint8) - processed_image = cv2.dilate(processed_image, kernel, iterations = 1) - # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel) - # processed_image = cv2.erode(processed_image, kernel, iterations = 1) - - # Done here: - return processed_image - - def scan(self, image, min_area = 0.125): - - """ - Looks for a rectangle in an image and flattens it out. No colour filters are applied here. - :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. - :param min_area: The minimum area occupied by the document in the image. - :return: The flattened image if any, else None. - """ - - # Open the image and read the data: - image = self.open_image(image) - if image is None: return None - height, width = image.shape[:2] - image_area = height * width - - # Enhance the image to have better visibility of edges: - # processed_image = self.enhance(image.copy()) - - # Detect edges in the image: - processed_image = self.get_edges(image.copy()) - - # Find the contours in the edges, - # and sort them in ascending order: - contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) - contours = imutils.grab_contours(contours) - contours = sorted(contours, key = cv2.contourArea, reverse = True) - - # We try to get clean and passable contours. - # For our purposes, 'clean' is when it can be approximated to exactly 4 sides, - # and 'passable' is when the approximation comes close to it: - clean_rects = [] - passable_rects = [] - for contour in contours: - perimeter = cv2.arcLength(contour, closed = True) - approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True) - if len(approximation) == 4: clean_rects.append(approximation) - elif len(approximation) <= 32: - candidate_contour = cv2.minAreaRect(approximation) - candidate_contour = cv2.boxPoints(candidate_contour) - candidate_contour = np.intp(candidate_contour) - passable_rects.append(candidate_contour) - clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True) - passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True) - - # Now we decide between the best candidate: - document_outline = None - if len(clean_rects) > 0: - candidate_rect = np.array([p[0] for p in clean_rects[0]]) - rectangle_area = cv2.contourArea(candidate_rect) - if rectangle_area / image_area >= min_area: document_outline = candidate_rect - if document_outline is None and len(passable_rects) > 0: - candidate_rect = cv2.minAreaRect(passable_rects[0]) - candidate_rect = cv2.boxPoints(candidate_rect) - candidate_rect = np.intp(candidate_rect) - rectangle_area = cv2.contourArea(candidate_rect) - if rectangle_area / image_area >= min_area: document_outline = candidate_rect - - # If there is no rectangular contour, - # we exit with failure: - if document_outline is None: return None - - # Since we did get the best candidate for the document, - # we figure out which point n the contour is which corner of the document: - document_rectangle = np.zeros((4, 2), dtype = "float32") - sum_points = document_outline.sum(axis = 1) - document_rectangle[0] = document_outline[np.argmin(sum_points)] - document_rectangle[2] = document_outline[np.argmax(sum_points)] - diff_points = np.diff(document_outline, axis = 1) - document_rectangle[1] = document_outline[np.argmin(diff_points)] - document_rectangle[3] = document_outline[np.argmax(diff_points)] - (top_left, top_right, bottom_right, bottom_left) = document_rectangle - - # let's compute the dimensions of the document in the image: - left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2)) - right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2)) - top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2)) - bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2)) - max_height = max(int(left_height), int(right_height)) - max_width = max(int(top_width), int(bottom_width)) - - # We compute the destination of the transform: - destination = np.array( - object = [ - [0, 0], - [max_width - 1, 0], - [max_width - 1, max_height - 1], - [0, max_height - 1] - ], - dtype = "float32" - ) - - # We apply the transform that flattens out the document: - matrix = cv2.getPerspectiveTransform(document_rectangle, destination) - flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height)) - - # Done here: - return flattened_image - - def extract_text( - self, - image, - min_confidence = 0.5, - margin = 0.05 - ): - - """ - To run OCR on the input image. - :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. - :param min_confidence: The minimum amount of confidence for detected text to be considered. - :param margin: The extra pixels to include when cropping into a section for OCR. - :return: The extracted text. - """ - - # Make a variable that will hold the final result: - extracted_text = {} - - # Open the image and read the data: - image = self.open_image(image) - height, width = image.shape[:2] - - # Run OCR on the entire canvas: - ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence] - extracted_text["fullDoc"] = " ".join(ocr_result) - - # Get the doc layout in the image: - document_sections = self.__yolo.predict(image) - document_sections = document_sections["boxes"] - - # For each section, we run the OCR process individually: - section_results = [] - for section in document_sections: - if section["class"] in self.__whitelisted_yolo_classes: - section_width = section["x2"] - section["x1"] - section_height = section["y2"] - section["y1"] - x1 = max(int(section["x1"] - (margin * section_width)), 0) - y1 = max(int(section["y1"] - (margin * section_height)), 0) - x2 = min(int(section["x2"] + (margin * section_width)), width) - y2 = min(int(section["y2"] + (margin * section_height)), height) - sub_image = image[y1:y2, x1:x2] - ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence] - section_results.append({ - "text": " ".join(ocr_result), - "class": section["class"], - "className": section["className"], - "x1": x1, - "y1": y1, - "x2": x2, - "y2": y2 - }) - extracted_text["bySection"] = section_results - - # Done here: - return extracted_text - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import time - - my_scanner = DocumentScanner( - layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", - whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], - # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], - ocr_languages = ["en"] - ) - - image = my_scanner.open_image(r"/home/developer/Downloads/talkaholics_card.jpg") - document_image = my_scanner.scan(image) - if document_image is not None: - cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image) - start_time = time.time() - document_text = my_scanner.extract_text( - document_image, - min_confidence = 0.5 - ) - print("OCR RESULT:") - print(document_text) - - print(f"FINISHED IN {time.time() - start_time} SECONDS!") - - else: print("No image") diff --git a/utils_v2/image/scanner/scanner_v2.py b/utils_v2/image/scanner/scanner_v2.py deleted file mode 100644 index fc63627..0000000 --- a/utils_v2/image/scanner/scanner_v2.py +++ /dev/null @@ -1,379 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 24th Sept., 2024 - - OBJECTIVE: - - To provide a way to easily scan documents like printouts and visiting cards. - - REFERENCES: - - 01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085 - 02. https://github.com/JaidedAI/ - 03. https://github.com/criistian14/flutter_document_scanner/blob/master/flutter_document_scanner_android/android/src/main/kotlin/com/christian/flutterDocumentScanner/OpenCVPlugin.kt - 04. https://www.geeksforgeeks.org/python-bilateral-filtering/ - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# To work with image processing: -import cv2 -import imutils -import numpy as np - -# To run OCR: -import easyocr - -# To download images from the web: -import requests - -# My utils: -from utils_v2.string import json -from utils_v2.ai.object_detection.yolo import YoloDetect - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class DocumentScanner: - - def __init__( - self, - layout_detection_yolo = None, - whitelisted_yolo_classes = None, - ocr_languages = None - ): - - self.__yolo = YoloDetect( - model_file = layout_detection_yolo, - debug = False - ) - self.__whitelisted_yolo_classes = whitelisted_yolo_classes or [] - self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"]) - pass - - @staticmethod - def open_image(image): - - """ - Opens an image from various sources. - :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. - :return: The image as a CV2 object. - """ - - # When the image is provided as a string, - # it could be either as a URL or a path to a local file: - if isinstance(image, str): - if image.startswith("https://") or image.startswith("http://"): - image = io.BytesIO(requests.get(image).content) - else: image = cv2.imread(image) - - # If the image is provided as a io.BytesIO buffer: - if isinstance(image, io.BytesIO): - image.seek(0) - image = np.asarray(bytearray(image.read()), dtype = np.uint8) - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - - # Done here: - return image - - @staticmethod - def show(image, title = "Preview", wait = True): - - """ - Just a quick wrapper to show the image in a window. - :param image: The image that you want to show. - :param title: The title of the window. - :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many - windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the - very last one. - :return: None. - """ - - cv2.imshow(title, image) - if wait: - cv2.waitKey() - cv2.destroyAllWindows() - - @staticmethod - def get_edges(image): - - """ - Apply all the preprocessing filters on the image before sending it off for contour-finding. - :param image: The image to pre-process. - :return: The pre-processed image. - """ - - # Denoise the image: - processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - processed_image = cv2.bilateralFilter( - processed_image, - d = 9, - sigmaColor = 75, - sigmaSpace = 75 - ) - - # Detect edges in the image: - processed_image = cv2.Canny( - processed_image, - threshold1 = 75, - threshold2 = 200 - ) - - # Close small gaps in the edges: - kernel = np.ones((5, 5), np.uint8) - processed_image = cv2.dilate(processed_image, kernel, iterations = 1) - # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel) - # processed_image = cv2.erode(processed_image, kernel, iterations = 1) - - # Done here: - return processed_image - - def scan(self, image, min_area = 0.125): - - """ - Looks for a rectangle in an image and flattens it out. No colour filters are applied here. - :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. - :param min_area: The minimum area occupied by the document in the image. - :return: The flattened image if any, else None. - """ - - # Open the image and read the data: - image = self.open_image(image) - if image is None: return None - height, width = image.shape[:2] - image_area = height * width - - # image = self.enhance(image) - processed_image = self.get_edges(image) - # self.show(processed_image) - - # Find the contours in the edges, - # and sort them in ascending order: - contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) - contours = imutils.grab_contours(contours) - contours = sorted(contours, key = cv2.contourArea, reverse = True) - - # We try to get clean and passable contours. - # For our purposes, 'clean' is when it can be approximated to exactly 4 sides, - # and 'passable' is when the approximation comes close to it: - clean_rects = [] - passable_rects = [] - for contour in contours: - perimeter = cv2.arcLength(contour, closed = True) - approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True) - if len(approximation) == 4: clean_rects.append(approximation) - elif len(approximation) <= 32: - candidate_contour = cv2.minAreaRect(approximation) - candidate_contour = cv2.boxPoints(candidate_contour) - candidate_contour = np.intp(candidate_contour) - passable_rects.append(candidate_contour) - clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True) - passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True) - - # Now we decide between the best candidate: - document_outline = None - if len(clean_rects) > 0: - candidate_rect = np.array([p[0] for p in clean_rects[0]]) - rectangle_area = cv2.contourArea(candidate_rect) - if rectangle_area / image_area >= min_area: document_outline = candidate_rect - if document_outline is None and len(passable_rects) > 0: - candidate_rect = cv2.minAreaRect(passable_rects[0]) - candidate_rect = cv2.boxPoints(candidate_rect) - candidate_rect = np.intp(candidate_rect) - rectangle_area = cv2.contourArea(candidate_rect) - if rectangle_area / image_area >= min_area: document_outline = candidate_rect - - # If there is no rectangular contour, - # we exit with failure: - if document_outline is None: return None - - # Since we did get the best candidate for the document, - # we figure out which point n the contour is which corner of the document: - document_rectangle = np.zeros((4, 2), dtype = "float32") - sum_points = document_outline.sum(axis = 1) - document_rectangle[0] = document_outline[np.argmin(sum_points)] - document_rectangle[2] = document_outline[np.argmax(sum_points)] - diff_points = np.diff(document_outline, axis = 1) - document_rectangle[1] = document_outline[np.argmin(diff_points)] - document_rectangle[3] = document_outline[np.argmax(diff_points)] - (top_left, top_right, bottom_right, bottom_left) = document_rectangle - - # let's compute the dimensions of the document in the image: - left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2)) - right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2)) - top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2)) - bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2)) - max_height = max(int(left_height), int(right_height)) - max_width = max(int(top_width), int(bottom_width)) - - # We compute the destination of the transform: - destination = np.array( - object = [ - [0, 0], - [max_width - 1, 0], - [max_width - 1, max_height - 1], - [0, max_height - 1] - ], - dtype = "float32" - ) - - # We apply the transform that flattens out the document: - matrix = cv2.getPerspectiveTransform(document_rectangle, destination) - flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height)) - - # Done here: - return flattened_image - - def extract_text( - self, - image, - min_confidence = 0.5, - margin = 0.05 - ): - - """ - To run OCR on the input image. - :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer. - :param min_confidence: The minimum amount of confidence for detected text to be considered. - :param margin: The extra pixels to include when cropping into a section for OCR. - :return: The extracted text. - """ - - # Make a variable that will hold the final result: - extracted_text = {} - - # Open the image and read the data: - image = self.open_image(image) - height, width = image.shape[:2] - - # Run OCR on the entire canvas: - ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence] - extracted_text["fullDoc"] = " ".join(ocr_result) - - # Get the doc layout in the image: - document_sections = self.__yolo.predict(image) - document_sections = document_sections["boxes"] - - # For each section, we run the OCR process individually: - section_results = [] - for section in document_sections: - if section["class"] in self.__whitelisted_yolo_classes: - section_width = section["x2"] - section["x1"] - section_height = section["y2"] - section["y1"] - x1 = max(int(section["x1"] - (margin * section_width)), 0) - y1 = max(int(section["y1"] - (margin * section_height)), 0) - x2 = min(int(section["x2"] + (margin * section_width)), width) - y2 = min(int(section["y2"] + (margin * section_height)), height) - sub_image = image[y1:y2, x1:x2] - ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence] - section_results.append({ - "text": " ".join(ocr_result), - "class": section["class"], - "className": section["className"], - "x1": x1, - "y1": y1, - "x2": x2, - "y2": y2 - }) - extracted_text["bySection"] = section_results - - # Done here: - return extracted_text - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import time - - my_scanner = DocumentScanner( - layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", - whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], - # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], - ocr_languages = ["en"] - ) - - image = my_scanner.open_image(r"/home/developer/Downloads/sushmita_card.jpg") - # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card.jpg") - # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card_2.jpg") - # image = my_scanner.open_image(r"/home/developer/Downloads/card_square.jpg") - document_image = my_scanner.scan(image) - if document_image is not None: - cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image) - start_time = time.time() - document_text = my_scanner.extract_text( - document_image, - min_confidence = 0.5 - ) - print("OCR RESULT:") - print(document_text) - - print(f"FINISHED IN {time.time() - start_time} SECONDS!") - - else: print("No image") diff --git a/utils_v2/logging/__init__.py b/utils_v2/logging/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/logging/__pycache__/__init__.cpython-310.pyc b/utils_v2/logging/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 3141c9c..0000000 Binary files a/utils_v2/logging/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/logging/__pycache__/model.cpython-310.pyc b/utils_v2/logging/__pycache__/model.cpython-310.pyc deleted file mode 100644 index cd8e999..0000000 Binary files a/utils_v2/logging/__pycache__/model.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/logging/context.py b/utils_v2/logging/context.py deleted file mode 100644 index 0591ae6..0000000 --- a/utils_v2/logging/context.py +++ /dev/null @@ -1,469 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 2nd Aug., 2024 - - OBJECTIVE: - - To provide an easy way to log all system activities by way of managing the context of what is going on. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - - NOTES: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To manage context: -import contextvars -from contextlib import contextmanager, asynccontextmanager - -# To make decorators: -from functools import wraps - -# For system-level activities: -import os - -# My utils: -from utils_v2.string import json -from utils_v2.date_time import date_time -from utils_v2.security import sanitizers -from utils_v2.api.codes import StatusCodes, HttpCodes -from utils_v2.api.log import APILogModel -from utils_v2.api.response import ResponseModel - -# The needed data models: -from utils_v2.logging.model import GeneralLogModel - -# To work with date and time: -import time -import datetime - -# For random strings: -import random - -# For debugging: -import traceback -import string - -# To work with datatypes: -from types import NoneType -import pandas as pd - -# To work with Pydantic objects: -from pydantic import BaseModel - -# For asynchronous activities: -import asyncio - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# Chars to choose from for random strings: -ALPHANUMERIC_CHARS = string.ascii_letters + string.digits - -# To capture system information: -PROCESS_ID = os.getppid() -PARENT_PROCESS_ID = os.getppid() - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** EXCEPTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def describe_exception(exc): - - """ - Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire - traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it - to the dev team over some service like WhatsApp/Telegram. - :param exc: The exception that occurred. - :return: The dict that explains the exception. - """ - - exc_desc = { - "type": type(exc).__name__, - "msg": str(exc), - "tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)] - } - - return exc_desc - - -# ***************************************************************************************************************** -# ***** **** -# *** Classes *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncMongoLogger: - - def __init__(self, db_conn, collection = "logs"): - - """ - This class uses an instance of 'AsyncMongo' and makes it usable as a logger. - :param db_conn: The instance of 'AsyncMongo' to use. - :param collection: The collection to write the log into. - """ - - self.__db_conn = db_conn - self.__collection = collection - - async def log(self, log_json): - - """ - Log something to the database using the connection provided when the object was made. - :param log_json: The dict to log. - :return: True if logged successfully, else False. - """ - - asyncio.create_task(self.__db_conn.insert_one( - collection = self.__collection, - document = log_json, - raise_exception = False - )) - - -# --------------------------------------------------------------------------------------------------------------------- - - -class AsyncLoggerContext: - - # Create the context-aware variable(s): - logger = contextvars.ContextVar("logger", default = None) - log_chain = contextvars.ContextVar("log_chain", default = None) - - @classmethod - @asynccontextmanager - async def logging_context(cls, logger, log_chain = None): - - """ - This function makes the context manager that makes the value of the log chain available to everything that is - called within the scope of the context. - :param logger: The object which is to be used to write the log. It should have a 'log' method which should take - in a dict as its input. - :param log_chain: The value of the log chain to be made available within the scope. - :return: None. - """ - - # Set the context: - token_logger = cls.logger.set(logger) - token_log_chain = cls.log_chain.set(log_chain) - - # Make the objects available within the context: - try: yield - - # Release the objects when the context is over: - finally: - cls.logger.reset(token_logger) - cls.log_chain.reset(token_log_chain) - - @staticmethod - def generate_log_id(count = 8): - return "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(min(8, count))) - - @classmethod - def get_logger(cls): - return cls.logger.get() - - @classmethod - def get_log_chain(cls): - return cls.log_chain.get() - - @staticmethod - def summarize( - value, - str_limit = 100, - expand: bool | int = False, - sensitive_keys: list[str] = None - ): - - """ - To summarize an input value to capture the essence without hoarding to much data. - :param value: Anything that you want to summarize. - :param str_limit: The max. no. of chars of a string to retain. - :param expand: Set to True for full expansion, False for no expansion, and an integer for a specific level of - expansion. Applicable on iterables and dicts. The smaller this number, the more concise the summary will be, - and vice versa. - :param sensitive_keys: The list of keys (of a dict) to obscure when summarizing. - :return: The summarized version of the input. - """ - - # If the input is a Pydantic class: - if isinstance(value, BaseModel): value = value.model_dump() - - # Check the sensitive keys: - if sensitive_keys is None: sensitive_keys = [] - - # Handle datatypes that you don't want to modify: - if isinstance(value, (int, float, bool, NoneType)): pass - - # When the value is a list or similar iterable: - elif isinstance(value, (list, tuple, set)): - if expand: - if not isinstance(expand, bool): expand -= 1 - value = [AsyncLoggerContext.summarize( - v, - expand = expand, - sensitive_keys = sensitive_keys - ) for v in value] - else: value = f"array of {len(value)} item(s)" - - # If the value is a dict: - elif isinstance(value, dict): - if expand: - if not isinstance(expand, bool): expand -= 1 - value = { - k: AsyncLoggerContext.summarize( - v, - expand = expand, - sensitive_keys = sensitive_keys - ) if k not in sensitive_keys else "********" - for k, v in value.items() - } - else: value = f"object of {len(value.keys())} field(s) [{', '.join(value.keys())}]" - - # When a dataframe is passed: - elif isinstance(value, pd.DataFrame): - cols = value.columns.to_list() - value = f"table with {len(cols)} col(s) [{', '.join(cols)}] and {len(value)} row(s)" - str_limit = 999 - - # If the input is some form of non-standard object: - else: value = str(value) - - # Handle strings: - if isinstance(value, str): - if len(value) > str_limit: value = value[:str_limit] + "..." - - # Done here: - return value - - @classmethod - def log_it( - cls, - api_version: str = None, - project: str = None, - log_type: str = None, - operation: str = None, - log_input: bool | int = True, - log_output: bool | int = True, - sensitive_keys: list = None - ): - - """ - A decorator factor that can be used to log the results of functions automatically. - :param api_version: A string that indicates the version code of the function being decorated. - :param project: A hint about which project is being worked on. - :param log_type: A hint about which module is being worked on. - :param operation: A hint about which action in a particular module is being worked on. - :param log_input: Set to True to capture everything that went into the function, False to capture the least - info, and set it to an integer to capture a certain depth of the input (applicable on iterables and dicts. - :param log_output: The same as 'log_input', but applicable to the response from the function. - :param sensitive_keys: Keys of a dict whose values must be obscured even if that depth is being captured. - :return: A decorator with the configuration. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Make variables and extract available info.: - exception = None - response = None - request_ts = date_time.get_current_utc_date_time() - start_ts = time.perf_counter() - cpu_start_ts = time.process_time() - - # Execute the function that is being wrapped: - try: response = await func(*args, **kwargs) - except Exception as exc: exception = exc - - # Do the next steps only if within the logging context: - if cls.get_logger() is not None: - - # Create the log: - if not args: args = None - if not kwargs: kwargs = None - func_log = GeneralLogModel( - pid = PROCESS_ID, - ppid = PARENT_PROCESS_ID, - project = project, - log = log_type or func.__name__, - operation = operation or func.__name__, - apiVer = api_version, - logId = cls.generate_log_id(), - logChain = cls.get_log_chain(), - ts = request_ts, - tat = time.perf_counter() - start_ts, - cpuTime = time.process_time() - cpu_start_ts, - func = func.__name__, - args = cls.summarize(args, expand = log_input, sensitive_keys = sensitive_keys), - kwargs = cls.summarize(kwargs, expand = log_input, sensitive_keys = sensitive_keys), - exception = None if exception is None else describe_exception(exception), - response = cls.summarize(response, expand = log_output, sensitive_keys = sensitive_keys), - ).model_dump() - - # Write the log: - await cls.get_logger().log(func_log) - - # Done here: - if exception is not None: raise exception - return response - - return wrapper - - return decorator - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - from utils_v2.database.async_mongo_v2 import AsyncMongo - - @AsyncLoggerContext.log_it( - api_version = "0.0.1", - project = "testProj", - log_type = "work", - operation = "someWork", - log_input = True, - log_output = True, - sensitive_keys = None - ) - async def some_work(*args, **kwargs): - print("SOME WORK:", AsyncLoggerContext.get_log_chain()) - await asyncio.sleep(max(2.0 * random.random(), 1.0)) - total = sum(args) - return total - - @AsyncLoggerContext.log_it( - api_version = "0.0.1", - project = "testProj", - log_type = "work", - operation = "moreWork", - log_input = True, - log_output = True, - sensitive_keys = ["password", "sessionToken"] - ) - async def more_work(*args, **kwargs): - print("MORE WORK:", AsyncLoggerContext.get_log_chain()) - await asyncio.sleep(max(2.0 * random.random(), 1.0)) - return {"success": True, "sessionToken": "1234567890"} - - @AsyncLoggerContext.log_it( - api_version = "0.0.1", - project = "testProj", - log_type = "work", - operation = "moreWork", - log_input = True, - log_output = True, - sensitive_keys = None - ) - async def last_work(*args, **kwargs): - print("LAST WORK:", AsyncLoggerContext.get_log_chain()) - await asyncio.sleep(max(2.0 * random.random(), 1.0)) - - async def main(chain = None): - - # Connect to MongoDB: - mongo = AsyncMongo( - connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external", - database_name = "converse", - max_connections = 10, - debug = True - ) - - # Convert the connection to a logger instance that can be injected - # into the context as a dependency: - mongo_logger = AsyncMongoLogger( - db_conn = mongo, - collection = "logs" - ) - - # Initialize the context: - async with AsyncLoggerContext.logging_context( - logger = mongo_logger, - log_chain = chain - ): - - # Run some functions within the context: - await some_work(1, 2, 3, 4, 5) - await more_work(username = "john.doe@domain.com", password = "mySecretPass") - - # Run something outside the context: - await last_work() - - # async def multi_main(): - # tasks = [ - # main(chain = "kPRwXdItb1"), - # main(chain = "456") - # ] - # await asyncio.gather(*tasks) - - asyncio.run(main(chain = "00wGHRFYPY123")) diff --git a/utils_v2/logging/model.py b/utils_v2/logging/model.py deleted file mode 100644 index 355d07e..0000000 --- a/utils_v2/logging/model.py +++ /dev/null @@ -1,131 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 11th Oct., 2024 - - OBJECTIVE: - - To have a structure for the logs maintained for regular function calls. - This is different from the logs maintained for API calls. - - REFERENCES: - - 01. https://chatgpt.com/share/6708b6c9-6ba4-800f-9ea9-00ec35067512 - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# System-level activities: -import distro -import socket -import platform - -# For data-modelling: -from pydantic import BaseModel, Field -from typing import Any, Optional, List, Literal - -# To work with date and time: -import datetime - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# Info for logging that will stay constant during runtime: -SERVER_HOSTNAME = str(socket.gethostname()) -PLATFORM_INFO = platform.uname() -HOST_OS = str(distro.name(True)) -HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class GeneralLogModel(BaseModel): - - # To identify the machine the code is running on. - # DO NOT MODIFY THESE: - hostname: str = SERVER_HOSTNAME - os: str = HOST_OS - cpu: str = HOST_CPU - # Can modify these: - pid: Optional[Any] = None - ppid: Optional[Any] = None - - # To identify the project and actions: - project: Optional[str] = None - log: str - operation: Optional[str] = None - apiVer: Optional[str] = None - logId: Optional[str] = None - logChain: Optional[str] = None - - # Timing metrics: - ts: datetime.datetime - tat: float - cpuTime: float - - # To understand the inputs: - func: str - args: Optional[Any] = None - kwargs: Optional[Any] = None - - # To understand the outputs: - exception: Optional[Any] = None - response: Optional[Any] = None - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/mail/__init__.py b/utils_v2/mail/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/mail/async_mail.py b/utils_v2/mail/async_mail.py deleted file mode 100644 index 839873c..0000000 --- a/utils_v2/mail/async_mail.py +++ /dev/null @@ -1,440 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 16th Jul, 2024 - - OBJECTIVE: - - To be able to send out mails from code. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For working with mails: -import aiosmtplib -from email.mime.multipart import MIMEMultipart -from email.mime.image import MIMEImage -from email.mime.text import MIMEText -from email.mime.base import MIMEBase -from email import encoders - -# My utils: -from utils import rate_limit_utils - -# Common: -from shared.statuses import StatusCodes - -# For random strings: -import string -import random - -# For system-level activities: -import os - -# For working with files in RAM: -import io - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class MailMessage: - - def __init__(self, to_email, subject): - - """ - Create an instance of the message that you would like to send. - :param to_email: The EMail ID of th recipient. - :param subject: The subject of the mail. - """ - - self.message = MIMEMultipart() - self.message["To"] = to_email - self.message["Subject"] = subject - - def add_text(self, text): - - """ - Add plain-text to the mail body. - :param text: The text to add to the mail body. - :return: None. - """ - - self.message.attach(MIMEText(text, "plain")) - - def add_html(self, html_text): - - """ - Add HTML text to the mail body. - :param html_text: The HTML text to add to the mail body. - :return: None. - """ - - self.message.attach(MIMEText(html_text, "html")) - - def add_inline_image(self, image_file, content_id = None): - - """ - Add an inline image to the body of the mail. - NOTE: This is NOT the same as sending an image as an attachment. - :param image_file: The image data to attach to the mail body. - :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not - specified, I will generate a random string. You may write a custom value here if you know what you are - doing. For most use cases, please ignore this field. - :return: None. - """ - - # Read the image as bytes: - image_bytes = None - if type(image_file) is str: - with open(image_file, "rb") as opened_image_file: - image_bytes = opened_image_file.read() - if type(image_file) is io.BytesIO: - image_file.seek(0) - image_bytes = image_file.getvalue() - - # Declare the part to be attached to the multipart message: - if image_bytes is not None: - - # Create the HTML block if the image pointer is blank: - if content_id is None: - content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) - self.add_html(f""" - - -

- - - """) - - # Then add the image: - image_part = MIMEImage(image_bytes) - image_part.add_header("Content-ID", f"<{content_id}>") - self.message.attach(image_part) - - def add_attachment(self, attachment_file, file_name = None): - - # Declare the part to be attached to the multipart message: - part = MIMEBase("application", "octet-stream") - - # If the attachment is a file stored in the local disk: - if type(attachment_file) is str: - file_name = file_name or os.path.split(attachment_file)[-1] - with open(attachment_file, "rb") as attachment: - part.set_payload(attachment.read()) - - # If the file is held in RAM: - if type(attachment_file) is io.BytesIO(): - attachment_file.seek(0) - part.set_payload(attachment_file.read()) - - # Encode and attach the file: - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f"attachment; filename= {file_name}", - ) - self.message.attach(part) - - def get_message(self): - return self.message - - -# --------------------------------------------------------------------------------------------------------------------- - - -class AsyncMailClient: - - # Constants: - SMTP_TLS_PORT = 587 - SMTP_SSL_PORT = 465 - - # variables: - __smtp = None - - def __init__( - self, - email, - password, - server, - port = 587, - rate_limiters = None, - wait_for_turn = True, - debug = True, - debug_prefix = "Mail (C) | " - ): - - """ - Set up the mail client. - :param email: The Email ID to use when sending out mails. - :param password: The password of the EMail ID that is being used. - :param server: The EMail server. - :param port: The port number to connect to the host. - :param rate_limiters: The rate limiters to use. Must have "get_turn" and "has_turn" methods. "get_turn" method - must wait for the turn, and "has_turn" method must only check if a turn is available. - :param wait_for_turn: To wait for turn if the rate limit has been exceeded, or to return with failure. - :param debug: Whether, or not, you want to print debugging messages. - :param debug_prefix: The prefix to identify the debugging messages. - """ - - # Initialize the debugger: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - # Note down the credentials and other details: - self.__email = email - self.__password = password - self.__server = server - self.__port = port - self.__rate_limiters = rate_limiters if type(rate_limiters) is list else ([rate_limiters] if rate_limiters is not None else []) - self.__wait_for_turn = wait_for_turn - - def enable_debug(self): - self.__printer.enable() - - def disable_debug(self): - self.__printer.disable() - - async def login(self): - - """ - To connect to the mail server and authenticate the user. - :return: True if authenticated, else False. - """ - - # Initialize the SMTP connection, - # and return with success if all goes well: - try: - self.__smtp = aiosmtplib.SMTP( - hostname = self.__server, - port = self.__port, - use_tls = False, - start_tls = False - ) - await self.__smtp.connect() - await self.__smtp.starttls() - await self.__smtp.login(self.__email, self.__password) - return True - - # Return with failure if something goes wrong: - except Exception as exception: - self.__printer(exception) - try: await self.__smtp.quit() - except Exception as exception: self.__printer(exception) - self.__smtp = None - return False - - async def ensure_connection(self): - - """ - Can be run before the sending operation to confirm that we are yet connected to the server. - If not connected, this code will reattempt to connect and log-in. - :return: True if connected, else False. - """ - - # If the login had failed initially, the object will be set to null. - # In such a case, we make an attempt to login: - if self.__smtp is None: - return await self.login() - - # If the login was successful, we check if the connection is active. - # If not, we try to re-login: - if self.__smtp.is_connected: - return True - else: - try: - await self.__smtp.connect() - await self.__smtp.starttls() - await self.__smtp.login(self.__email, self.__password) - return True - except Exception as exception: - self.__printer(exception) - try: await self.__smtp.quit() - except Exception as exception: self.__printer(exception) - self.__smtp = None - return False - - async def logout(self): - - """ - Closes the connection to the SMTP client. - :return: True by default. - """ - - if self.__smtp is not None: - try: await self.__smtp.quit() - except Exception as exception: self.__printer(exception) - self.__smtp = None - return True - - async def send(self, mail: MailMessage): - - """ - Send out the mail. - :param mail: The instance of 'MailMessage' with all the content populated. - :return: A dict with 'success' and 'message'. - """ - - # Return with failure if we aren't connected, - # and our attempt to (re)connect fails: - if not await self.ensure_connection(): - return { - "success": False, - "message": "login failed" - } - - # Comply with the rate-limit: - for rate_limiter in self.__rate_limiters: - if not self.__wait_for_turn: - if not await rate_limiter.has_turn(): return False - got_turn = await rate_limiter.get_turn() - if not got_turn: - return { - "success": False, - "message": "rate-limit wait timeout" - } - - # Try to send the message: - try: - mail.message["From"] = self.__email - await self.__smtp.send_message(mail.message) - return { - "success": True, - "message": "mail sent" - } - - # If something goes wrong: - except Exception as exception: - self.__printer(exception) - return { - "success": False, - "message": str(exception) - } - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - from utils_v2.string import json - from utils_v2.mail.mail_message import MailMessage - - async def test(): - - rate_lim = rate_limit_utils.TokenBucket( - rate_limit = 1, - seconds = 60.0, - ) - - mail_client = AsyncMailClient( - email = "sender@gmail.com", - password = "secret_password", - server = "smtp.gmail.com", - rate_limiters = rate_lim - ) - - my_mail = MailMessage( - to_email = "recipient@gmail.com", - subject = "Bhopli is the best!" - ) - my_mail.add_html( - """ - - - - - Sample HTML String - - - -

Hello, Bhopli!

-

Bhopli is the best, most well-behaved cat in the known universe.

- - - """ - ) - my_mail.add_text("This is how you should pet her 👇") - my_mail.add_inline_image(r"/path/to/image/cat_petting.png") - my_mail.add_attachment(r"/path/to/file/sample_label.pdf") - - await mail_client.login() - result = await mail_client.send(my_mail) - print("MAIL RESULT:", json.to_json_string(result)) - await mail_client.logout() - - asyncio.run(test()) diff --git a/utils_v2/mail/mail_message.py b/utils_v2/mail/mail_message.py deleted file mode 100644 index 5887aae..0000000 --- a/utils_v2/mail/mail_message.py +++ /dev/null @@ -1,268 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 16th Jul, 2024 - - OBJECTIVE: - - To be able to send out mails from code. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For working with mails: -import aiosmtplib -from email.mime.multipart import MIMEMultipart -from email.mime.image import MIMEImage -from email.mime.text import MIMEText -from email.mime.base import MIMEBase -from email import encoders - -# My utils: -from utils import rate_limit_utils - -# Common: -from shared.statuses import StatusCodes - -# For random strings: -import string -import random - -# For system-level activities: -import os - -# For working with files in RAM: -import io - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class MailMessage: - - def __init__(self, to_email, subject): - - """ - Create an instance of the message that you would like to send. - :param to_email: The EMail ID of th recipient. - :param subject: The subject of the mail. - """ - - self.message = MIMEMultipart() - self.message["To"] = to_email - self.message["Subject"] = subject - - def add_text(self, text): - - """ - Add plain-text to the mail body. - :param text: The text to add to the mail body. - :return: None. - """ - - self.message.attach(MIMEText(text, "plain")) - - def add_html(self, html_text): - - """ - Add HTML text to the mail body. - :param html_text: The HTML text to add to the mail body. - :return: None. - """ - - self.message.attach(MIMEText(html_text, "html")) - - def add_inline_image(self, image_file, content_id = None): - - """ - Add an inline image to the body of the mail. - NOTE: This is NOT the same as sending an image as an attachment. - :param image_file: The image data to attach to the mail body. - :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not - specified, I will generate a random string. You may write a custom value here if you know what you are - doing. For most use cases, please ignore this field. - :return: None. - """ - - # Read the image as bytes: - image_bytes = None - if type(image_file) is str: - with open(image_file, "rb") as opened_image_file: - image_bytes = opened_image_file.read() - if type(image_file) is io.BytesIO: - image_file.seek(0) - image_bytes = image_file.getvalue() - - # Declare the part to be attached to the multipart message: - if image_bytes is not None: - - # Create the HTML block if the image pointer is blank: - if content_id is None: - content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) - self.add_html(f""" - - -

- - - """) - - # Then add the image: - image_part = MIMEImage(image_bytes) - image_part.add_header("Content-ID", f"<{content_id}>") - self.message.attach(image_part) - - def add_attachment(self, attachment_file, file_name = None): - - # Declare the part to be attached to the multipart message: - part = MIMEBase("application", "octet-stream") - - # If the attachment is a file stored in the local disk: - if type(attachment_file) is str: - file_name = file_name or os.path.split(attachment_file)[-1] - with open(attachment_file, "rb") as attachment: - part.set_payload(attachment.read()) - - # If the file is held in RAM: - if type(attachment_file) is io.BytesIO(): - attachment_file.seek(0) - part.set_payload(attachment_file.read()) - - # Encode and attach the file: - encoders.encode_base64(part) - part.add_header( - "Content-Disposition", - f"attachment; filename= {file_name}", - ) - self.message.attach(part) - - def get_message(self): - return self.message - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - from utils import json_utils - from utils_v2.mail.async_mail import AsyncMailClient - - async def test(): - - rate_lim = rate_limit_utils.TokenBucket( - rate_limit = 1, - seconds = 60.0, - ) - - mail_client = AsyncMailClient( - email = "sender@gmail.com", - password = "secret_password", - server = "smtp.gmail.com", - rate_limiters = rate_lim - ) - - my_mail = MailMessage( - to_email = "recipient@gmail.com", - subject = "Bhopli is the best!" - ) - my_mail.add_html( - """ - - - - - Sample HTML String - - - -

Hello, Bhopli!

-

Bhopli is the best, most well-behaved cat in the known universe.

- - - """ - ) - my_mail.add_text("This is how you should pet her 👇") - my_mail.add_inline_image(r"/path/to/image/cat_petting.png") - my_mail.add_attachment(r"/path/to/file/sample_label.pdf") - - await mail_client.login() - result = await mail_client.send(my_mail) - print("MAIL RESULT:", json_utils.to_json_string(result)) - await mail_client.logout() - - asyncio.run(test()) diff --git a/utils_v2/network/__init__.py b/utils_v2/network/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/network/ip.py b/utils_v2/network/ip.py deleted file mode 100644 index 134078f..0000000 --- a/utils_v2/network/ip.py +++ /dev/null @@ -1,173 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 12th Jul., 2024 - - OBJECTIVE: - - To provide an easy way to get geolocation information of an IP address. - - REFERENCES: - - 1) https://medium.com/@tubelwj/how-to-retrieve-ip-geolocation-information-in-python-929e15041e3e - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For working with IP Addresses: -import ipaddress - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def ipv4_to_int(ip_string): - - """ - Converts an IP (v4) string to an integer value. - :param ip_string: The IP address (v4) that you want to convert to integer format. - :return: An integer representation of the IP (v4) address. - """ - - ip_numerical = int(ipaddress.IPv4Address(ip_string)) - return ip_numerical - - -# --------------------------------------------------------------------------------------------------------------------- - - -def int_to_ipv4(ip_numerical): - - """ - Interprets the IP (v4) value from the given integer value. - :param ip_numerical: The integer value that represents an IP (v4) address. - :return: - """ - - ip_string = str(ipaddress.IPv4Address(ip_numerical)) - return ip_string - - -# --------------------------------------------------------------------------------------------------------------------- - - -def ipv4_to_bin(ip_string): - - """ - Converts an input IP (v4) address to the binary string that represents the 32 bits. - :param ip_string: The IP (v4) string in a format like "192.168.0.1" - :return: The binary representation (as a string) of the input IP address. - """ - - ip_binary = bin(int(ipaddress.IPv4Address(ip_string)))[2:].zfill(32) - return ip_binary - - -# --------------------------------------------------------------------------------------------------------------------- - - -def bin_to_ipv4(ip_binary): - - """ - Interprets the IP (v4) value from the given binary string. - :param ip_binary: The string of 1s and 0s that represents the IP (v4) address. - :return: The IP (v4) address as a string. - """ - - ip_string = str(ipaddress.IPv4Address(int(ip_binary, 2))) - return ip_string - - -# --------------------------------------------------------------------------------------------------------------------- - - -def get_ipv4_range(ip_string, as_string = True): - - """ - Given a network description in the format "88.95.100.128/25", this function tells you the first and last IP - addresses of that network. Useful for determining if an IP address lies in a network. - :param ip_string: The input network description in the format "88.95.100.128/25" - :param as_string: To select between integer and string formats for the IP range output. - :return: The first and last IP addresses of the input network, and the count. - """ - - # Extract the components of the string: - ip_components = ip_string.split("/") - ip_addr = ipv4_to_int(ip_components[0]) - ip_bits = int(ip_components[1]) - - # Convert the mask number to binary representation: - ip_mask = (1 << ip_bits) - 1 - ip_mask = ip_mask << (32 - ip_bits) - inv_ip_mask = (~ip_mask) & 0xFFFF - - # Figure out the start and end IP addresses: - start_ip = ip_addr & ip_mask - end_ip = ip_addr | inv_ip_mask - count = end_ip - start_ip + 1 - - # If the IPs are needed as strings, we perform the conversion: - if as_string: - start_ip = int_to_ipv4(start_ip) - end_ip = int_to_ipv4(end_ip) - - # Done here: - return start_ip, end_ip, count - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - print(ipv4_to_int("255.255.255.255")) - print(ipv4_to_int("x.x.x.x")) diff --git a/utils_v2/network/ping.py b/utils_v2/network/ping.py deleted file mode 100644 index 39e42c7..0000000 --- a/utils_v2/network/ping.py +++ /dev/null @@ -1,218 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Friday, 28th Jun, 2024 - - OBJECTIVE: - - To provide a way to ping a server and get the traceroute dump. - - REFERENCES: - - 1) https://www.geeksforgeeks.org/traceroute-implementation-on-python/ - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -# --- -import sys -sys.path.append(".") -sys.path.append("..") - -# Utils: -# --- -from utils import json_utils -from utils import time_utils -from utils import regex_utils - -# For networking: -# --- -import socket -from scapy.all import * - -# For running the script from the terminal: -# --- -import argparse - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def get_name_and_addr(destination): - - # If the user provided the IP address: - # --- - if ( - regex_utils.match(destination, regex_utils.REGEX_IPV4) or - regex_utils.match(destination, regex_utils.REGEX_IPV6) - ): - try: destination_name = socket.gethostbyaddr(destination)[0] - except Exception as exception: destination_name = "*" - destination_ip = destination - - # If the provided destination was the domain name: - # --- - else: - destination_name = destination - try: destination_ip = socket.gethostbyname(destination) - except Exception as exception: destination_ip = "*" - - # Done here: - # --- - return destination_name, destination_ip - - -# --------------------------------------------------------------------------------------------------------------------- - - -def tracert( - destination, - max_hops = 30, - timeout = 2.0, - port = 33434 -): - - # Initialize the variables: - # --- - destination_name, destination_ip = get_name_and_addr(destination) - full_trace = [] - ttl = 1 - - # Keep noting hops till the limit is reached: - # --- - while ttl <= max_hops: - - # Create a JSON for this stage: - # --- - this_hop = { - "destAddr": destination_ip, - "destName": destination_name, - "hopNo": ttl - 1, - "isDest": False, - "hopAddr": None, - "hopName": None, - "ts": None - } - - # Create the IP and UDP headers and combine them: - # --- - ip_packet = IP(dst = destination, ttl = ttl) - udp_packet = UDP(dport = port) - trace_packet = ip_packet / udp_packet - - # Send the packet and receive a reply and note down the timestamp: - # --- - reply = sr1(trace_packet, timeout = timeout, verbose = 0) - this_hop["ts"] = time_utils.get_current_utc_datetime(as_string = True) - - # No response: - # --- - if reply is None: this_hop["hopAddr"] = this_hop["hopName"] = "*" - - # If some response was received, we note the values and break out if this was the destination hop: - # --- - else: - this_hop["hopName"], this_hop["hopAddr"] = get_name_and_addr(f"{reply.src}") - if reply.type == 3: - this_hop["isDest"] = True - full_trace.append(this_hop) - break - - # Carry on to the next hop: - # --- - full_trace.append(this_hop) - ttl += 1 - - # Done here: - # --- - return full_trace - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description = "Traceroute Implementation in Python!") - - parser.add_argument( - "dest", - help = "Destination (Name or IP address)." - ) - parser.add_argument( - "-m", - "--max-hops", - type = int, - default = 30, - help = - "Maximum number of hops (default: 30)." - ) - parser.add_argument( - "-t", - "--timeout", - type = float, - default = 2.0, - help = "Timeout for each packet in seconds (default: 2.0)." - ) - parser.add_argument( - "-p", - "--port", - type = int, - default = 33434, - help = "Timeout for each packet in seconds (default: 33434)." - ) - - args = parser.parse_args() - - trace = tracert( - destination = args.dest, - max_hops = args.max_hops, - timeout = args.timeout, - port = args.port - ) - print("TRACE:") - print(json_utils.to_json_string(trace)) diff --git a/utils_v2/pdf/__init__.py b/utils_v2/pdf/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/pdf/pdf_maker.py b/utils_v2/pdf/pdf_maker.py deleted file mode 100644 index fb97cdd..0000000 --- a/utils_v2/pdf/pdf_maker.py +++ /dev/null @@ -1,1642 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 13th Aug., 2024 - - OBJECTIVE: - - To generate custom PDFs. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For conversion to PDF: -from PyPDF2 import PdfMerger, PdfReader, PdfWriter, Transformation - -# For barcode and QR code generation: -import xml.etree.ElementTree as ET -from barcode import Code128 -from barcode.writer import ImageWriter, SVGWriter -import segno -from PIL import Image, ExifTags, ImageFilter - -# System-level activities: -import os -import io -import inspect -import contextlib - -# For calculations: -import math - -# For PDF generation: -import fitz -from reportlab.pdfgen import canvas -from reportlab.lib.units import mm, cm, inch -from reportlab.lib.styles import ParagraphStyle -from reportlab.platypus import Paragraph -from reportlab.pdfbase import pdfmetrics -from reportlab.pdfbase.ttfonts import TTFont -from reportlab.lib.utils import ImageReader -from reportlab.graphics import renderSVG - -# My utils: -from utils_v2.string import json -from utils_v2.system import files -from utils_v2.string import regex - -# For debugging: -from icecream import IceCreamDebugger -import traceback - -# For random string generation: -import random -import string - -# To validate the font file: -from fontTools import ttLib - -# To make http calls: -import requests -import httpx - -# To work with base64 coding: -import base64 - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make async API calls: -max_connections = 5 -limits = httpx.Limits( - max_keepalive_connections = max_connections, - max_connections = max_connections, - keepalive_expiry = 3600 -) -http_client = httpx.AsyncClient( - limits = limits, - follow_redirects = False, - timeout = httpx.Timeout(None) -) - -# Headers for downloading files: -HEADERS_DOWNLOAD = { - "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:129.0) Gecko/20100101 Firefox/129.0", - "Accept": "*/*", - "Accept-Encoding": "gzip, deflate" -} - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class PDFMaker: - - def __init__( - self, - width, - height, - unit = "mm", - dpi = 72, - debug = True, - debug_prefix = "PDF (M) | ", - raise_exception = True, - **kwargs - ): - - """ - IMPORTANT: To keep things intuitively oriented (left-to-right, and top-to-bottom), the internal bottom-up - coordinates have been flipped mathematically. All measurements start from the top-left corner. Moreover, I have - ensured that all numbers follow the same units. So, if you set your unit to "inch", everything will be measured - in the same unit. I hope this creates maximum consistency for you. - A NOTE ON IMAGES: All images will be measured in pixels when operating on them directly (like cropping and - resizing), but will be measured in the 'unit' of the canvas when placing it in the PDF. - A NOTE OF FONT SIZES: Font sizes (and their line spacings) are measured in 'points'. As per the standard, there - are 72 'points' in one inch. While everything else in this utility uses the unit of measurement defined in the - 'unit' param, fonts need to be specified in 'points' to maintain compatibility with HTML rendering engines to - work with well-formatted paragraphs. - WARNING: I STRONGLY RECOMMEND NOT USING DPI AS A SETTING. THE PDF FORMAT ITSELF DOESN'T HAVE ANY PROVISION FOR - PIXEL DENSITY. THIS SETTING HAS BEEN MADE AVAILABLE FOR PLACEBO EFFECT WHEN CLIENTS ASK FOR VERY HIGH RESOLUTION - FILES WITHOUT WANTING TO UNDERSTAND ANY TECHNICALITY. USE WITH CAUTION. - :param width: [int|float] The width of the page (in the specified unit). - :param height: [int|float] The height of the page (in the specified unit). - :param unit: [str] The unit ("mm", "cm", "inch") to use. - :param dpi: [int|float] The pixel density. AVOID USING THIS. - :param debug: [bool] Whether, or not, you want to print debugging messages. Can be changed on the fly. - :param debug_prefix: [str] The prefix to show in debugging messages. - :param raise_exception: If set to True, any failure will immediately stop the PDF building process. If set to - False, the PDF building will continue by ignoring failed steps. Set to True by default for an all-or-nothing - output behaviour. - :param kwargs: Just a placeholder, does nothing for now (20240904). - """ - - # For debugging: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - # Note down the exception-handling choice: - self.__raise_exception = raise_exception - - # We pick the unit from the user's choice: - self.__unit = { - "mm": mm, - "cm": cm, - "inch": inch - }[unit] - - # To know the canvas size multipliers, we need a few parameters: - # 01. The DPI Factor is the multiplier to apply to achieve the target DPI considering a default DPI of 72, - # 02. The Unit Factor is the multiplier to apply to the unit to achieve a high-res result. - self.__dpi_factor = (dpi / 72.0) - self.__unit_factor = self.__unit * self.__dpi_factor - - # Now we create the canvas: - self.__pdf_in_ram = io.BytesIO() - self.__canvas_width = width * self.__unit_factor - self.__canvas_height = height * self.__unit_factor - self.__canvas = canvas.Canvas( - self.__pdf_in_ram, - pagesize = (self.__canvas_width, self.__canvas_height) - ) - - def enable_debug(self): - - """ - To enable the debugging text in the terminal. - :return: None. - """ - - self.__printer.enable() - - def disable_debug(self): - - """ - To disable the debugging text in the terminal. - :return: None. - """ - - self.__printer.disable() - - # ┏┓┓ ┓ ┳┳ •┓• • - # ┗┓┣┓┏┓┏┓┏┓┏┫ ┃┃╋┓┃┓╋┓┏┓┏ - # ┗┛┛┗┗┻┛ ┗ ┗┻ ┗┛┗┗┗┗┗┗┗ ┛ - - @staticmethod - async def download_from_url(url, follow_redirects = False): - - """ - Download a file from a URL. - :param url: The URL to download the file from. - :param follow_redirects: Whether, or not, to follow along with any redirects when downloading the file. - :return: The downloaded file or None. - """ - - file = io.BytesIO((await http_client.get( - url, - headers = HEADERS_DOWNLOAD, - follow_redirects = follow_redirects - )).content) - file.seek(0) - return file - - @staticmethod - def read_to_ram(file_path): - - """ - Reads a file into a BytesIO object in RAM. - NOTE: The method doesn't itself verify if the file is a PDF. - :param file_path: The path to the file on disk. - :return: The file in a BytesIO object. - """ - - with open(file_path, "rb") as file: - file_data = file.read() - file_in_ram = io.BytesIO(file_data) - file_in_ram.seek(0) - return file_in_ram - - def __parse_color(self, input_color): - - """ - The reportlab 'reportlab' needs the colour to be specified in an array of RGS values where each value is a - number in the range 0-1. - :param input_color: Either a hex string or an array of numbers. - :return: An array that can be used by 'reportlab'. - """ - - # Start by assuming failure: - processed_color = None - - # If the input color is a string, we convert from hex to an array of hex codes: - if isinstance(input_color, str): - input_color = input_color.lstrip("#") - color_tuple = ( - input_color[0:2].upper(), - input_color[2:4].upper(), - input_color[4:6].upper() - ) - input_color = color_tuple - - # If the input color is a list/tuple of hex-strings: - if isinstance(input_color, (list, tuple)): - try: - if all(isinstance(item, str) for item in input_color): - processed_color = [int(item, 16) / 255.0 for item in input_color] - elif all(isinstance(item, (float, int)) for item in input_color): - processed_color = [max(min(item, 255.0), 0.0) / 255.0 for item in input_color] - except: - if self.__raise_exception: raise - - # Done here: - return processed_color - - def __parse_x(self, *args): - - """ - To adjust the horizontal coordinates for the canvas. - :param args: As many horizontal coordinates as you would like to adjust. - :return: The adjusted coordinates. - """ - - if len(args) == 0: return None - parsed_x = [x * self.__unit_factor for x in args] - parsed_x = [x if x >= 0 else x + self.__canvas_width for x in parsed_x] - return tuple(parsed_x) if len(parsed_x) > 1 else parsed_x[0] - - def __parse_y(self, *args, flip = False): - - """ - To adjust the vertical coordinates for the canvas. - :param args: As many vertical coordinates as you would like to adjust. - :param flip: To convert from bottom-up system to top-down system. - :return: The adjusted coordinates. - """ - - if len(args) == 0: return None - parsed_y = [y * self.__unit_factor for y in args] - parsed_y = [y if y >= 0 else y + self.__canvas_height for y in parsed_y] - if flip: parsed_y = [self.__canvas_height - y for y in parsed_y] - return tuple(parsed_y) if len(parsed_y) > 1 else parsed_y[0] - - def __parse_font_size(self, *args): - - """ - Adjust the font size. - :param args: The font sizes to adjust - :return: The adjusted font-size. - """ - - if len(args) == 0: return None - parsed_font_sizes = args - return parsed_font_sizes if len(parsed_font_sizes) > 1 else parsed_font_sizes[0] - - @staticmethod - def _parse_coordinates_for_rotation( - original_x, - original_y, - angle - ): - - """ - Adjusts the x and y coordinates to compensate for the effect of a rotation operation. - :param original_x: The x coordinate before applying rotation. - :param original_y: The y coordinate before applying rotation. - :param angle: The angle (in degrees) by which the rotation will be applied. - :return: The adjusted x and y coordinates that compensate for the effect of the canvas's rotation. - """ - - # Get the angle in radians: - angle_radians = math.radians(angle) - angle_cos = math.cos(angle_radians) - angle_sin = math.sin(angle_radians) - - # Compute the inverse rotation: - adjusted_x = original_x * angle_cos + original_y * angle_sin - adjusted_y = -original_x * angle_sin + original_y * angle_cos - - # Done here: - return adjusted_x, adjusted_y - - # ┳ ┏┓ • - # ┃┏┓┏╋┏┓┏┓┏┏┓ ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏ - # ┻┛┗┛┗┗┻┛┗┗┗ ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛ - # ┛ - - @property - def canvas(self): - - """ - Use this to fetch just the canvas with all the inputs applied to it. - :return: The canvas (page) that is being used currently in the instance. - """ - - return self.__canvas - - @property - def unit_factor(self): - - """ - The scaling factor after taking into consideration the physical unit of measurement being use (e.g.: "mm"). - :return: The unit-factor. - """ - - return self.__unit_factor - - @property - def dpi_factor(self): - - """ - The scaling factor derived from comparing the default DPI of 72 and a custom DPI specified when creating the - instance. Since the PDF format doesn't have a native DPI configuration, this scaling factor will help in - adjusting the sizes of whatever you wish to add to the canvas. - :return: The DPI scaling factor. - """ - - return self.__dpi_factor - - @property - def shape(self): - - """ - an array of the dimensions (width, height) of the canvas. - :return: The width and height of the canvas. - """ - - return self.__canvas_width, self.__canvas_height - - # ┏┓ ┏┓ ┏┓•┓ - # ┃ ┏┓┏┓┓┏┏┓┏ ┣╋ ┣ ┓┃┏┓ - # ┗┛┗┻┛┗┗┛┗┻┛ ┗┻ ┻ ┗┗┗ - - def save(self, path: str = None): - - """ - Save the Canvas as a PDF either to RAM or a file on disk. - :param path: The path you want to save the PDF to on disk. If not specified, the PDF will be saved to RAM and - returned as a BytesIO object. - :return: A BytesIO object, or True/False if a path is supplied and the file gets saved. - """ - - # Save the changes to the file in RAM. - self.__canvas.save() - self.__pdf_in_ram.seek(0) - - # If the user has not specified a path, return the buffer: - if path is None: return self.__pdf_in_ram - - # In case a path was specified, - # We try to save the file in the specified path: - try: - with open(path, "wb") as file: file.write(self.__pdf_in_ram.getvalue()) - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - def next_page(self): - - """ - To start working on the next page. - The library that is being used doesn't support moving back and forth between pages in any random order, it, - instead, needs to go in a sequential fashion from one page to the next. - :return: None. - """ - - self.__canvas.showPage() - - # ┏┓ - # ┣ ┏┓┏┓╋┏ - # ┻ ┗┛┛┗┗┛ - - @staticmethod - async def register_font_from_url(name, url, follow_redirects = False): - - """ - Download a font from a URL and then register it for use. - :param name: The name you would later refer to the font by. - :param url: The URL to download the font from. - :param follow_redirects: Whether, or not, to follow along with any redirects when downloading the file. - :return: True if registered, else False. - """ - - try: return PDFMaker.register_font( - name, - await PDFMaker.download_from_url(url, follow_redirects = follow_redirects) - ) - except Exception as exception: - return False - - @staticmethod - def register_fonts_from_directory(directory): - - """ - Register all the fonts from a directory to be used later in the PDF. - :param directory: The directory that has the TTF files to register. - :return: A dict describing the fonts that were registered. - """ - - # List out all the files in the folder: - all_files = files.list_files(directory, full_path = True) - - # Keep only TTF files and make a name-to-path map: - font_map = { - os.path.split(file)[-1].split(".")[0].lower(): file - for file in all_files if file.lower().rstrip().endswith(".ttf") - } - - # Register the custom fonts: - registered_fonts = [] - for font_name, font_path in font_map.items(): - if PDFMaker.register_font(font_name, font_path): - registered_fonts.append({"name": font_name, "file": font_path}) - - # Done here: - return registered_fonts - - @staticmethod - def register_font(name, file): - - """ - Register one TTF font from either a BytesIO object or a path on the disk. - :param name: The name of the font as you would like to use when invoking it. - :param file: The font file (TTF format). - :return: True if registered, else False. - """ - - try: - - if isinstance(file, io.BytesIO): file.seek(0) - pdfmetrics.registerFont(TTFont(name, file)) - return True - - except Exception as exception: - return False - - @staticmethod - def get_font_height(font, size, count_ascent = True, count_descent = True): - - """ - Compute the height of the font for the given size. - :param font: [str] The name of the font (as registered). - :param size: [int|float] The size of the font (in points). - :param count_ascent: [bool] Whether, or not, you would like to consider the ascent of the font. - :param count_descent: [bool] Whether, or not, you would like to consider the descent of the font. - :return: The height of the font in points. - """ - - # Get font's information and compute the height: - ascent, descent = pdfmetrics.getAscentDescent(font, size) - if not count_ascent: ascent = 0.0 - if not count_descent: descent = 0.0 - height = ascent - descent - return height - - @staticmethod - def font_is_registered(font): - - """ - Checks if a particular font is available for use. - :param font: [str] The name of the font to check. - :return: True if registered, else False - """ - - return font in pdfmetrics.getRegisteredFontNames() - - @staticmethod - def get_first_registered_font(): - - """ - Returns the name of the first registered font. Useful for cases like those when your font fails to load and you - need a default to fall back on. - :return: The name of the first font that has been registered. - """ - - return pdfmetrics.getRegisteredFontNames()[0] - - @staticmethod - def list_registered_fonts(): - - """ - Returns the list of names of all the registered fonts. - :return: The list of names of the fonts that have been registered. - """ - - return pdfmetrics.getRegisteredFontNames() - - @staticmethod - def is_valid_ttf(file): - - """ - Checks if a font file is valid and usable or not. - :param file: [str|io.BytesIO] The file to check. - :return: True if valid, else False. - """ - - if isinstance(file, io.BytesIO): file.seek(0) - try: font = ttLib.TTFont(file)["head"].fontRevision - except Exception as exception: return False - return True - - # ┳ - # ┃┏┳┓┏┓┏┓┏┓┏ - # ┻┛┗┗┗┻┗┫┗ ┛ - # ┛ - - @staticmethod - def make_barcode_image( - data, - bar_width = 2, - bar_height = 125, - border = 10, - bar_color = "#000000", - background_color = "#FFFFFF", - dpi = 300, - format = "png", - as_pil = True - ): - - """ - Makes a barcode in Code128 format. - :param data: [str] The data to encode in the barcode. - :param bar_width: [int] The base bar-module width in pixels. - :param bar_height: [int] The bar-module height in pixels. - :param border: [int] The size of the border around all sides of the barcode in pixels. - :param bar_color: [str] The colour of the bars as a hex string. - :param background_color: [str] The colour of the background as a hex string. - :param dpi: [int] The pixel density to note in the image. Leave it to default for most use cases. - :param format: [str] The file format to save the image in. - :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). - :return: Either a PIL object or a PNG file in RAM. - """ - - # Create the barcode in RAM: - barcode_in_ram = io.BytesIO() - barcode_options = { - "module_width": (bar_width / dpi) * 25.4, - "module_height": (bar_height / dpi) * 25.4, - "quiet_zone": 0, - "foreground": bar_color, - "background": background_color, - "format": format, - "dpi": dpi - } - Code128(str(data), writer = ImageWriter()).write(barcode_in_ram, options = barcode_options) - barcode_in_ram.seek(0) - - # Remove the rasterized text at the bottom, and create a border of the desired size: - barcode_image = Image.open(barcode_in_ram, formats = [format]) - width, height = barcode_image.size - barcode_image = barcode_image.crop(( - 0, - int(dpi * 0.0393701), - width, - int(dpi * 0.0393701) + bar_height - )) - width, height = barcode_image.size - background_image = Image.new( - mode = "RGB", - size = (width + border + border, height + border + border), - color = background_color - ) - background_image.paste(barcode_image, (border, border)) - barcode_image = background_image - - # If the user has asked for a PIL object: - if as_pil: return barcode_image - - # Else, ave the PNG in ram: - barcode_in_ram = io.BytesIO() - barcode_image.save(barcode_in_ram, format = "png") - barcode_in_ram.seek(0) - - # Done: - return barcode_in_ram - - @staticmethod - def make_qr_image( - data, - scale = 5, - border = 2, - foreground_color = "#000000", - background_color = "#FFFFFF", - border_color = "#FFFFFF", - background_art = None, - format = "png", - as_pil = True - ): - - """ - Create a QR code. - :param data: [str] The data that you want to encode in the QR code. - :param scale: [int] The size (in pixels) for each tiny block in the QR code. - :param border: [int] The size (in multiples of scale) of the quiet zone. - :param foreground_color: [str] The color of the foreground, a.k.a. the dark color. - :param background_color: [str] The color of the background, a.k.a. the light color. - :param border_color: [str] The color of the quiet zone. - :param background_art: [io.BytesIO|PIL.Image] Any image that you would like to feed into the background instead - of a plain QR code. This can be an animated GIF image as well. - :param format: [str] The type of output file that you want. - :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). - :return: Either a PIL object or a PNG file in RAM. - """ - - qr_code = segno.make_qr(str(data)) - params = { - "scale": scale, - "border": border, - "dark": foreground_color, - "light": background_color, - "quiet_zone": border_color, - "kind": format - } - - qr_image = io.BytesIO() - - if background_art is not None: - if isinstance(background_art, Image.Image): - buffer = io.BytesIO() - background_art.save(buffer, format = format) - buffer.seek(0) - background_art = buffer - params["background"] = background_art - params["target"] = qr_image - qr_code.to_artistic(**params) - - else: - params["out"] = qr_image - qr_code.save(**params) - - qr_image.seek(0) - if as_pil: qr_image = Image.open(qr_image, formats = [format]) - return qr_image - - async def image_from_url(self, url, follow_redirects = False, as_pil = True, format = "png"): - - """ - Asynchronously downloads an image and returns it as a PIL object. - Use this instead of just passing the URL to 'draw_image' for better efficiency. - :param url: [str] The URL to download the image from. - :param follow_redirects: [bool] Whether, or not, to follow redirect URLs when downloading the file. - :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). - :param format: [str] The type of output file that you want. Not applicable for PIL objects. - :return: The image as a PIL object or as a file in RAM. - """ - - try: - - pil_image = Image.open(await PDFMaker.download_from_url(url, follow_redirects = follow_redirects)) - if as_pil: return pil_image - image_in_memory = io.BytesIO() - pil_image.save(image_in_memory, format = format) - image_in_memory.seek(0) - return image_in_memory - - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - - def image_from_base64(self, data, as_pil = True): - - """ - Converts a base64 string to an image. - :param data: [str] The base-64 representation of the image data. - :param as_pil: [bool] If True, a PIL object will be returned, else a PNG file will be returned in RAM (BytesIO). - :return: The image as a PIL object or as a file in RAM. - """ - - try: - - image = io.BytesIO(base64.b64decode(data)) - image.seek(0) - if not as_pil: return image - image = Image.open(image) - return image - - except Exception as exception: - if self.__raise_exception: raise - - @staticmethod - def crop_image_to_aspect_ratio(image, target_aspect_ratio, anchor = "c"): - - """ - Shaves off pixels from an image to achieve the target aspect ratio. - :param image: [PIL.Image] The image to work on, held in a PIL object. - :param target_aspect_ratio: [int|float] The aspect ratio you want to achieve. - :param anchor: [str] Which part of the image you want to retain. Imagine it like a compass - "nw", "n", "ne", - "e", "se", "s", "sw", "w", and "c". - :return: The adjusted image. - """ - - # Get the existing figures: - width, height = image.size - original_aspect_ratio = width / height - - # Compute the new dimensions: - if original_aspect_ratio > target_aspect_ratio: - scaling_factor = target_aspect_ratio / original_aspect_ratio - new_width, new_height = int(width * scaling_factor), height - else: - scaling_factor = original_aspect_ratio / target_aspect_ratio - new_width, new_height = width, int(height * scaling_factor) - - # Now we figure out the start and end points of cropping: - match anchor.strip().lower(): - case "nw": - start_x = 0 - start_y = 0 - case "n": - start_x = int(abs(new_width - width) / 2.0) - start_y = 0 - case "ne": - start_x = int(abs(new_width - width)) - start_y = 0 - case "e": - start_x = int(abs(new_width - width)) - start_y = int(abs(new_height - height) / 2.0) - case "se": - start_x = int(abs(new_width - width)) - start_y = int(abs(new_height - height)) - case "s": - start_x = int(abs(new_width - width) / 2.0) - start_y = int(abs(new_height - height)) - case "sw": - start_x = 0 - start_y = int(abs(new_height - height)) - case "w": - start_x = 0 - start_y = int(abs(new_height - height) / 2.0) - case _: - start_x = int(abs(new_width - width) / 2.0) - start_y = int(abs(new_height - height) / 2.0) - - end_x = start_x + new_width - end_y = start_y + new_height - - # And we finally crop and return the image: - return image.crop((start_x, start_y, end_x, end_y)) - - @staticmethod - def crop_image(image, start_x, start_y, end_x, end_y): - - """ - Crops an image. - :param image: [PIL.Image] The image to work on, held in a PIL object. - :param start_x: [int|float] The left coordinate (in the same unit as the dimension of the canvas). - :param start_y: [int|float] The top coordinate (in the same unit as the dimension of the canvas). - :param end_x: [int|float] The right coordinate (in the same unit as the dimension of the canvas). - :param end_y: [int|float] The bottom coordinate (in the same unit as the dimension of the canvas). - :return: The cropped image. - """ - - return image.crop((start_x, start_y, end_x, end_y)) - - @staticmethod - def resize_image_to_aspect_ratio(image, target_aspect_ratio): - - """ - Resizes the image (by stretching and squishing) till it fits the target aspect ratio. - :param image: [PIL.Image] The image to adjust. - :param target_aspect_ratio: [int|float] The aspect ratio that you want to achieve. - :return: The adjusted image. - """ - - # Get the existing figures: - width, height = image.size - original_aspect_ratio = width / height - - # Compute the new dimensions: - if original_aspect_ratio > target_aspect_ratio: - new_width, new_height = width, int(width / target_aspect_ratio) - else: - new_width, new_height = int(height * target_aspect_ratio), height - - # Resize and return the image: - return image.resize((new_width, new_height)) - - @staticmethod - def resize_image(image, width, height): - - """ - Resizes an image to the specified width and height. - :param image: [PIL.Image] The image as a PIL object. - :param width: [int|float] The new width. - :param height: [int|float] The new height. - :return: The resized image. - """ - - return image.resize((width, height)) - - @staticmethod - def fit_image_to_aspect_ratio( - image: Image, - target_aspect_ratio: float, - blur_strength = 0.01, - alpha: float = 0.8 - ): - - """ - Fits an image to any aspect ratio by adding a blurred background to it. - :param image: [PIL.Image] The image to adjust. - :param target_aspect_ratio: [int|float] The aspect ratio that you want to achieve. - :param blur_strength: How intense the blur must be. - :param alpha: The opacity of the blurred background. - :return: The adjusted image. - """ - - # Get the existing figures: - width, height = image.size - original_aspect_ratio = width / height - - # Compute the new dimensions: - if original_aspect_ratio > target_aspect_ratio: - new_width, new_height = width, int(width / target_aspect_ratio) - scaling_factor = new_height / height - paste_x = 0 - paste_y = int((new_height - height) / 2) - else: - new_width, new_height = int(height * target_aspect_ratio), height - scaling_factor = new_width / width - paste_x = int((new_width - width) / 2) - paste_y = 0 - - # Prepare the background image: - # bg_image = image.copy().resize((new_width, new_height)) - bg_image = image.copy().resize((int(width * scaling_factor), int(height * scaling_factor))) - bg_image = bg_image.filter(ImageFilter.GaussianBlur(radius = int((new_width + new_height) / 2) * blur_strength)) - bg_image = PDFMaker.crop_image_to_aspect_ratio(bg_image, target_aspect_ratio) - - # darken the background a little: - bg_image = Image.blend( - bg_image, - Image.new("RGB", bg_image.size, (0, 0, 0)), - alpha = 1.0 - alpha - ) - - # Paste the original image on top of the background: - bg_image.paste(image, (paste_x, paste_y)) - - # Resize and return the image: - return bg_image - - @staticmethod - def adjust_image_orientation_from_exif(image): - - """ - Rotates the image as per the instructions in the EXIF tags. - :param image: The PIL object that holds the image data. - :return: The adjusted image. - """ - - # Guard clause to return if the attribute is not even found: - if not hasattr(image, "getexif"): return image - - # Else we start extracting EXIF data: - exif = image.getexif() - if exif is None: return image - - # We look for the tag that indicates orientation: - # ORIENTATION CODES: - # 1: Normal (Landscape) - # 2: Flipped horizontally - # 3: Upside down (Reverse Landscape) - # 4: Flipped vertically - # 5: Rotated 90° clockwise and flipped horizontally - # 6: Rotated 90° clockwise (Portrait) - # 7: Rotated 90° counter-clockwise and flipped horizontally - # 8: Rotated 90° counter-clockwise (Reverse Portrait) - orientation = exif.get(274, 0) - if orientation == 1: pass - elif orientation == 2: image = image.transpose(Image.FLIP_LEFT_RIGHT) - elif orientation == 3: image = image.rotate(180, expand = True) - elif orientation == 4: image = image.rotate(180, expand = True).transpose(Image.FLIP_LEFT_RIGHT) - elif orientation == 5: image = image.rotate(270, expand = True).transpose(Image.FLIP_LEFT_RIGHT) - elif orientation == 6: image = image.rotate(270, expand = True) - elif orientation == 7: image = image.rotate(90, expand = True).transpose(Image.FLIP_LEFT_RIGHT) - elif orientation == 8: image = image.rotate(90, expand = True) - - # Done here: - return image - - # ┏┓┳┓┏ - # ┃┃┃┃╋ - # ┣┛┻┛┛ - - def draw_line( - self, - start_x, - start_y, - end_x, - end_y, - thickness = 1.0, - color = (0, 0, 0), - alpha = 1.0 - ): - - """ - Draw a line in the PDF page. - :param start_x: [int|float] The left coordinate. - :param start_y: [int|float] The top coordinate. - :param end_x: [int|float] The right coordinate. - :param end_y: [int|float] The bottom coordinate. - :param thickness: [int|float] The thickness of the line (in the same unit as the canvas's dimensions). - :param color: [str] The RGB color in an array. - :param alpha: [int|float] The opacity of the line. 1 is 100% and 0 is 0% - :return: True or False based on the success of the operation. - """ - - # Handle color: - color = self.__parse_color(color) - - # Adjust the dimensions: - thickness *= self.__unit_factor - start_x, end_x = self.__parse_x(start_x, end_x) - start_y, end_y = self.__parse_y(start_y, end_y, flip = True) - - try: - - self.__canvas.setLineWidth(thickness) - self.__canvas.setStrokeColorRGB(color[0], color[1], color[2], alpha) - self.__canvas.line( - start_x, - start_y, - end_x, - end_y - ) - return True - - # In case something goes wrong: - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - def draw_circle( - self, - x, - y, - radius, - fill_color = (255, 255, 255), - fill_alpha = 0.0, - stroke_color = (0, 0, 0), - stroke_alpha = 1.0, - thickness = 1.0 - ): - - """ - Draw a circle in the PDF page. - :param x: [int|float] The horizontal coordinate of the center of the circle. - :param y: [int|float] The vertical coordinate of the center of the circle. - :param radius: [int|float] The radius of the circle. - :param fill_color: [str] The color-array (0-255) or hex string to define the inner fill color of the rectangle. - :param fill_alpha: [int|float] The opacity of the inner fill color. - :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. - :param stroke_alpha: [int|float] The opacity of the outline color. - :param thickness: [int|float] The thickness of the line (in the same unit as the canvas's dimensions). - :return: True or False based on the success of the operation. - """ - - try: - - # Handle color: - fill_color = self.__parse_color(fill_color) - stroke_color = self.__parse_color(stroke_color) - - # Adjust the dimensions: - x = self.__parse_x(x) - y = self.__parse_y(y, flip = True) - thickness *= self.__unit_factor - radius *= self.__unit_factor - - # Draw the circle here: - self.__canvas.setLineWidth(thickness) - self.__canvas.setFillColorRGB(fill_color[0], fill_color[1], fill_color[2]) - self.__canvas.setFillAlpha(fill_alpha) - self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) - self.__canvas.setStrokeAlpha(stroke_alpha) - self.__canvas.circle(x, y, radius, stroke = 1, fill = 1) - - # In case something goes wrong: - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - def draw_rect( - self, - start_x, - start_y, - end_x, - end_y, - radius = 0.0, - fill_color = (255, 255, 255), - fill_alpha = 0.0, - stroke_color = (0, 0, 0), - stroke_alpha = 1.0, - thickness = 1.0, - ): - - """ - Draws a rectangle on the page. - :param start_x: [int|float] The left coordinate. - :param start_y: [int|float] The top coordinate. - :param end_x: [int|float] The right coordinate. - :param end_y: [int|float] The bottom coordinate. - :param radius: [int|float] The corner radius. - :param fill_color: [str] The color-array (0-255) or hex string to define the inner fill color of the rectangle. - :param fill_alpha: [int|float] The opacity of the inner fill color. - :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. - :param stroke_alpha: [int|float] The opacity of the outline color. - :param thickness: [int|float] The line thickness of the outline (in the same unit as the canvas's dimensions). - :return: True or False based on the success of the operation. - """ - - try: - - # Handle color: - fill_color = self.__parse_color(fill_color) - stroke_color = self.__parse_color(stroke_color) - - # Adjust the dimensions: - thickness *= self.__unit_factor - start_x, end_x = self.__parse_x(start_x, end_x) - start_y, end_y = self.__parse_y(start_y, end_y, flip = False) - - # Compute the width and height desired by the user: - width = end_x - start_x - height = end_y - start_y - - # Draw the rectangle here: - self.__canvas.setLineWidth(thickness) - self.__canvas.setFillColorRGB(fill_color[0], fill_color[1], fill_color[2]) - self.__canvas.setFillAlpha(fill_alpha) - self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) - self.__canvas.setStrokeAlpha(stroke_alpha) - self.__canvas.roundRect( - start_x, - self.__canvas_height - start_y - height, - width, - height, - radius * self.__unit_factor, - stroke = 1, - fill = 1 - ) - - return True - - # In case something goes wrong: - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - def make_paragraph_style( - self, - font_name = "Helvetica", - font_size = 12, - font_color = (0, 0, 0), - line_spacing = 12, - border_size = 0.0, - border_color = (255, 255, 255), - align = "left", - style_name = "CustomStyle" - ): - - """ - Creates a text style for using in a paragraph. - :param font_name: [str] The name of the font. Has to be registered. - :param font_size: [int|float] The size that you want the font to be printed in. - :param font_color: [str] The color-array (0-255) or hex string that you want the font to be printed in. - :param line_spacing: [int|float] The line spacing to apply. - :param border_size: [int|float] The thickness of the border. - :param border_color: [str] The color-array (0-255) or hex string of the color you wan the border to be in. - :param align: [str] Text alignment ("left", "center", or "right"). - :param style_name: [str] A name that you would like to give this style. - :return: The paragraph style. - """ - - # Make adjustments: - font_color = self.__parse_color(font_color) - font_size = self.__parse_font_size(font_size) - border_color = self.__parse_color(border_color) - border_size *= self.__unit_factor - - return ParagraphStyle( - name = style_name, - fontName = font_name, - fontSize = font_size, - textColor = (font_color[0], font_color[1], font_color[2]), - alignment = { - "left": 0, - "center": 1, - "right": 2 - }[align], - leading = line_spacing * font_size, - spaceBefore = 0, - spaceAfter = 0, - borderWidth = border_size, - borderColor = border_color - ) - - def write_paragraph( - self, - text, - start_x, - start_y, - end_x, - end_y, - alpha = 1.0, - style = None - ): - - """ - To write text into a bounding box in the PDF. - This can be either plaintext or an HTML string. - :param text: [str] The text to type in the bounding box. - :param start_x: [int|float] The left coordinate (in the same unit as used while setting up the canvas). - :param start_y: [int|float] The top coordinate (in the same unit as used while setting up the canvas). - :param end_x: [int|float] The right coordinate (in the same unit as used while setting up the canvas). - :param end_y: [int|float] The bottom coordinate (in the same unit as used while setting up the canvas). - :param alpha: [int|float] The opacity of the text. 1 is 100% and 0 is 0% - :param style: The paragraph-style generated by using "make_paragraph_style" method. Avoid using this if you are - passing in an HTML string. - :return: True or False based on the success of the operation. You will get a False even if the text flows out of - the bounding box (which will cause it to not render). - """ - - try: - - # Adjust the dimensions: - start_x, end_x = self.__parse_x(start_x, end_x) - start_y, end_y = self.__parse_y(start_y, end_y, flip = False) - - # Compute the width and height desired by the user: - width = end_x - start_x - height = end_y - start_y - - # Make the paragraph: - paragraph = Paragraph(text, style = style) - - # Wrap the paragraph to compute the needed dimensions: - needed_width, needed_height = paragraph.wrap(width, height) - - # Adjust the y coordinates (for top-down system): - start_y = self.__canvas_height - start_y - needed_height - - # Write the text to the canvas if the text fits. - # Return true in case of success, else false: - if width >= needed_width and height >= needed_height: - self.__canvas.setFillAlpha(alpha) - paragraph.drawOn(self.__canvas, start_x, start_y, _sW = 0) - return True - else: return False - - # In case something goes wrong: - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - def write_string( - self, - text, - x, - y, - font = "Helvetica", - size = 12, - color = (0, 0, 0), - alpha = 1.0, - align = "left", - angle = 0 - ): - - """ - Writes a string onto the canvas. - :param text: [str] The text to be written. - :param x: [int|float] The horizontal coordinate (of the baseline). - :param y: [int|float] The vertical coordinate (of the baseline). - :param font: [str] The name of the font to be used. - :param size: [int|float] The size of the font to be used (in points). - :param color: [str] The color-array (0-255) or hex string that you want the font to be printed in. - :param alpha: [int|float] The opacity of the text. 1 is 100% and 0 is 0% - :param align: [str] The alignment to use ("left", "center", "right"). - :param angle: The angle at which you want the text to be printed in. - :return: True or False based on the success of the operation. - """ - - try: - - # Handle the color - color = self.__parse_color(color) - - # Adjust the dimensions: - x = self.__parse_x(x) - y = self.__parse_y(y, flip = True) - size = self.__parse_font_size(size) - - # Adjust the canvas's values: - self.__canvas.setFont(font, size) - self.__canvas.setFillColorRGB(color[0], color[1], color[2]) - self.__canvas.setFillAlpha(alpha) - - # Handle angular adjustments: - self.__canvas.rotate(angle) - x, y = self._parse_coordinates_for_rotation( - original_x = x, - original_y = y, - angle = angle - ) - - # Adjust the font coordinates as per the angle: - text_width = self.__canvas.stringWidth(text, fontName = font, fontSize = size) - if align == "center": x -= text_width / 2 - elif align == "right": x -= text_width - - # Draw the string: - self.__canvas.drawString(x, y, text) - - # Reset angular adjustments: - self.__canvas.rotate(-angle) - - # Return with success if nothing broke till here: - return True - - # In case something goes wrong: - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - def draw_image( - self, - image, - start_x, - start_y, - end_x, - end_y, - crop = False, - crop_anchor = "c", - alpha = 1.0, - stroke_color = (0, 0, 0), - stroke_alpha = 0.0, - thickness = 1.0, - identifier: str = None, - format = "jpeg" - ): - - """ - Draw an image onto the PDF page in the specified bounding box. The image can be drawn in either "fit" mode where - the image is scaled down to fit in the bounding box, or in "fill" mode where the image is cropped to fill up the - space in the bounding box by matching their aspec ratios. - :param image: The image data as a PIL object or a valid URL or a file path. NOTE: If given as a URL, the image - will be downloaded synchronously. - :param start_x: [int|float] The left coordinate. - :param start_y: [int|float] The top coordinate. - :param end_x: [int|float] The right coordinate. - :param end_y: [int|float] The bottom coordinate. - :param crop: [bool] Set to True for "fill" mode, False for "fit" mode. - :param crop_anchor: [str] If cropping is to be performed, what should the anchor be for cropping. Refer to the - description in 'crop_image_to_aspect_ratio'. - :param alpha: [int|float] The opacity of the image. 1 is 100% and 0 is 0% - :param stroke_color: [str] The color-array (0-255) or hex string to define the outline color of the rectangle. - :param stroke_alpha: [int|float] The opacity of the outline color. - :param thickness: [int|float] The line thickness of the outline (in the same unit as the canvas's dimensions). - :param identifier: [str] A unique name for the image. - :param format: The file format of the image (when it will be embedded in the PDF). - :return: True or False based on the success of the operation. - """ - - try: - - # Cleaning: - format = format.strip().lower().split(".")[-1] - - # Handle color: - stroke_color = self.__parse_color(stroke_color) - - # The identifier can allow us to use the same image multiple times in a PDF file. - # If the identifier is not provided, we give the file a random identifier. - if identifier is None: identifier = "".join([random.choice(string.ascii_lowercase) for _ in range(8)]) - - # Adjust the dimensions: - start_x, end_x = self.__parse_x(start_x, end_x) - start_y, end_y = self.__parse_y(start_y, end_y, flip = True) - - # Compute the width and height desired by the user: - width = end_x - start_x - height = end_y - start_y - - # In case the image is passed as a URL, we try to download it and open it as a PIL object: - if isinstance(image, str): - if image.startswith("https://") or image.startswith("http://"): - image = Image.open(io.BytesIO(requests.get(image).content)) - else: image = Image.open(image) - - # The image can have orientation data in its EXIF tags. - # If so, we adjust for it so that reportlab can work with it properly: - image = self.adjust_image_orientation_from_exif(image) - - # If the image needs to be cropped, we compute the aspect ratio and get the image adjusted: - if crop: image = self.crop_image_to_aspect_ratio( - image, - abs(end_x - start_x) / abs(end_y - start_y), - anchor = crop_anchor - ) - - # Else we add a blurred background to the image: - else: image = self.fit_image_to_aspect_ratio( - image, - abs(end_x - start_x) / abs(end_y - start_y), - blur_strength = 0.1 - ) - - # We set the image opacity and outline settings here: - self.__canvas.setFillAlpha(alpha) - self.__canvas.setLineWidth(thickness * self.__unit_factor) - self.__canvas.setStrokeColorRGB(stroke_color[0], stroke_color[1], stroke_color[2]) - self.__canvas.setStrokeAlpha(stroke_alpha) - - # Draw the image onto the canvas: - if ( - image.mode == "P" or - format in ["jpeg", "jpg"] - ): - image = image.convert("RGB") - image_buffer = io.BytesIO() - image.save(image_buffer, format = format) - image_buffer.seek(0) - self.__canvas.drawImage( - ImageReader(image_buffer, ident = identifier), - start_x, - start_y, - width = width, - height = height, - preserveAspectRatio = True, - mask = "auto", - anchor = "c", - showBoundary = True - ) - - return True - - # In case something goes wrong: - except Exception as exception: - self.__printer(exception) - if self.__raise_exception: raise - return False - - # def draw_svg( - # self, - # svg, - # start_x, - # start_y, - # end_x, - # end_y - # ): - # - # renderSVG.draw(svg, sta) - - # ┏┓┳┓┏ ┏┳┓ ┓ - # ┃┃┃┃╋ ┃ ┏┓┏┓┃┏ - # ┣┛┻┛┛ ┻ ┗┛┗┛┗┛ - - @staticmethod - def make_thumbnails( - pdf_file, - pages = None, - scale = 1.0, - as_pil = True, - format = "png" - ): - - """ - Makes thumbnails of pages of the PDF and returns them - :param pdf_file: The PDF file whose pages must be converted to thumbnails. - :param pages: The list of page nos. to make thumbnails of. If not specified, all pages will be converted. - :param scale: The scaling multiplier, applied to both axes, to change the output size. - :param as_pil: If True, the output will be an array of PIL objects, else the output will be an array of - file-like BytesIO objects. - :param format: The file format of the output image. - :return: The array of thumbnails, either as PIL objects, or as BytesIO objects. - """ - - # Open the PDF, and ensure that we have the pages numbers: - if isinstance(pdf_file, io.BytesIO): - pdf_file.seek(0) - pdf_document = fitz.open(stream = pdf_file) - pdf_file.seek(0) - else: pdf_document = fitz.open(pdf_file) - if pages is None: pages = list(range(len(pdf_document))) - - # Iterate over the pages and make the thumbnails: - thumbnails = [] - for page_no in pages: - img = pdf_document.load_page(page_no).get_pixmap(matrix = fitz.Matrix(scale, scale)) - img = io.BytesIO(img.tobytes(format)) - if as_pil: img = Image.open(img, formats = [format]) - thumbnails.append(img) - - # Done here: - return thumbnails - - @staticmethod - def join_from_ram(pdf_files, raise_exception = True): - - """ - Joins a set of PDF files held in RAM (as BytesIO objects). - :param pdf_files: The list of PDF files (as BytesIO objects). - :param raise_exception: Set to True for an all-or-nothing process. - :return: The joined PDF file (as a BytesIo object). - """ - - try: - - # Create a file in RAM that will hold the merged PDF: - joined_pdf_in_ram = io.BytesIO() - - # Merge the files into one: - merger = PdfMerger() - for pdf_file in pdf_files: merger.append(pdf_file) - merger.write(joined_pdf_in_ram) - merger.close() - - # Done here: - joined_pdf_in_ram.seek(0) - return joined_pdf_in_ram - - # In case something goes wrong: - except Exception as exception: - if raise_exception: raise - return None - - @staticmethod - def grid_from_ram( - pdf_files, - input_width, - input_height, - output_width, - output_height, - cutting_width, - cutting_height, - unit = "mm", - dpi = 72 - ): - - """ - Lays out a set of input PDFs into a larger pdf in a grid. - NOTE: The 'unit' and 'dpi' choice will be used for both (input and output). - WARNING: I STRONGLY RECOMMEND NOT USING DPI AS A SETTING. THE PDF FORMAT ITSELF DOESN'T HAVE ANY PROVISION FOR - PIXEL DENSITY. THIS SETTING HAS BEEN MADE AVAILABLE FOR PLACEBO EFFECT WHEN CLIENTS ASK FOR VERY HIGH RESOLUTION - FILES WITHOUT WANTING TO UNDERSTAND ANY TECHNICALITY. USE WITH CAUTION. - :param pdf_files: The list of PDF files (as BytesIO objects) to lay onto the larger canvas. - :param input_width: The width of the input PDFs. - :param input_height: The height of the input PDFs - :param output_width: The width of the large PDF on which the smaller PDFs will be laid out. - :param output_height: The height of the large PDF on which the smaller PDFs will be laid out. - :param cutting_width: The horizontal buffer to leave between the grid. - :param cutting_height: The vertical buffer to leave between the grid. - :param unit: The unit to use (common to the input and output PDFs). - :param dpi: The pixel density to consider (common to the input and output PDFs). - :return: The PDF grid. - """ - - # Create the base PDF: - base_canvas = PDFMaker( - width = output_width, - height = output_height, - unit = unit, - dpi = dpi - ) - base_canvas.draw_line(0, 1, 1, 0, alpha = 0.0) - base_pdf = base_canvas.save() - - # Adjust all units to match the pixel density adjustments: - input_width *= base_canvas.unit_factor - input_height *= base_canvas.unit_factor - output_width *= base_canvas.unit_factor - output_height *= base_canvas.unit_factor - cutting_width *= base_canvas.unit_factor - cutting_height *= base_canvas.unit_factor - - # Calculate how many input pdfs will fit and the page count needed: - per_row = math.floor((output_width - cutting_width) / (input_width + cutting_width)) - per_column = math.floor((output_height - cutting_height) / (input_height + cutting_height)) - per_page = per_row * per_column - output_page_count = math.ceil(len(pdf_files) / per_page) - - # Calculate the starting point on the page from where the pasting will start: - page_start_x = (output_width - (per_row * (input_width + cutting_width)) + cutting_width) / 2.0 - page_start_y = (output_height - (per_column * (input_height + cutting_height)) + cutting_height) / 2.0 - - # We make a writer object to write pages to the output file, - # and we make a list that will hold the individual output pages: - grid_pdf = PdfWriter() - - # We create each needed page: - for page_number in range(output_page_count): - - # Open the base PDF and create a page here: - grid_page = PdfReader(base_pdf).pages[0] - - # Shortlist the input pdfs to be pasted on this page: - starting_offset = page_number * per_page - ending_offset = starting_offset + per_page - ending_offset = min(ending_offset, len(pdf_files)) - pdf_files_for_page = pdf_files[starting_offset:ending_offset] - - # We paste all the input PDFs one-by-one: - for index, input_pdf in enumerate(pdf_files_for_page): - - # Compute the coordinates to paste on: - row_number = math.floor(index / per_row) - column_number = index if index < per_row else index % per_row - paste_x = page_start_x + (column_number * input_width) + (column_number * cutting_width) - paste_y = page_start_y + (row_number * input_height) + (row_number * cutting_height) - - # The default system for the reportlab is bottom-up, - # we convert to top-down for intuitive operations: - paste_y = base_canvas.shape[1] - paste_y - input_height - - # Open th smaller PDF and paste it on those coordinates: - input_page = PdfReader(input_pdf).pages[0] - input_page.add_transformation(Transformation().translate( - tx = paste_x, - ty = paste_y - ), expand = True) - grid_page.merge_page(input_page) - - # Add the grid page to the grid PDF: - grid_pdf.add_page(grid_page) - - # Join and return the result: - grid_pdf_in_ram = io.BytesIO() - grid_pdf.write(grid_pdf_in_ram) - return grid_pdf_in_ram - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - - async def main(): - - my_pdf = PDFMaker(width = 210, height = 297) - # my_image = await my_pdf.image_from_url(r"https://images.pexels.com/photos/28973930/pexels-photo-28973930/free-photo-of-historic-saigon-central-post-office-architecture.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1") - # my_image = await my_pdf.image_from_url(r"") - # my_image = await my_pdf.image_from_url(r"") - my_image = await my_pdf.image_from_url(r"https://images.pexels.com/photos/18317748/pexels-photo-18317748/free-photo-of-peoples-committee-of-ho-chi-minh.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=1") - # my_image.show() - blurred_image = PDFMaker.fit_image_to_aspect_ratio(my_image, 1.49) - blurred_image.show() - - asyncio.run(main()) diff --git a/utils_v2/queue/__init__.py b/utils_v2/queue/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/queue/async_kafka.py b/utils_v2/queue/async_kafka.py deleted file mode 100644 index d6c0a82..0000000 --- a/utils_v2/queue/async_kafka.py +++ /dev/null @@ -1,554 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Saturday, 13th Jul, 2024 - - OBJECTIVE: - - To be able to work with keys - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To use Kafka: -from aiokafka import AIOKafkaProducer -from aiokafka import AIOKafkaConsumer - -# For working with JSON strings: -from utils_v2.string import json -from utils_v2.serialization.json_serializer import JSONSerializer - -# For debugging: -from icecream import IceCreamDebugger - -# For SSL security: -import ssl - -# For asynchronous activities: -import asyncio - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def get_ssl_context( - ca_file, - cert_file, - key_file -): - - """ - Generate the SSL context to use with the Kafka instances. - :param ca_file: The Certificate Authority file as a path to a local file. - :param cert_file: The Certificate file as a path to a local file. - :param key_file: The Key file as a path to a local file. - :return: The SSL context instance as a path to a local file. - """ - - ssl_context = ssl.create_default_context() - ssl_context.load_verify_locations(ca_file) - ssl_context.load_cert_chain(certfile = cert_file, keyfile = key_file) - return ssl_context - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class ProducerKafka: - - def __init__( - self, - topic, - serializer = None, - debug = True, - debug_prefix = "Kafka (P) | ", - **kwargs - ): - - """ - Create a Kafka Producer. - :param topic: The topic to produce on. - :param serializer: The serializer to use. - :param debug: Whether, or not, you want to print the debug strings. - :param debug_prefix: The prefix to use while debugging. - :param kwargs: Any configuration parameters for the Kafka instances. - """ - - # Initialize the debugger: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - # initialize the Kafka producer: - self.__topic = topic - self.__kwargs = kwargs - self.__producer = None - self.__connected = False - self.__serializer = serializer or JSONSerializer() - - # For establishing connection: - self.__exclusive_semaphore = asyncio.Semaphore(1) - - def enable_debug(self): - self.__printer.enable() - - def disable_debug(self): - self.__printer.disable() - - async def connect(self): - - """ - Connects to the Kafka server if not connected. - :return: True or False based on the success of the operation. - """ - - async with self.__exclusive_semaphore: - if not self.__connected: - try: - self.__producer = AIOKafkaProducer(**self.__kwargs) - await self.__producer.start() - self.__connected = True - except Exception as exception: self.__printer(exception) - return self.__connected - - async def ensure_connection(self): - - """ - Connects to the Kafka server if not connected. - :return: True or False based on the success of the operation. - """ - - if not self.__connected: await self.connect() - return self.__connected - - async def close(self): - - """ - Terminates the connection. - :return: None. - """ - - if self.__connected: - try: - await self.__producer.stop() - self.__printer("Producer closed!") - self.__connected = False - except Exception as exception: self.__printer(exception) - - async def produce(self, value, key = None, topic = None, encoding = "utf-8"): - - """ - Sends one message to the Kafka server on the topic that has been set for this instance. - :param value: The message to send. - :param key: The key to use when you want the messages to follow an order. - :param topic: A custom topic for this message, else the topic defined during the creation of this instance will - be used by default. - :param encoding: The encoding format. - :return: True or False based on the success of the operation. - """ - - # Ensure connectivity to the server. - # If not connected, return with failure immediately. - if not await self.ensure_connection(): return False - - try: - - # Send the message: - await self.__producer.send_and_wait( - topic = topic or self.__topic, - value = self.__serializer.serialize(data = value, encoding = encoding), - key = key - ) - - # Return with success if no exception occurred: - return True - - # Return with failure if something went wrong: - except Exception as exception: - self.__printer(exception, self.__topic, type(value), value) - return False - - -# --------------------------------------------------------------------------------------------------------------------- - - -class ConsumerKafka: - - def __init__( - self, - topic, - serializer = None, - debug = True, - debug_prefix = "Kafka (C) | ", - **kwargs - ): - - """ - Create a Kafka Consumer. - :param topic: The topic to consumer on. - :param serializer: The serializer to use. - :param debug: Whether, or not, you want to print the debug strings. - :param debug_prefix: The prefix to use while debugging. - :param kwargs: Any configuration parameters for the Kafka instances. - """ - - # Initialize the debugger: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - # initialize the Kafka producer: - self.__topic = topic - self.__kwargs = kwargs - self.__consumer = None - self.__connected = False - self.__serializer = serializer or JSONSerializer() - - # For establishing connection: - self.__exclusive_semaphore = asyncio.Semaphore(1) - - def enable_debug(self): - self.__printer.enable() - - def disable_debug(self): - self.__printer.disable() - - async def connect(self): - - """ - Connects to the Kafka server if not connected. - :return: True or False based on the success of the operation. - """ - - async with self.__exclusive_semaphore: - if not self.__connected: - try: - self.__consumer = AIOKafkaConsumer(self.__topic, **self.__kwargs) - await self.__consumer.start() - self.__connected = True - except Exception as exception: self.__printer(exception) - return self.__connected - - async def ensure_connection(self): - - """ - Connects to the Kafka server if not connected. - :return: True or False based on the success of the operation. - """ - - if not self.__connected: await self.connect() - return self.__connected - - async def close(self): - - """ - Terminates the connection. - :return: None. - """ - - if self.__connected: - try: - await self.__consumer.stop() - self.__printer("Consumer closed!") - self.__connected = False - except Exception as exception: self.__printer(exception) - - async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"): - - """ - Get messages from the Kafka server. - :param count: The number of messages to get from the Kafka server. - :param timeout: The time in seconds to wait for retrieval. - :param encoding: The encoding to use. - :return: The messages that were received. If no messages are available, an empty list will be returned. - """ - - # Ensure connectivity to the server. - # If not connected, return with failure immediately. - if not await self.ensure_connection(): return [] - - # Make a variable that will hold the final results: - messages = [] - - try: - - # Read some messages: - results = await self.__consumer.getmany( - max_records = max(1, count), - timeout_ms = int(timeout * 1_000) - ) - - # Format the received messages: - if results: - for topic_partition, records in results.items(): - for record in records: - record_dict = record.__dict__ - record_dict["value"] = self.__serializer.deserialize( - data = record_dict["value"], - encoding = encoding - ) - messages.append(record_dict) - - # Debugging print if something went wrong: - except Exception as exception: self.__printer(exception) - - # Done here: - return messages - - -# --------------------------------------------------------------------------------------------------------------------- - - -class BidirectionalKafka: - - # The 'roles' that the instance can take. - # The master talks on the channel (topic) that the slave listens on and vice versa. - # Master-Slave is only for deciding who talks on which channel and who listens on which. - # In a two-party system, one must be the master, the other must be the slave. - # There are no extra privileges that the master enjoys. The naming convention was borrowed from common protocols - # used in electronics (like I2C). - ROLE_MASTER = 1 - ROLE_SLAVE = 0 - - def __init__( - self, - role, - topic, - ack_topic: str = None, - group: str = None, - serializer = None, - debug = True, - debug_prefix = "Kafka (B) | ", - **kwargs - ): - - """ - Creates a walkie-talkie type setup to use Kafka in a bidirectional manner. Fo more information on all the - individual methods, please read through the doc-strings of the component classes 'ProducerKafka', and - 'ConsumerKafka'. - :param role: Select from "ROLE_MASTER" and "ROLE_SLAVE". Between the two parties that are talking, one will be - the master and the other will be the slave. The channel that the master uses to speak will the one the slave - uses to listen, and vice versa. - :param topic: The topic to communicate on. Will be the same between the master and the slave. - :param ack_topic: Explicitly provide this for the second channel, or it will be created from the name of the - topic itself. Will be the same between the master and the slave. - :param group: The group to assign the instance to. - :param debug: Whether, or not, you want to print the debug strings. - :param debug_prefix: The prefix to use while debugging. - :param kwargs: Any configuration parameters for the Kafka instances. - """ - - # Not down the basic variables: - self.__role = role - self.__topic = topic - self.__ack_topic = ack_topic or topic + "Ack" - self.__group = group - - # In case the current instance is the master, - # it will talk on "topic", and listen on "ack_topic": - if self.__role == self.ROLE_MASTER: - self.__producer_kwargs = kwargs.copy() - self.__producer = ProducerKafka( - topic = self.__topic, - serializer = serializer, - debug = debug, - debug_prefix = debug_prefix.strip() + " (P) | ", - **self.__producer_kwargs - ) - self.__consumer_kwargs = kwargs.copy() - self.__consumer_kwargs["group_id"] = self.__group - self.__consumer = ConsumerKafka( - topic = self.__ack_topic, - group = group, - serializer = serializer, - debug = debug, - debug_prefix = debug_prefix.strip() + " (C) | ", - **self.__consumer_kwargs - ) - - # On the other hand, if the current instance is a slave, - # It will listen on "topic", and talk on "ack_topic": - else: - self.__producer_kwargs = kwargs.copy() - self.__producer = ProducerKafka( - topic = self.__ack_topic, - serializer = serializer, - debug = debug, - debug_prefix = debug_prefix.strip() + " (P) | ", - **self.__producer_kwargs - ) - self.__consumer_kwargs = kwargs.copy() - self.__consumer_kwargs["group_id"] = self.__group - self.__consumer = ConsumerKafka( - topic = self.__topic, - group = group, - serializer = serializer, - debug = debug, - debug_prefix = debug_prefix.strip() + " (C) | ", - **self.__consumer_kwargs - ) - - def enable_debug(self): - self.__producer.enable_debug() - self.__consumer.enable_debug() - - def disable_debug(self): - self.__producer.disable_debug() - self.__consumer.disable_debug() - - async def ensure_connection(self): - await self.__producer.ensure_connection() - await self.__consumer.ensure_connection() - - async def close(self): - await self.__producer.close() - await self.__consumer.close() - - async def produce(self, message, encoding = "utf-8"): - return await self.__producer.produce(message, encoding = encoding) - - async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"): - return await self.__consumer.consume(count = count, timeout = timeout, encoding = encoding) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - import time - from data_models.kafka_message import KafkaMessage - - ssl_ctx = get_ssl_context( - ca_file = r"/home/developer/PycharmProjects/utils/cred/kafka/cert_authority.pem", - cert_file = r"/home/developer/PycharmProjects/utils/cred/kafka/fullchain.pem", - key_file = r"/home/developer/PycharmProjects/utils/cred/kafka/privkey.pem" - ) - - async def consumer_test(): - - consumer = ConsumerKafka( - topic = "kft_file_upload", - # group_id = "assessImg", - group_id = "updateMedia", - bootstrap_servers = "wtt.ditscentre.in:9092", - security_protocol = "SSL", - ssl_context = ssl_ctx - ) - await consumer.connect() - await asyncio.sleep(1.5) - print("READY!") - - while True: - messages = await consumer.consume(count = 1) - if len(messages) > 0: print("MESSAGE:", json.to_string(messages[0], default=str)) - await asyncio.sleep(1.0) - - async def producer_test(): - - producer = ProducerKafka( - topic = "kft_file_upload", - bootstrap_servers = "del.ditscentre.in:9092", - security_protocol = "SSL", - ssl_context = ssl_ctx - ) - await producer.connect() - print("READY!") - - while True: - my_msg = KafkaMessage( - data = { - "accepted": False, - "reason": "low resolution" - }, - media = { - "name": "pikachu_poster.jpg", - "ext": "jpg", - "url": "https://nexcom.ditscentre.in/utils/files/small/download/66ded1c1c1c05139a618b5ff", - "attr": { - "user": "SarangKabir", - "project": "ACE-PGP", - "id": 173, - "campaignActivityId": "25", - "idCampaign": 49, - "phoneNo": "7977821877" - } - }, - appId = "aceWockhardt", - proc = { - "name": "_assessImg", - "attr": { - "blurThreshold": 0.25, - "clarityThreshold": 0.65, - "nsfwThreshold": 0.25, - "minWidth": 512, - "minHeight": 512 - } - }, - ack = None - ) - success = await producer.produce(my_msg.model_dump()) - print("produced...") - time.sleep(1.0) - break - - await producer.close() - - - asyncio.run(consumer_test()) diff --git a/utils_v2/rate_limit/__init__.py b/utils_v2/rate_limit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/rate_limit/async_token_bucket.py b/utils_v2/rate_limit/async_token_bucket.py deleted file mode 100644 index 4fb85e7..0000000 --- a/utils_v2/rate_limit/async_token_bucket.py +++ /dev/null @@ -1,212 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Wednesday, 15th May, 2024 - - OBJECTIVE: - - To provide a rate-limiting mechanism using the simple token bucket algorithm. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# For date and time keeping: -import datetime -import time - -# For asynchronous activities: -import asyncio - -# For mathematical operations: -import math - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class TokenBucket: - - """ - MODES: "add" - Add 'rate_limit' no. of tokens to the bucket every interval. - "reset" - Reset the token count to 'rate_limit' every interval - """ - - MODE_RESET = 0 - MODE_ADD = 1 - - ONE_YEAR = 3_15_36_000.0 - ONE_MONTH = 26_78_400.0 - ONE_WEEK = 6_04_800.0 - ONE_DAY = 86_400.0 - ONE_HOUR = 3_600.0 - ONE_MINUTE = 60.0 - ONE_SECOND = 1.0 - - def __init__(self, rate_limit, seconds = 1.0, mode = "reset", sleep = 0.1, in_sequence = False): - - """ - Initialize the rate controller. - :param rate_limit: The no. of operations allowed per unit of time. - :param seconds: The time period in seconds in which the tokens get reset or added. - :param mode: To select what happens when the period is over. 'reset' mode means that the remaining tokens from - the previous period are discarded and the counter is set back to the rate limit, and 'add' mode means that - new tokens are added on top of the exiting ones that were unused in the previous period. - :param sleep: The delay to add before checking back to see if tokens are available. Keep it longer for longer - periods (which is adjusted by the 'seconds' parameter). - :param in_sequence: Whether, or not, you want to maintain the sequence in which the requests were made - (experimental). - """ - - if rate_limit is not None: - rate_limit = int(rate_limit) - self.__semaphore = asyncio.Semaphore(1 if in_sequence else rate_limit) - mode = mode.lower() - if mode not in [self.MODE_RESET, self.MODE_ADD]: mode = self.MODE_RESET - self.__mode = mode - self.__rate_limit = rate_limit - self.__token_count = rate_limit - self.__last_token_generation_time = time.time() - self.__sleep = sleep - self.__seconds = seconds - - async def has_turn(self): - - """ - Just to check if a token is available. - Should be used only if you don't want to wait for turn in case it is unavailable. - :return: True if the token is available, False if not. - """ - - # If the user doesn't want any rate-limits, we return immediately. - # Otherwise, we check if the user has any tokens available: - if self.__rate_limit is None: return True - if self.__token_count > 0: return True - else: return False - - async def get_turn(self, timeout = None): - - """ - To wait till either you get your turn or the wait gets timed-out. - :param timeout: The time (in seconds) to wait to get a turn before exiting with failure. - :return: True if a turn was received, else False if timed-out. - """ - - # If the user doesn't want any rate-limits, we return immediately: - if self.__rate_limit is None: return - - # Start by making variables: - got_turn = False - wait_start = time.time() - - # Otherwise, we wait for the semaphore: - async with self.__semaphore: - - # Wait till you get your turn or the attempt gets timed-out: - while True: - - # If the timeout has been exceeded: - if ( - timeout is not None and - time.time() - wait_start >= timeout - ): break - - # If it is time to generate new tokens: - time_delta = time.time() - self.__last_token_generation_time - if time_delta >= self.__seconds: - if self.__mode == self.MODE_RESET: self.__token_count = self.__rate_limit - else: self.__token_count += int(self.__rate_limit * math.floor(time_delta / self.__seconds)) - self.__last_token_generation_time = time.time() - - # If a token is available (or not): - if self.__token_count > 0: - self.__token_count -= 1 - got_turn = True - break - else: await asyncio.sleep(self.__sleep) - - # Done here: - return got_turn - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import random - - async def single_test(rate_controller, count): - has_turn = await rate_controller.has_turn() - got_turn = await rate_controller.get_turn() - print(f"TURN: {count:.<5} {'Y' if has_turn else '-'} / {'Y' if got_turn else '-'} ({datetime.datetime.now()})") - if got_turn: await asyncio.sleep(random.random()) - - async def multi_test(max_count): - rate_controller = TokenBucket( - rate_limit = 10, - seconds = 1, - in_sequence = False, - sleep = 0.025, - mode = "reset", - timeout = 3 - ) - tasks = [single_test(rate_controller, count + 1) for count in range(max_count)] - await asyncio.gather(*tasks) - - - start_time = time.time() - asyncio.run(multi_test(100)) - print(f"FINISHED IN {time.time() - start_time} SECONDS!") diff --git a/utils_v2/security/__init__.py b/utils_v2/security/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/security/__pycache__/__init__.cpython-310.pyc b/utils_v2/security/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 0a34e9d..0000000 Binary files a/utils_v2/security/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc b/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc deleted file mode 100644 index 62e16b6..0000000 Binary files a/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/security/hash.py b/utils_v2/security/hash.py deleted file mode 100644 index b3fc582..0000000 --- a/utils_v2/security/hash.py +++ /dev/null @@ -1,245 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Monday, 26th Aug., 2024 - - OBJECTIVE: - - To provide an easy way to hash inputs. - - REFERENCES: - - 1) Book: Full Stack Python Security - Dennis Byrne - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# For hashing: -import hashlib -import hmac -import secrets -from bcrypt import hashpw, gensalt - -# To work with buffers: -import io - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class Hasher: - - def __init__( - self, - algorithm = hashlib.sha256, - key = None - ): - - """ - hashes data or a message or a file. Uses HMAC if 'key' is specified, else performs simple hashing. - :param algorithm: The algorithm to use. SHA256 by default. - :param key: Specify this as either a string or as bytes to use HMAC. Leave as null for simple hashing. - """ - - # Initialize the hasher: - self.__hasher = None, - self.__algorithm = algorithm - self.__hmac_key = key - if self.__hmac_key is not None: - self.__hmac_key = self.__hmac_key.encode("utf-8") if isinstance(self.__hmac_key, str) else self.__hmac_key - self.reset() - - @staticmethod - def generate_key(byte_count = 32, url_safe = False): - - """ - A mechanism to generate key/salt values. - NOTE: For a proper salt for passwords, I recommend using "generate_salt" method. It's far better. - :param byte_count: The number of bytes to have in the key. The hex output (as string) will have 2x the - characters. - :param url_safe: Set to True if you need the generated output to be a part of a URL. - :return: The generate key/salt. - """ - - return secrets.token_urlsafe(byte_count) if url_safe else secrets.token_hex(byte_count) - - @staticmethod - def generate_salt(): - - """ - Generate a salt to use while hashing things like passwords. - :return: The salt as bytes. - """ - - return gensalt() - - @staticmethod - def hash_password(password, salt): - - """ - Hashes a password with the given salt. - :param password: The password to hash, either as a string or as bytes. - :param salt: The salt to hash the password with, either as a string or as bytes. - :return: The hashed string. - """ - - return hashpw( - password = password.encode("utf-8") if isinstance(password, str) else password, - salt = salt.encode("utf-8") if isinstance(salt, str) else salt - ) - - @staticmethod - def compare_hashes(hash_0, hash_1): - - """ - Compares two hashes to see if they match. - Comparison is done in constant time to avoid timing-based side-channel attacks. - :param hash_0: One of the hashes to compare. - :param hash_1: The other hash to compare. - :return: True if they match, else False. - """ - - return hmac.compare_digest(hash_0, hash_1) - - def reset(self): - - """ - Resets the hasher by removing all the data that was fed into it. - :return: None. - """ - - if self.__hmac_key is not None: - self.__hasher = hmac.new( - key = self.__hmac_key, - digestmod = self.__algorithm - ) - else: self.__hasher = self.__algorithm() - - def update(self, data): - - """ - Adds data to the hash to update it. - :param data: The data to be hashed. - :return: None. - """ - - data = data.encode("utf-8") if isinstance(data, str) else data - self.__hasher.update(data) - - def digest(self): - - """ - Returns the hexadecimal representation of the hash as a string. - :return: The hexadecimal representation of the hash as a string - """ - - return self.__hasher.digest() - - def hexdigest(self): - - """ - Returns the hexadecimal representation of the hash as a string. - :return: The hexadecimal representation of the hash as a string - """ - - return self.__hasher.hexdigest() - - def hash_message(self, message, as_hex = True): - - """ - Hashes one message and returns the result, and then resets the instance. - :param message: The data you want to hash. - :param as_hex: Invokes 'hexdigest' if True, else 'digest'. - :return: The hash of the message in either hexadecimal string form or binary form. - """ - - self.update(message) - hash_result = self.hexdigest() if as_hex else self.digest() - self.reset() - return hash_result - - def hash_file(self, file, chunk_size = 4096, as_hex = True): - - """ - Hashes one file and returns the result, and then resets the instance. - :param file: The file you want to hash either as a path or as some buffer (like io.BytesIO). - :param chunk_size: The size of data (in bytes) that you would like to pick at one time. - :param as_hex: Invokes 'hexdigest' if True, else 'digest'. - :return: The hash of the file in either hexadecimal string form or binary form. - """ - - # In case the file was given as a io.BytesIO buffer: - if isinstance(file, io.BytesIO): - file.seek(0) - while True: - chunk = file.read(chunk_size) - if not chunk: break - self.update(chunk) - - # In case the file was given as a path: - else: - with open(file, "rb") as f: - for chunk in iter(lambda: f.read(chunk_size), b""): - self.update(chunk) - - # Now we capture the results, reset the instance, and return the result: - hash_result = self.hexdigest() if as_hex else self.digest() - self.reset() - return hash_result - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - print( - Hasher.hash_password( - password = "mic test, mic test, 123", - salt = Hasher.generate_salt() - ) - ) diff --git a/utils_v2/security/otp.py b/utils_v2/security/otp.py deleted file mode 100644 index 49edcd2..0000000 --- a/utils_v2/security/otp.py +++ /dev/null @@ -1,156 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 10th Sept., 2024 - - OBJECTIVE: - - To provide a way to generate and verify OTPs. - - REFERENCES: - - 01. https://pyauth.github.io/pyotp/# - 02. https://en.wikipedia.org/wiki/Google_Authenticator - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To work with OTPs: -import pyotp -import base64 -import hashlib - -# To work with date and time: -import time -import datetime - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class HashedOTP: - - def __init__( - self, - secret - ): - - """ - Used to generate and verify HMAC-based OTPs. - :param secret: The key to use to generate and verify OTPs. - """ - - # Note down the input variables: - self.__secret = secret - self.__otp_client = pyotp.HOTP(secret) - - @staticmethod - def generate_secret(message = None): - - """ - Generate a secret key to then use to generate and verify the OTPs. - You may override the random generator by giving a "message" of any length. - :param message: A custom value to convert into a key. Avoid using this for better security, but this can be used - to generate keys based on user identifiers. THERE IS NO RANDOMNESS IF YOU USE THIS FEATURE. IT IS FOR - CONVENIENCE ONLY. NOT RECOMMENDED. - :return: The key (as a string) that can be used to generate and verify the OTPs. - """ - - # If the user wants to generate a key from a custom input: - if message: - - # Ensure we have a bytes object: - if not isinstance(message, (str, bytes)): message = str(message) - if isinstance(message, str): message = message.encode("utf-8") - - # Hash the bytes object: - sha256_hash = hashlib.sha256() - sha256_hash.update(message) - hashed_key = sha256_hash.digest() - - # Convert to base-32: - return base64.b32encode(hashed_key).decode("utf-8") - - # If the user wants a totally random key: - else: return pyotp.random_base32() - - def generate_otp(self, count: int): - - """ - Generates the OTP at a particular step. - :param count: The step at which the OTP needs to be generated. - :return: The OTP string (6 digits). - """ - - return str(self.__otp_client.at(count)) - - def verify_otp(self, otp, count: int): - - """ - Verifies the claimed OTP. - :param otp: The OTP as claimed by the end user. - :param count: The step at which the OTP needs to be verified. - :return: True if the OTP is valid, else False. - """ - - return self.__otp_client.verify(otp, counter = count) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/security/sanitizers.py b/utils_v2/security/sanitizers.py deleted file mode 100644 index 26dfe5b..0000000 --- a/utils_v2/security/sanitizers.py +++ /dev/null @@ -1,135 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Thursday, 25th Jul., 2024 - - OBJECTIVE: - - To provide a set of data cleaning functions for inputs like phone numbers, emails, etc. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# My utils: -from utils_v2.string import regex - -# For random strings and tokens: -import string -import random - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def file_name(input_string: str): - - """ - Cleans up the string to allow it to safely become a filename. - :param input_string: The string that you want to make safe for using as a filename. - :return: The string that can safely be used as a filename. - """ - - return regex.replace( - text = input_string.replace("\n", " "), - pattern = r"[^a-zA-Z0-9 \-_\.]", - substitute_text = "" - ) - - -# --------------------------------------------------------------------------------------------------------------------- - - -def for_mongo(input_document): - - """ - Sanitizes and disarms any JSON-like input that could be used for NoSQL-injection attacks. - :param input_document: The list or dict to be sanitized. - :return: The sanitized list or dict. - """ - - # A special function that disarms any input string by dealing with special characters - # that Mongo may consider to be instructions: - def disarm(input_string): - input_string = regex.replace( - text = input_string, - pattern = r"[^a-zA-Z0-9,_\-\.\\\/:;'\(\) ]", - substitute_text = "" - ) - return input_string - - # Initially we assign the value of the input to the output: - sanitized_document = input_document - - # Handle the case where the input is an array: - if isinstance(input_document, list): - sanitized_document = [for_mongo(document) for document in input_document] - - # Handle the case when the input is a document: - elif isinstance(input_document, dict): - sanitized_document = {} - for k, v in input_document.items(): - sanitized_document[disarm(k)] = v if type(v) not in [list, dict] else for_mongo(v) - - # Done here: - return sanitized_document - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/serialization/__init__.py b/utils_v2/serialization/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/serialization/json_serializer.py b/utils_v2/serialization/json_serializer.py deleted file mode 100644 index 87595e0..0000000 --- a/utils_v2/serialization/json_serializer.py +++ /dev/null @@ -1,129 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Sunday 1st Sept. 2024. - - OBJECTIVE: - - To provide a way to convert any input data to serialized bytes, and back. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To work with files: -from utils_v2.string import json - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class JSONSerializer: - - def __init__(self): - - """ - Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and - python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour. - """ - - pass - - @staticmethod - def serialize(data, encoding = "utf-8"): - - """ - Serializes the data that is given to it. - The input has to be JSON-compatible. - :param data: The data to serialize. - :param encoding: The encoding to use. - :return: The bytes representing the data. - """ - - # If the data is not already a JSON string, parse it. Then return it as bytes: - data = data if isinstance(data, str) else json.to_string(data, no_space = True) - return data.encode(encoding) - - @staticmethod - def deserialize(data, encoding = "utf-8"): - - """ - Deserializes the bytes that are given to it. - The input has to be JSON-compatible. - :param data: The bytes to deserialize. - :param encoding: The encoding to use. - :return: The data from the bytes that described it. - """ - - data = data.decode(encoding) - return json.from_string(data) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/serialization/pickle_serializer.py b/utils_v2/serialization/pickle_serializer.py deleted file mode 100644 index 8dd2b65..0000000 --- a/utils_v2/serialization/pickle_serializer.py +++ /dev/null @@ -1,122 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Saturday 26th Oct. 2024. - - OBJECTIVE: - - To provide a way to convert any input data to serialized bytes, and back. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To work with pickling: -import pickle - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class PickleSerializer: - - def __init__(self): - - """ - Use this serializer when working with custom python objects. This is meant to be fully flexible, but efficiency - is not guaranteed. - """ - - pass - - @staticmethod - def serialize(data): - - """ - Serializes the data that is given to it. - :param data: The data to serialize. - :return: The bytes representing the data. - """ - - return pickle.dumps(data) - - @staticmethod - def deserialize(data): - - """ - Deserializes the bytes that are given to it. - :param data: The bytes to deserialize. - :return: The data from the bytes that described it. - """ - - return pickle.loads(data) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/serialization/universal_serializer.py b/utils_v2/serialization/universal_serializer.py deleted file mode 100644 index 23818c8..0000000 --- a/utils_v2/serialization/universal_serializer.py +++ /dev/null @@ -1,265 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Sunday 1st Sept. 2024. - - OBJECTIVE: - - To provide a way to convert any input data to serialized bytes, and back. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To work with files: -from utils_v2.string import json - -# To work with tabulated data: -import pandas as pd - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class UniversalSerializer: - - def __init__(self): - - """ - Use this when you are working with varied datatypes. You can add custom data-converters also using the - 'add_converters' method. Otherwise, most default pythonic datatypes are supported out of the box. Note that this - is NOT recommended because of how large the serialized messages become. Try using 'JSONSerializer' when you know - you will be working specifically with JSOn-compatible inputs. - """ - - # These are the converters to use when serializing data: - self.__forward_converters = { - "set": lambda x: list(x), - "tuple": lambda x: list(x), - "complex": lambda x: {"r": x.real, "i": x.imag}, - "DataFrame": lambda x: x.to_dict() - } - - # These are the converters to use when deserializing data: - self.__reverse_converters = { - "set": lambda x: set(x), - "tuple": lambda x: tuple(x), - "complex": lambda x: complex(x["r"], x["i"]), - "DataFrame": lambda x: pd.DataFrame.from_dict(x) - } - - def add_converters( - self, - type_name, - forward_converter_func, - reverse_converter_func - ): - - """ - Add custom datatype converters. - RULES: - 01. Each of the converter functions must take in exactly on argument and return one output of native python - type. This is very important. - 02. Each forward and reverse converters must give symmetric results. - :param type_name: The name of the datatype. HINT: type(obj).__name__ - :param forward_converter_func: The function to handle conversion to bytes. Use when serializing. - :param reverse_converter_func: The function to handle conversion from bytes. Used when deserializing. - :return: None. - """ - - self.__forward_converters[type_name] = lambda x: forward_converter_func(x) - self.__reverse_converters[type_name] = lambda x: reverse_converter_func(x) - - def __describe(self, data): - - """ - Notes down the input datatypes of everything. - Does everything upto conversion to byes. - :param data: The data to process. - :return: The description of the datatypes and values of what was given. - """ - - # Note down the type of data that was sent as the input: - data_type = type(data).__name__ - - # Handle iterables: - if isinstance(data, list): data = [self.__describe(item) for item in data] - elif isinstance(data, set): data = [self.__describe(item) for item in data] - elif isinstance(data, tuple): data = [self.__describe(item) for item in data] - elif isinstance(data, dict): data = [ - { - "k": self.__describe(k), - "v": self.__describe(v) - } for k, v in data.items() - ] - - # Convert here, and return: - conv = self.__forward_converters.get(data_type) - if conv is not None: data = conv(data) - return {"d": data, "t": data_type} - - def serialize(self, data, encoding = "utf-8"): - - """ - Serializes the data that is given to it. - :param data: The data to serialize. - :param encoding: The encoding to use. - :return: The bytes representing the data. - """ - - data = self.__describe(data) - data = json.to_string(data, no_space = True) - return data.encode(encoding) - - def __interpret(self, data): - - """ - Interprets the types of data that were serialized originally. - :param data: The data in the serialized form. - :return: Data where the appropriate datatypes have been applied. - """ - - # Handle iterables: - if data["t"] == "list": data = [self.__interpret(item) for item in data["d"]] - elif data["t"] == "set": data = set([self.__interpret(item) for item in data["d"]]) - elif data["t"] == "tuple": data = tuple([self.__interpret(item) for item in data["d"]]) - elif data["t"] == "dict": data = { - self.__interpret(item["k"]): self.__interpret(item["v"]) - for item in data["d"] - } - - # Handle custom types: - else: - conv = self.__reverse_converters.get(data["t"]) - data = data["d"] - if conv is not None: data = conv(data) - - # Done here - return data - - def deserialize(self, data, encoding = "utf-8"): - - """ - Deserializes the bytes that are given to it. - :param data: The bytes to deserialize. - :param encoding: The encoding to use. - :return: The data from the bytes that described it. - """ - - data = data.decode(encoding) - data = json.from_string(data) - return self.__interpret(data) - - -# --------------------------------------------------------------------------------------------------------------------- - - -class JSONSerializer: - - def __init__(self): - - """ - Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and - python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour. - """ - - pass - - @staticmethod - def serialize(data, encoding = "utf-8"): - - """ - Serializes the data that is given to it. - The input has to be JSON-compatible. - :param data: The data to serialize. - :param encoding: The encoding to use. - :return: The bytes representing the data. - """ - - # If the data is not already a JSON string, parse it. Then return it as bytes: - data = data if isinstance(data, str) else json.to_string(data, no_space = True) - return data.encode(encoding) - - @staticmethod - def deserialize(data, encoding = "utf-8"): - - """ - Deserializes the bytes that are given to it. - The input has to be JSON-compatible. - :param data: The bytes to deserialize. - :param encoding: The encoding to use. - :return: The data from the bytes that described it. - """ - - data = data.decode(encoding) - return json.from_string(data) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/sms/__init__.py b/utils_v2/sms/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/sms/async_nimbus.py b/utils_v2/sms/async_nimbus.py deleted file mode 100644 index 1d4b926..0000000 --- a/utils_v2/sms/async_nimbus.py +++ /dev/null @@ -1,252 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Monday, 9th Sept., 2024 - - OBJECTIVE: - - To be able to send SMSs from Nimbus's API and manage the templates and other things from one place. - - REFERENCES: - - 01. https://github.com/innovativevijay/SmsHitApiSample - 02. https://nimbusit.net/appforms/apimanual.php - - DOWNLOADS: - - N/A - - WEB-PORTAL: - - 01. http://nimbusit.net/ - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To make API Calls: -import httpx - -# For debugging: -from icecream import IceCreamDebugger - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class AsyncNimbusSMS: - - MESSAGE_TYPE_REGULAR = 0 - MESSAGE_TYPE_UNICODE = 1 - - def __init__( - self, - entity_id, - sender_id, - user_id, - api_key, - debug = True, - debug_prefix = "Nimbus SMS | " - ): - - """ - Sets up an instance of the SMS sender through Nimbus IT. - :param entity_id: The entity id as registered with DLT. - :param sender_id: The 6-char code like "HDFCBK", "NSESMS", "ZRODHA" that you see in your SMS inbox. - :param user_id: The 6-digit id that Nimbus has assigned to you. - :param api_key: The API key generated through Nimbus's portal. - :param debug: Whether, or not, you would like to show debugging messages (can be changed on the fly). - :param debug_prefix: The prefix text to show with the debug string. - """ - - # Create the debugging tools: - self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) - if not debug: self.__printer.disable() - - # Create an HTTP client to work with: - limits = httpx.Limits( - max_connections = 5, - max_keepalive_connections = 5, - keepalive_expiry = 3600 - ) - self.__http_client = httpx.AsyncClient(limits = limits, timeout = 120) - - # Capture the input config: - self.__entity_id = entity_id - self.__sender_id = sender_id - self.__user_id = user_id, - self.__api_key = api_key - - async def get_balance(self): - - """ - Checks the balance in the Nimbus wallet. - :return: The balance (float) if the request was successful, else None. - """ - - balance = None - - try: - - # Call the API: - response = await self.__http_client.get( - url = r"http://nimbusit.net/api/balance", - params = {"user": self.__user_id, "authkey": self.__api_key} - ) - - # The response of a successful API call looks like "BALANCE:599". We need just the number: - if response.status_code in [200]: balance = float(response.content.decode().split(":")[-1].strip()) - - except Exception as exception: - self.__printer(exception) - - return balance - - async def send_sms( - self, - template_id, - recipient_number, - message, - message_type = MESSAGE_TYPE_REGULAR - ): - - """ - Sends one SMS through Nimbus IT's system. The text of the message must match the template that had been - submitted. A mismatch may cause the message to fail at best, and raise troubles in the real-world with - government bodies at worst. Be careful. - :param template_id: The id of the SMS template as registered on Nimbus's portal. - :param recipient_number: The phone number of the recipient. You can send an array of numbers, too, BUT IT IS - STRONGLY RECOMMENDED TO NOT DO THAT TO AVOID BEING BLOCKED BY DLT. - :param message: The message to send to the recipient. Should match the template that is being sent. - :param message_type: Choose between 'AsyncNimbusSMS.MESSAGE_TYPE_REGULAR' (default) and - 'AsyncNimbusSMS.MESSAGE_TYPE_UNICODE' based on the type of characters being sent. Both are class variables. - :return: The dict of all the details of the message that was sent including whether, or not, it was successfully - sent. Other details depend on the service provider (Nimbus IT in this case). - """ - - # Construct the basic structure of the response of this method: - summary = { - "success": False, - "info": None, - "sender": self.__sender_id, - "recipient": recipient_number, - "message": message, - "length": len(message), - "template_id": template_id, - "raw": None - } - - try: - - # Pre-process the recipient's number: - if not isinstance(recipient_number, (list, set, tuple)): recipient_number = [recipient_number] - - # Call the API: - response = await self.__http_client.get( - url = r"http://nimbusit.net/api/pushsms", - params = { - "user": self.__user_id, - "authkey": self.__api_key, - "sender": self.__sender_id, - "mobile": ",".join([str(num) for num in recipient_number]), - "text": message, - "entityid": self.__entity_id, - "templateid": template_id, - "type": message_type - } - ) - - # For a successful API call: - if response.status_code == 200: - response_json = response.json() - summary["success"] = True if response_json.get("STATUS", "ERROR").lower() in ["ok"] else False - summary["info"] = response_json.get("RESPONSE", {}).get("INFO") - summary["raw"] = { - "http_code": response.status_code, - "response": response_json, - } - - # For any other code that indicates some form of failure: - else: summary["raw"] = { - "http_code": response.status_code, - "response": response.content.decode() - } - - except Exception as exception: - self.__printer(exception) - - return summary - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import asyncio - - async def main(): - - sender = AsyncNimbusSMS( - entity_id = "", - sender_id = "", - user_id = "", - api_key = "" - ) - - response = await sender.send_sms( - template_id = "", - recipient_number = "", - message = "" - ) - print("SMS API RESPONSE:", response) - - my_balance = await sender.get_balance() - print("REMAINING BALANCE:", my_balance) - - - asyncio.run(main()) diff --git a/utils_v2/string/__init__.py b/utils_v2/string/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils_v2/string/__pycache__/__init__.cpython-310.pyc b/utils_v2/string/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 8836d67..0000000 Binary files a/utils_v2/string/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/string/__pycache__/json.cpython-310.pyc b/utils_v2/string/__pycache__/json.cpython-310.pyc deleted file mode 100644 index ddd97e0..0000000 Binary files a/utils_v2/string/__pycache__/json.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/string/__pycache__/regex.cpython-310.pyc b/utils_v2/string/__pycache__/regex.cpython-310.pyc deleted file mode 100644 index a0a5de8..0000000 Binary files a/utils_v2/string/__pycache__/regex.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/string/fuzzy.py b/utils_v2/string/fuzzy.py deleted file mode 100644 index 34c08d4..0000000 --- a/utils_v2/string/fuzzy.py +++ /dev/null @@ -1,168 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Saturday, 18th May, 2024 - - OBJECTIVE: - - To provide a quick set of functions to work with fuzzy logic. - - REFERENCES: - - 1) https://www.w3schools.com/python/python_json.asp - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To apply fuzzy logic: -from thefuzz import fuzz, process - -# To work with tabulated data: -import pandas as pd - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def get_best_match( - target, - choices, - threshold = 0.70, - partial = False -): - - if partial: scorer = fuzz.partial_token_sort_ratio - else: scorer = fuzz.ratio - - result = process.extractOne( - target, - choices, - score_cutoff = threshold * 100, - scorer = scorer - ) - - try: return result[0] - except: return None - - -# --------------------------------------------------------------------------------------------------------------------- - - -def rank(target, choices, partial = True): - - if partial: scorer = fuzz.partial_token_sort_ratio - else: scorer = fuzz.ratio - - result = process.extract( - target, - choices, - limit = len(choices), - scorer = scorer - ) - - result = pd.DataFrame(result, columns = ["choice", "closeness"]) - result["closeness"] = result["closeness"] / 100.0 - - return result - - -# --------------------------------------------------------------------------------------------------------------------- - - -def match(targets, choices, threshold = 0.7, partial = False, allow_null = False): - - all_matches_df = None - all_matches = {target: None for target in targets} - something_is_null = False - - for target in targets: - match_df = rank(target, choices, partial = partial) - match_df["target"] = target - if all_matches_df is None: all_matches_df = match_df - else: all_matches_df = pd.concat([all_matches_df, match_df]) - - all_matches_df = all_matches_df.sort_values(by = ["closeness"], ascending = False).reset_index(drop = True) - - for target in targets: - target_df = all_matches_df[all_matches_df["target"] == target].reset_index(drop = True) - if target_df.empty: continue - if target_df.at[0, "closeness"] >= threshold: - choice = target_df.at[0, "choice"] - all_matches[target] = choice - all_matches_df = all_matches_df[all_matches_df["choice"] != choice] - else: - all_matches[target] = None - something_is_null = True - - # print(all_matches) - if something_is_null and not allow_null: return None - else: return all_matches - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - import async_json_utils - - awb_numbers = [ - "SF1111BIC", - "SF2222BIC", - "SF3333BIC", - "SF4444BIC", - ] - - chat_text = "SF1112BIC" - - # print(chat_text == names[0]) - best_match = get_best_match(chat_text, awb_numbers, threshold = 0.60, partial = False) - print(f"Best match for '{chat_text}' is '{best_match}'") diff --git a/utils_v2/string/help.py b/utils_v2/string/help.py deleted file mode 100644 index 85dcc97..0000000 --- a/utils_v2/string/help.py +++ /dev/null @@ -1,177 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Saturday, 24th Aug., 2024 - - OBJECTIVE: - - To provide an overview of any function or class in a string. - The generated overview can then either be shown on the terminal, or transmitted over some other medium for - collaborative work. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# System-level activities: -import io -import inspect - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def get_help_for_class(cls, skip_methods = None): - - """ - Returns the help documentation to use this class. - :param cls: The class whose help string is desired. - :param skip_methods: A list of methods to NOT include in the help text. - :return: This help documentation. - """ - - separator = "\n\n" + ("=" * 120) + "\n\n" - if skip_methods is None: skip_methods = [] - elif not isinstance(skip_methods, list): skip_methods = [skip_methods] - - # Get class name and docstring: - class_name = cls.__name__ - docstring = inspect.getdoc(cls) or "" - help_string = "HELP FOR:\n\n" - help_string += class_name + "\n\n" - help_string += "This document has upto 120 chars per line.\n" - help_string += "Best viewed with monospaced font :)" - help_string += docstring + separator - - # Get all methods and their docstrings. - # Then note the documentation of the methods while ignoring the blacklisted ones: - members = inspect.getmembers(cls, predicate = inspect.isfunction) - func_help = [] - for name, method in members: - - # Ignore if asked, or extract the details: - if name in skip_methods or name.startswith(f"_{class_name}__"): continue - else: func_help.append(get_help_for_function(method)) - - # Put all the things together: - func_help = separator.join(func_help) - help_string += func_help - - # Done here: - return help_string - - -# --------------------------------------------------------------------------------------------------------------------- - - -def get_help_for_function(func): - - """ - Get the help string for one function. - It could be a standalone function, or a method of a class. - :param func: The function (or method) whose help string is needed. - :return: The help string of the function. - """ - - # Get the name and documentation: - func_name = func.__name__ - async_indicator = " (async)" if inspect.iscoroutinefunction(func) else "" - func_doc = inspect.getdoc(func) or "" - - # Create the decorated header: - func_decorator = "-" * (len(func_name) + 2) - func_head = "." + func_decorator + f".\n| {func_name} |{async_indicator}\n`" + func_decorator + "`\n\n" - - # Add the 'args' and 'kwargs': - func_args = [] - for name, param in inspect.signature(func).parameters.items(): - default = param.default - if isinstance(default, str): default = f"\"{default}\"" - if default == inspect.Parameter.empty: func_args.append(f"{name}") - else: func_args.append(f"{name}: {type(default).__name__} = {default}") - if len(func_args) > 0: func_args = f"{func_name} (\n\t" + "\n\t".join(func_args) + "\n):\n\n" - else: func_args = f"{func_name} ():\n\n" - - # Get the params and return value part from the doc: - params_start = func_doc.find(":param") - return_start = func_doc.find(":return") - func_params = "\n" + func_doc[params_start:return_start] if params_start >= 0 else "" - func_return = "\n" + func_doc[return_start:] if return_start >= 0 else "" - - # Isolate the documentation part: - if params_start >= 0: func_doc = func_doc[:params_start] - elif return_start >= 0: func_doc = func_doc[:return_start] - - # Done here: - return func_head + func_args + func_doc + func_params + func_return - - -# --------------------------------------------------------------------------------------------------------------------- - - -def get_help(entity, skip_methods = None): - - """ - Get the help documentation for anything from its docstring. - :param entity: The entity you want to get help for. - :param skip_methods: A list of methods to ignore if inspecting a class. Not valid for standalone functions. - :return: The help string. - """ - - if inspect.isclass(entity): return get_help_for_class(entity, skip_methods = skip_methods) - else: return get_help_for_function(entity) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/string/json.py b/utils_v2/string/json.py deleted file mode 100644 index 67d39f7..0000000 --- a/utils_v2/string/json.py +++ /dev/null @@ -1,202 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Create: Saturday, 18th May, 2022 - Update: Thursday, 22nd Aug. 2024 - - OBJECTIVE: - - To provide an easy way to work with '.json' data and files. - - REFERENCES: - - 1) https://www.w3schools.com/python/python_json.asp - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# To work with the JSON standard: -import json - -# To work with files: -from utils_v2.system import files - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def from_string(json_data): - - """ - Decodes a JSON string to a pythonic variable like a dict. - :param json_data: The JSON string to decode. - :return: The decoded pythonic variable. - """ - - python_data = json.loads(json_data) - return python_data - - -# --------------------------------------------------------------------------------------------------------------------- - - -def to_string( - python_data, - indent = 4, - default = None, - separators = None, - no_space = False -): - - """ - Converts the given pythonic data to a JSON string. - :param python_data: The input data like a dict. - :param indent: The tab-width for pretty presentation. - :param default: The function to use on something that cannot be directly parsed into a JSON string. - :param separators: Custom separators to use. - :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not - good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT - 'default'. - :return: The JSON string representation of the input pythonic data. - """ - - if no_space: - json_data = json.dumps( - python_data, - default = default, - separators = (',', ':') - ) - - else: - json_data = json.dumps( - python_data, - indent = indent, - default = default, - separators = separators - ) - - return json_data - - -# --------------------------------------------------------------------------------------------------------------------- - - -def from_file(file): - - """ - Reads a JSON file and returns it as a pythonic variable like a dict. - :param file: The path to the file on the disk or a file held in RAM as a BytesIO object. - :return: The decoded pythonic variable. - """ - - if isinstance(file, io.BytesIO): - file.seek(0) - json_data = file.getvalue() - else: json_data = files.read_file(file) - python_data = from_string(json_data) - return python_data - - -# --------------------------------------------------------------------------------------------------------------------- - - -def to_file( - file, - python_data, - indent = 4, - default = None, - separators = None, - no_space = False -): - - """ - - :param file: Either a path to a file on disk, or a buffer in RAM in the form of a BytesIO object. - :param python_data: The pythonic data to be converted to the JSON string. - :param indent: The tab-width for pretty presentation. - :param default: The function to use on something that cannot be directly parsed into a JSON string. - :param separators: Custom separators to use. - :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not - good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT - 'default'. - :return: True/False if a path was given, else the same BytesIO object with the written JSON data. - """ - - json_data = to_string( - python_data, - indent = indent, - default = default, - separators = separators, - no_space = no_space - ) - - if isinstance(file, io.BytesIO): - file.write(json_data.encode("utf-8")) - file.seek(0) - return file - - else: - try: - files.write_file(file, json_data, mode = "w") - return True - except: return False - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/utils_v2/string/regex.py b/utils_v2/string/regex.py deleted file mode 100644 index 469a942..0000000 --- a/utils_v2/string/regex.py +++ /dev/null @@ -1,288 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Sunday, 28th Apr., 2024 - - OBJECTIVE: - - To provide a convenient way to perform RegEx operations like finding patterns and substituting them. - - REFERENCES: - - 1) https://www.w3schools.com/python/python_regex.asp - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# To work with RegEx: -import re - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# Common RegEx patterns: -REGEX_EMAIL_ID = r"[\d\w_.+]*@[\d\w_]*.[\d\w]{2,}" -REGEX_PASSWORD = r"^(?=.*[a-z])(?=.*[A-Z])(?=.*[\d])(?=.*[!@#$%^&*()_+{}\[\]:;<>,.?~\\\/-]).{8,}$" -REGEX_NAME = r"^[\d\w .\-]{1,30}$" -REGEX_USERNAME = r"^[\d\w_]{8,25}$" -REGEX_CONTACT_NUMBER = r"\+?\d{0,3}\s*\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}" -REGEX_DATE = r"\b(?:\d{4}-\d{2}-\d{2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},?\s+\d{4}|\d{1,2}\/\d{1,2}\/\d{4}|\d{1,2}-\d{1,2}-\d{2}|\d{1,2}(?:st|nd|rd|th)\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?),?\s+\d{4})\b" -REGEX_GSTIN = r"[0-9]{2}[A-Z]{5}[0-9]{4}[A-Z]{1}[1-9A-Z]{1}Z[0-9A-Z]{1}" -REGEX_PAN = r"[A-Z]{5}[0-9]{4}[A-Z]{1}" -REGEX_IPV4 = (r"[0-9]{1,3}\." * 3) + r"[0-9]{1,3}" -REGEX_IPV6 = (r"[0-9a-fA-F]{1,4}:" * 7) + r"[0-9a-fA-F]{1,4}" -REGEX_IFSC = r"[A-Z]{4}0[A-Z0-9]{6}" -REGEX_UPI = r"[a-zA-Z0-9\.\-]{2,256}@[a-zA-Z][a-zA-Z]{2,64}" -REGEX_MAC_ADDRESS = r"([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})|([0-9a-fA-F]{4}\\.[0-9a-fA-F]{4}\\.[0-9a-fA-F]{4})" -REGEX_METRIC_WEIGHT = r"[\d\.]+[ ]?[k]?g" - - -# RegEx chars (append them to the patterns if needed): -REGEX_START = "^" -REGEX_END = "$" - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def find(text, pattern, case_sensitive = True, dot_all = False): - - """ - Returns a list of substrings that match the given RegEx pattern in the input text. - :param text: The text in which the pattern needs to be found. - :param pattern: The RegEx pattern to look for. - :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. - :param dot_all: Allow all characters to be matched in ".". - :return: An array (list) of substring that match the pattern. Can be an empty list as well. - """ - - # Prepare the flags: - flags = 0 - if not case_sensitive: flags |= re.IGNORECASE - if dot_all: flags |= re.DOTALL - - # Perform the RegEx operation, and clean the results: - matches = [match if type(match) is str else match[1] for match in re.findall(pattern, text, flags = flags)] - matches = [match for match in matches if len(match) > 0] - - # Return the results: - return matches - - -# --------------------------------------------------------------------------------------------------------------------- - - -def find_first(text, pattern, case_sensitive = True, dot_all = False): - - """ - Returns the first substring that matches the given RegEx pattern in the input text. - :param text: The text in which the pattern needs to be found. - :param pattern: The RegEx pattern to look for. - :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. - :param dot_all: Allow all characters to be matched in ".". - :return: The first match as a string, or None if no match was found.. - """ - - matches = find( - text = text, - pattern = pattern, - case_sensitive = case_sensitive, - dot_all = dot_all - ) - - if not matches: return None - else: return matches[0] - - -# --------------------------------------------------------------------------------------------------------------------- - - -def replace(text, pattern, substitute_text, case_sensitive = True, dot_all = False): - - """ - Replaces any substring in the text that matches the RegEx pattern. - :param text: The text in which the substitutions need to be made. - :param pattern: The RegEx pattern that needs to be substituted. - :param substitute_text: The text that will replace the matches that were found. - :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. - :param dot_all: Allow all characters to be matched in ".". - :return: The text with the substitutions. If no matches are found, the original string is returned. - """ - - # Prepare the flags: - flags = 0 - if not case_sensitive: flags |= re.IGNORECASE - if dot_all: flags |= re.DOTALL - - # Perform the RegEx operation, and return the results: - return re.sub(pattern, substitute_text, text, flags = flags) - - -# --------------------------------------------------------------------------------------------------------------------- - - -def search(text, pattern, case_sensitive = True, dot_all = False): - - """ - Checks if the given RegEx pattern occurs ANYWHERE in the text that was provided. - :param text: The text that needs to be matched against the pattern. - :param pattern: The RegEx pattern to look for. - :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. - :param dot_all: Allow all characters to be matched in ".". - :return: True if the pattern matches, else False. - """ - - # Prepare the flags: - flags = 0 - if not case_sensitive: flags |= re.IGNORECASE - if dot_all: flags |= re.DOTALL - - # Perform the RegEx operation, and return the results: - if re.search(pattern, text, flags = flags): return True - else: return False - - -# --------------------------------------------------------------------------------------------------------------------- - - -def match(text, pattern, case_sensitive = True, dot_all = False): - - """ - Checks if the given text matches the RegEx pattern that was provided. The check is made only at the start of the - input string. - :param text: The text that needs to be matched against the pattern. - :param pattern: The RegEx pattern to look for. - :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. - :param dot_all: Allow all characters to be matched in ".". - :return: True if the pattern matches, else False. - """ - - # Prepare the flags: - flags = 0 - if not case_sensitive: flags |= re.IGNORECASE - if dot_all: flags |= re.DOTALL - - # Perform the RegEx operation, and return the results: - if re.match(pattern, text, flags = flags): return True - else: return False - - -# --------------------------------------------------------------------------------------------------------------------- - - -def split(text, pattern, case_sensitive = True, dot_all = False): - - """ - Splits an input string based on the pattern that is being matched. - :param text: The text that needs to be matched against the pattern. - :param pattern: The RegEx pattern to look for. - :param case_sensitive: Whether, or not, you want the operation to be case-sensitive. - :param dot_all: Allow all characters to be matched in ".". - :return: True if the pattern matches, else False. - """ - - # Prepare the flags: - flags = 0 - if not case_sensitive: flags |= re.IGNORECASE - if dot_all: flags |= re.DOTALL - - # Perform the RegEx operation, and return the results: - substrings = re.split(pattern, text, flags = flags) - if len(substrings) > 0 and substrings[0] == "": substrings.pop(0) - return substrings - - -# --------------------------------------------------------------------------------------------------------------------- - - -def to_json(text, pattern, case_sensitive = True, dot_all = False): - - """ - Gives out a dict from the extracted features in a string. It is based on the concept of Named Groups. - Consider the following example (assuming the search is case-insensitive): - TEXT: "UPI/309258561479/14:17:35/UPI/omsainurses@okhdfc" - PATTERN: "upi/.*/(?P