Squashed 'utils_v2/' content from commit ef9630d

git-subtree-dir: utils_v2
git-subtree-split: ef9630d728847764d4832ce8f3f956571901383a

.gitignore

@@ -0,0 +1,4 @@
+/.venv/
+/.idea/
+/__pycache__/
+*.pem

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,38 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="8">
+            <item index="0" class="java.lang.String" itemvalue="python-telegram-bot" />
+            <item index="1" class="java.lang.String" itemvalue="aiomysql" />
+            <item index="2" class="java.lang.String" itemvalue="charset-normalizer" />
+            <item index="3" class="java.lang.String" itemvalue="ultralytics-thop" />
+            <item index="4" class="java.lang.String" itemvalue="ultralytics" />
+            <item index="5" class="java.lang.String" itemvalue="kafka-python" />
+            <item index="6" class="java.lang.String" itemvalue="numpy" />
+            <item index="7" class="java.lang.String" itemvalue="aiofiles" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E701" />
+          <option value="E251" />
+          <option value="E722" />
+          <option value="E402" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="quart.app.Quart.*" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/material_theme_project_new.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MaterialThemeProjectNewConfig">
+    <option name="metadata">
+      <MTProjectMetadataState>
+        <option name="migrated" value="true" />
+        <option name="pristineConfig" value="false" />
+        <option name="userId" value="-302aa62e:18eb29d7b04:-7ffe" />
+      </MTProjectMetadataState>
+    </option>
+  </component>
+</project>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.10 (utils_v2)" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (utils_v2)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/utils_v2.iml" filepath="$PROJECT_DIR$/.idea/utils_v2.iml" />
+    </modules>
+  </component>
+</project>

.idea/utils_v2.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

ai/image_classification/async_blur.py

@@ -0,0 +1,202 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 2nd Sept., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to assess images for blurriness.
+        Tried and implemented using HuggingFace models.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# For using the AI model:
+import torch
+from transformers import pipeline
+
+# To download images from URLs:
+import requests
+
+# To read images:
+from PIL import Image
+
+# Common:
+from shared import variables
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AssessImageBlur:
+
+    def __init__(
+            self,
+            model,
+            blur_label = "Blur",
+            clarity_label = "Normal",
+            blur_threshold = 0.25,
+            clarity_threshold = 0.65
+    ):
+
+        """
+        This class assesses the input image and tells if it is blurry or clear.
+        :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is
+            stored.
+        :param blur_threshold: The max allowed blurriness (0 to 1 range).
+        :param clarity_threshold: The minimum needed clarity (0 to 1 range).
+        """
+
+        # Note down the config:
+        self.__model = model
+        self.__device = "gpu" if torch.cuda.is_available() else "cpu"
+        self.__blur_label = blur_label
+        self.__clarity_label = clarity_label
+        self.__blur_threshold = blur_threshold
+        self.__clarity_threshold = clarity_threshold
+
+        # Initialize the classifier:
+        self.__classifier = pipeline(
+            task = "image-classification",
+            model = self.__model,
+            device = self.__device
+        )
+
+    @staticmethod
+    async def read_image(source):
+
+        """
+        Reads an image in whichever format it is provided and returns it as a PIL object.
+        :param source: The image as either a path or a URL or a io.BytesIO object.
+        :return: The image opened as a PIL object.
+        """
+
+        # If the input image is already a PIL image:
+        if isinstance(source, Image.Image): return source
+
+        # If a buffer is provided:
+        elif isinstance(source, io.BytesIO):
+            source.seek(0)
+            return Image.open(source)
+
+        # If a string is provided (local path or URL):
+        elif isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://"):
+                response = await variables.http_client.get(source)
+                return Image.open(io.BytesIO(response.content))
+            else: return Image.open(source)
+
+    async def classify(self, image):
+
+        """
+        To get the prediction of the model from the given input image.
+        :param image: The image (as a PIL object or file path or a URL).
+        :return: The dictionary of classes with their respective predictions.
+        """
+
+        image = await self.read_image(image)
+        predictions = self.__classifier(image)
+        classes = {p["label"]: p["score"] for p in predictions}
+        return classes
+
+    async def is_ok(
+            self,
+            image,
+            blur_threshold = None,
+            clarity_threshold = None
+    ):
+
+        """
+        Checks if the image given to it can be used, or should be rejected.
+        :param image: The image (as a PIL object or file path or a URL).
+        :param blur_threshold: A custom threshold to test against. If not provided, the default will be taken that was
+            provided when the instance was created.
+        :param clarity_threshold: A custom threshold to test against. If not provided, the default will be taken that
+            was provided when the instance was created.
+        :return: True if the image is okay, else False.
+        """
+
+        image = await self.read_image(image)
+        blur_threshold = blur_threshold or self.__blur_threshold
+        clarity_threshold = clarity_threshold or self.__clarity_threshold
+        classes = await self.classify(image)
+        if (
+                classes[self.__blur_label] <= blur_threshold and
+                classes[self.__clarity_label] >= clarity_threshold
+        ): return True
+        return False
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+
+    async def main():
+        image = r"/home/developer/Downloads/low-res-check.png"
+        assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/image_classification/BlurOrBokeh")
+        usable = await assessor.is_ok(image)
+        classes = await assessor.classify(image)
+        print("IS OKAY:", usable)
+        print("CLASSES:", classes)
+
+    asyncio.run(main())

ai/image_classification/async_nsfw.py

@@ -0,0 +1,188 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 2nd Sept., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to assess images for adult content.
+        Tried and implemented using HuggingFace models.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# For using the AI model:
+import torch
+from transformers import pipeline
+
+# To download images from URLs:
+import requests
+
+# To read images:
+from PIL import Image
+
+# Common:
+from shared import variables
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AssessImageNSFW:
+
+    def __init__(
+            self,
+            model,
+            nsfw_label = "nsfw",
+            nsfw_threshold = 0.25,
+    ):
+
+        """
+        This class assesses the input image and tells if it is blurry or clear.
+        :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is
+            stored.
+        :param nsfw_threshold: The max allowed blurriness (0 to 1 range).
+        """
+
+        # Note down the config:
+        self.__model = model
+        self.__device = "gpu" if torch.cuda.is_available() else "cpu"
+        self.__nsfw_label = nsfw_label
+        self.__nsfw_threshold = nsfw_threshold
+
+        # Initialize the classifier:
+        self.__classifier = pipeline(
+            task = "image-classification",
+            model = self.__model,
+            device = self.__device
+        )
+
+    @staticmethod
+    async def read_image(source):
+
+        """
+        Reads an image in whichever format it is provided and returns it as a PIL object.
+        :param source: The image as either a path or a URL or a io.BytesIO object.
+        :return: The image opened as a PIL object.
+        """
+
+        # If the input image is already a PIL image:
+        if isinstance(source, Image.Image): return source
+
+        # If a buffer is provided:
+        elif isinstance(source, io.BytesIO):
+            source.seek(0)
+            return Image.open(source)
+
+        # If a string is provided (local path or URL):
+        elif isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://"):
+                response = await variables.http_client.get(source)
+                return Image.open(io.BytesIO(response.content))
+            else: return Image.open(source)
+
+    async def classify(self, image):
+
+        """
+        To get the prediction of the model from the given input image.
+        :param image: The image (as a PIL object or file path or a URL).
+        :return: The dictionary of classes with their respective predictions.
+        """
+
+        image = await self.read_image(image)
+        predictions = self.__classifier(image)
+        classes = {p["label"]: p["score"] for p in predictions}
+        return classes
+
+    async def is_ok(
+            self,
+            image,
+            nsfw_threshold = None
+    ):
+
+        """
+        Checks if the image given to it can be used, or should be rejected.
+        :param image: The image (as a PIL object or file path or a URL).
+        :param nsfw_threshold: A custom threshold to test against. If not provided, the default will be taken that was
+            provided when the instance was created.
+        :return: True if the image is okay, else False.
+        """
+
+        image = await self.read_image(image)
+        nsfw_threshold = nsfw_threshold or self.__nsfw_threshold
+        classes = await self.classify(image)
+        return True if classes[self.__nsfw_label] <= nsfw_threshold else False
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+
+    async def main():
+
+        assessor = AssessImageNSFW(model = r"/path/to/model")
+        usable = await assessor.is_ok(r"/path/to/image/img.jpg")
+        classes = await assessor.classify(r"https://...")
+
+
+    asyncio.run(main())

ai/image_segmentation/async_dichotomous_segmentation.py

@@ -0,0 +1,223 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 14th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to get masks from dichotomous image segmentation.
+
+        This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait".
+        You may experiment with other models too, but make sure that model is made for "dichotomous" behaviour. This
+        means that the model should have only two classes like "foreground", and "background". The specified model was
+        trained for implementing portrait mode style blurring of backgrounds.
+
+        The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not
+        support drop-in replacement for other models, please be aware about this.
+
+    REFERENCES:
+
+        01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait
+        02. https://github.com/ZhengPeng7/BiRefNet
+
+    DOWNLOADS:
+
+        01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait/tree/main
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For system-level activities:
+import io
+
+# To work with PIL images:
+from PIL import Image
+
+# To make asynchronous HTTP calls:
+from shared.variables import http_client
+
+# For using the AI model:
+import torch
+from torchvision import transforms
+from transformers import AutoModelForImageSegmentation
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DichotomousSegmenter:
+
+    def __init__(
+            self,
+            model = r"zhengpeng7/BiRefNet-portrait"
+    ):
+
+        """
+        This class uses Dichotomous Image Segmentation to produce a mask of what is in the foreground (or what is of
+        interest in a given image).
+        :param model: The path of the model.
+        """
+
+        # Check if GPU is available for faster predictions:
+        self.__cuda_is_available = True if torch.cuda.is_available() else False
+
+        # Initialize the model:
+        self.__model = AutoModelForImageSegmentation.from_pretrained(
+            pretrained_model_name_or_path = model,
+            trust_remote_code = True
+        )
+        torch.set_float32_matmul_precision(["high", "highest"][0])
+        self.__model.to("cuda" if self.__cuda_is_available else "cpu")
+
+    @staticmethod
+    async def read_image(source):
+
+        """
+        Reads an image in whichever format it is provided and returns it as a PIL object.
+        :param source: The image as either a path or a URL or a io.BytesIO object.
+        :return: The image opened as a PIL object.
+        """
+
+        # If the input image is already a PIL image:
+        if isinstance(source, Image.Image): return source
+
+        # If a buffer is provided:
+        elif isinstance(source, io.BytesIO):
+            source.seek(0)
+            return Image.open(source)
+
+        # If a string is provided (local path or URL):
+        elif isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://"):
+                response = await http_client.get(source)
+                return Image.open(io.BytesIO(response.content))
+            else: return Image.open(source)
+
+    async def get_mask(self, image):
+
+        """
+        Process the image to figure out the mask. In some cases (like the image of a sunset) you will have no white
+        pixels in the mask. White pixels represent the areas that the AI considered to be the foreground (area of
+        interest) and the black parts are the background.
+        :param image: The image data as either a PIL object, or a path to a file on the local disk, or a URL.
+        :return: The mask as a PIL object.
+        """
+
+        # Prepare the image transformer:
+        image_size = (1024, 1024)
+        transform_image = transforms.Compose([
+            transforms.Resize(image_size),
+            transforms.ToTensor(),
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+        ])
+
+        # Read and transform the image:
+        image = await self.read_image(image)
+        input_image = transform_image(image).unsqueeze(0).to("cuda" if self.__cuda_is_available else "cpu")
+
+        # Make the prediction:
+        with torch.no_grad():
+            if self.__cuda_is_available: mask = self.__model(input_image)[-1].sigmoid().cuda()
+            else: mask = self.__model(input_image)[-1].sigmoid().cpu()
+
+        # Create a PIL object of the mask data:
+        mask = mask[0].squeeze()
+        mask = transforms.ToPILImage()(mask)
+        mask = mask.resize(image.size)
+
+        # Done here:
+        return mask
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    import numpy as np
+
+    async def main():
+
+        model = DichotomousSegmenter()
+
+        images = [
+            # r"/home/developer/Downloads/kate.jpg",
+            # r"/home/developer/Downloads/IMG-20240730-WA0001.jpg",
+            # r"/home/developer/Downloads/pexels-pixabay-57416.jpg",
+            # r"/home/developer/Downloads/summer_clothes.jpg",
+            # r"/home/developer/Downloads/low_res_cat - upscaled.png",
+            # r"/home/developer/Downloads/blurry_traffic.jpg",
+            r"/home/developer/Downloads/card_0.jpg",
+            r"/home/developer/Downloads/card_1.jpg",
+            r"/home/developer/Downloads/card_2.jpg",
+            r"/home/developer/Downloads/card_3.jpg",
+            r"/home/developer/Downloads/sushmita_card.jpg",
+            r"/home/developer/Downloads/niranjan_card.jpg",
+            r"/home/developer/Downloads/niranjan_card_2.jpg",
+        ]
+
+        for image in images:
+            print(image.split("/")[-1])
+            image = await DichotomousSegmenter.read_image(image)
+            mask = await model.get_mask(image)
+            image.show()
+            mask.show()
+            print("MASK!")
+            array = np.array(mask)
+            print("Dimensions of the array:", array.shape)
+            masked_count = np.sum(array > 125)
+            print(masked_count, type(masked_count), np.size, type(np.size))
+            percentage = masked_count / np.prod(array.shape)
+            print("Area:", round(percentage * 100, 4))
+            # image.putalpha(mask)
+            # image.show()
+            # mask.show()
+
+
+    asyncio.run(main())

ai/object_detection/yolo.py

@@ -0,0 +1,176 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 4th Jul, 2024
+
+    OBJECTIVE:
+
+        To provide a class to detect objects in images using YOLO models.
+
+    REFERENCES:
+
+        1) Code Examples: https://docs.ultralytics.com/usage/python/
+        2) Models: https://docs.ultralytics.com/models/yolov8/#supported-tasks-and-modes
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import os
+
+# To use YOLO architecture:
+from ultralytics import YOLO
+
+# To work with images:
+from PIL import Image
+
+# utils:
+from utils_v2.string import json
+
+# Common:
+from shared import constants
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class YoloDetect:
+
+    def __init__(
+            self,
+            model_file,
+            debug = True,
+            debug_prefix = "YOLO | "
+    ):
+
+        # Load the model:
+        self._model = YOLO(model = model_file)
+
+        # Initialize the dbugging tool:
+        self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self._printer.disable()
+
+    def disable_debug(self):
+        self._printer.disable()
+
+    def enable_debug(self):
+        self._printer.enable()
+
+    @staticmethod
+    def _process_result(raw_result):
+
+        # Extract the various types of results available from the inference:
+        speed = {k: (v / 100.0) for k, v in raw_result.speed.items()}
+        class_mapping = raw_result.names
+        probabilities = raw_result.probs
+        boxes = raw_result.boxes
+        masks = raw_result.masks
+
+        # Construct a default response:
+        response_json = {
+            "speed": speed,
+            "classes": class_mapping,
+            "boxes": None
+        }
+
+        # Object-Detection results:
+        formatted_boxes = []
+        for box in boxes:
+            detected_class = int(box.cls.numpy()[0])
+            x1, y1, x2, y2 = box.xyxy[0]
+            formatted_boxes.append(
+                {
+                    "class": detected_class,
+                    "className": class_mapping[detected_class],
+                    "confidence": float(box.conf.numpy()[0]),
+                    "x1": int(x1.numpy()),
+                    "y1": int(y1.numpy()),
+                    "x2": int(x2.numpy()),
+                    "y2": int(y2.numpy())
+                }
+            )
+        response_json["boxes"] = formatted_boxes
+
+        # Done here:
+        return response_json
+
+    def predict(self, image, show = False, verbose = False):
+
+        # Process the input image with the given task:
+        results = self._model(
+            source = image,
+            show = show,
+            verbose = verbose
+        )
+
+        # Based on the task, interpret the results:
+        results_json = self._process_result(results[0])
+
+        # Done here:
+        return results_json
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "models", "yolov8x.pt")
+    model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt"
+    # sample_image_path = r"/home/developer/Downloads/2_cats.jpg"
+    sample_image_path = r"/home/developer/Downloads/flattened_image.jpg"
+    my_yolo = YoloDetect(model_file = model_file_path)
+    results = my_yolo.predict(image = Image.open(sample_image_path))
+    print("FINAL RESULTS:", json.to_string(results))
+

ai/tts/easy_tts.py

@@ -0,0 +1,175 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 26th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a fast way to make audio from TTS engines.
+
+    REFERENCES:
+
+        01. Usage: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#python-api
+        02. Installation: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#linux-and-macos-install
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To use the AI:
+from melo.api import TTS
+import numpy as np
+import soundfile
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class EasyTTS:
+
+    __sampling_rate = 44_100
+
+    def __init__(
+            self,
+            language = "EN",
+            speed = 1.0
+    ):
+
+        self.__audio = np.zeros(1)
+        self.__language = language
+        self.__speed = speed
+        self.__model = TTS(language = language, device = "auto")
+        self.__speakers = self.__model.hps.data.spk2id
+
+    def list_speakers(self):
+        return list(self.__speakers.keys())
+
+    def speak(
+            self,
+            text,
+            speaker
+    ):
+
+        this_audio = self.__model.tts_to_file(
+            text,
+            self.__speakers[speaker],
+            speed = self.__speed,
+            quiet = True
+        )
+
+        self.__audio = np.concatenate((self.__audio, this_audio))
+
+    def pause(self, seconds):
+
+        self.__audio = np.concatenate((
+            self.__audio,
+            np.zeros(int(self.__sampling_rate * seconds))
+        ))
+
+    def to_wav(self, path = None):
+
+        if path is None:
+            audio_buffer = io.BytesIO()
+            soundfile.write(audio_buffer, self.__audio, self.__sampling_rate)
+            return audio_buffer
+
+        else: soundfile.write(path, self.__audio, self.__sampling_rate)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    speaker = "EN-BR"
+    tts_maker = EasyTTS(language = "EN", speed = 0.9)
+
+    tts_maker.speak(
+        text = """
+            Imagine delighting your doctors with a personalized calendar,  
+            crafted from their own cherished memories.
+            """,
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = """
+            Every day, as they turn the page,  
+            they’ll not only relive those special moments but also remember you, 
+            the one who made it happen.
+            """,
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.75)
+    tts_maker.speak(
+        text = "STEP 1:",
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = "Start by identifying the doctors you’d like to engage with, and add them to our app.",
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = "No rush, you can add their photographs later as well.",
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = "With this, your engagement funnel is created.",
+        speaker = speaker
+    )
+
+    tts_maker.to_wav(r"/home/developer/Downloads/voiceover.wav")

api/codes.py

@@ -0,0 +1,164 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 12th Sept., 2024
+
+    OBJECTIVE:
+
+        To maintain all status codes in one place.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+from enum import Enum, unique
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+@unique
+class HttpCodes(Enum):
+
+    """
+    Commonly used standard HTTP status codes.
+    Can be sent after an API Call is processed.
+    Refer to: https://http.dev/status
+    NOTE: THIS LIST IS NOT EXHAUSTIVE!
+    """
+
+    # 1XX - Informational:
+    CONTINUE = 100  # .............. https://http.dev/102
+    SWITCHING_PROTOCOLS = 101  # ... https://http.dev/101
+    PROCESSING = 102  # ............ https://http.dev/102
+    EARLY_HINTS = 103  # ........... https://http.dev/103
+
+    # 2XX - Success:
+    SUCCESS = 200  # .................. https://http.dev/200
+    CREATED = 201  # .................. https://http.dev/201
+    ACCEPTED = 202  # ................. https://http.dev/202
+    NON_AUTHORITATIVE_INFO = 203  # ... https://http.dev/203
+    NO_CONTENT = 204  # ............... https://http.dev/204
+    RESET_CONTENT = 205  # ............ https://http.dev/205
+    PARTIAL_CONTENT = 206  # .......... https://http.dev/206
+    MULTI_STATUS = 207  # ............. https://http.dev/207
+    ALREADY_REPORTED = 208  # ......... https://http.dev/208
+    THIS_IS_FINE = 218  # ............. https://http.dev/218
+    IM_USED = 226  # .................. https://http.dev/226
+
+    # 3XX - Redirection:
+    MULTIPLE_CHOICES = 300  # ..... https://http.dev/300
+    MOVED_PERMANENTLY = 301  # .... https://http.dev/301
+    MOVED_TEMPORARILY = 302  # .... https://http.dev/302
+    SEE_OTHER = 303  # ............ https://http.dev/303
+    NOT_MODIFIED = 304  # ......... https://http.dev/304
+    USE_PROXY = 305  # ............ https://http.dev/305
+    SWITCH_PROXY = 306  # ......... https://http.dev/306
+    TEMPORARY_REDIRECT = 307  # ... https://http.dev/307
+    PERMANENT_REDIRECT = 308  # ... https://http.dev/308
+
+    # 4XX - Client Errors:
+    BAD_REQUEST = 400  # ..................... https://http.dev/401
+    UNAUTHORIZED = 401  # .................... https://http.dev/401
+    PAYMENT_REQUIRED = 402  # ................ https://http.dev/402
+    FORBIDDEN = 403  # ....................... https://http.dev/403
+    NOT_FOUND = 404  # ....................... https://http.dev/404
+    METHOD_NOT_ALLOWED = 405  # .............. https://http.dev/405
+    NOT_ACCEPTABLE = 406  # .................. https://http.dev/406
+    PROXY_AUTH_REQUIRED = 407  # ............. https://http.dev/407
+    REQUEST_TIMEOUT = 408  # ................. https://http.dev/408
+    CONFLICT = 409  # ........................ https://http.dev/409
+    GONE = 410  # ............................ https://http.dev/410
+    LENGTH_REQUIRED = 411  # ................. https://http.dev/411
+    PRECONDITION_FAILED = 412  # ............. https://http.dev/412
+    PAYLOAD_TOO_LARGE = 413  # ............... https://http.dev/413
+    URI_TOO_LONG = 414  # .................... https://http.dev/414
+    UNSUPPORTED_MEDIA_TYPE = 415  # .......... https://http.dev/415
+    PAGE_EXPIRED = 419  # .................... https://http.dev/419
+    TOO_MANY_REQUESTS = 429  # ............... https://http.dev/429
+    UNAVAILABLE_FOR_LEGAL_REASONS = 451  # ... https://http.dev/451
+    INVALID_TOKEN = 498  # ................... https://http.dev/498
+    CLIENT_CLOSED_REQUEST = 499  # ........... https://http.dev/499
+
+    # 5XX - Server Errors:
+    INTERNAL_SERVER_ERROR = 500  # ........... https://http.dev/500
+    NOT_IMPLEMENTED = 501  # ................. https://http.dev/501
+    BAD_GATEWAY = 502  # ..................... https://http.dev/502
+    SERVICE_UNAVAILABLE = 503  # ............. https://http.dev/503
+    GATEWAY_TIMEOUT = 504  # ................. https://http.dev/504
+    HTTP_VERSION_NOT_SUPPORTED = 505  # ...... https://http.dev/505
+    VARIANT_ALSO_NEGOTIATES = 506  # ......... https://http.dev/506
+    INSUFFICIENT_STORAGE = 507  # ............ https://http.dev/507
+    LOOP_DETECTED = 508  # ................... https://http.dev/508
+    BANDWIDTH_LIMIT_EXCEEDED = 509  # ........ https://http.dev/509
+    WEB_SERVER_DOWN = 521  # ................. https://http.dev/521
+    ORIGIN_IS_UNREACHABLE = 523  # ........... https://http.dev/523
+    SERVICE_IS_OVERLOADED = 529  # ........... https://http.dev/529
+    NETWORK_READ_TIMEOUT_ERROR = 598  # ...... https://http.dev/598
+    NETWORK_CONNECT_TIMEOUT_ERROR = 599  # ... https://http.dev/599
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+@unique
+class StatusCodes(Enum):
+
+    """
+    To be used internally within the context of your service. Customize these to match your service.
+    The format is: (SUCCESS_INDICATOR, INTERNAL_NUMERIC_CODE, HTTP_CODE)
+    Example: (True, 1, 200)
+    """
+
+    # Legacy Codes:
+    OK = (True, 1, HttpCodes.SUCCESS.value)
+    FAILED = (False, 0, HttpCodes.INTERNAL_SERVER_ERROR.value)
+    PARTIAL_SUCCESS = (True, 2, HttpCodes.PARTIAL_CONTENT.value)
+    PARTIAL_FAILURE = (False, 3, HttpCodes.PARTIAL_CONTENT.value)
+
+    # Authentication Codes:
+    LOGGED_IN_SUCCESSFULLY = (True, 200, HttpCodes.SUCCESS.value)
+    LOGIN_FAILED = (False, 201, HttpCodes.UNAUTHORIZED.value)
+    INVALID_SESSION_TOKEN = (False, 202, HttpCodes.UNAUTHORIZED.value)
+    AUTHENTICATION_DETAILS_INCOMPLETE = (False, 203, HttpCodes.BAD_REQUEST.value)
+
+    # Authorization Codes:
+    AUTHORIZED_SUCCESSFULLY = (True, 300, HttpCodes.SUCCESS.value)
+    NOT_ALLOWED = (False, 301, HttpCodes.FORBIDDEN.value)
+    AUTHORIZATION_DETAILS_INCOMPLETE = (False, 302, HttpCodes.BAD_REQUEST.value)
+    AUTHORIZATION_FAILED = (False, 303, HttpCodes.UNAUTHORIZED.value)
+
+    # General failures:
+    DOWN_FOR_MAINTENANCE = (False, 800, HttpCodes.SERVICE_UNAVAILABLE.value)
+    UNKNOWN_ERROR = (False, 801, HttpCodes.INTERNAL_SERVER_ERROR.value)
+    DATA_INCOMPLETE = (False, 802, HttpCodes.BAD_REQUEST.value)
+    HEADERS_INCOMPLETE = (False, 803, HttpCodes.BAD_REQUEST.value)
+    FILES_MISSING = (False, 804, HttpCodes.BAD_REQUEST.value)
+    CLIENT_CLOSED_REQUEST = (False, 805, HttpCodes.CLIENT_CLOSED_REQUEST.value)
+
+    # Validation failure:
+    DATA_VALIDATION_FAILURE = (False, 900, HttpCodes.BAD_REQUEST.value)
+    RATE_LIMIT_EXCEEDED = (False, 901, HttpCodes.TOO_MANY_REQUESTS.value)

api/metrics_prometheus.py

@@ -0,0 +1,198 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+        Sharvil J Daiya
+
+    DATE:
+
+        Saturday, 28th Sept., 2024
+
+    OBJECTIVE:
+
+        To have one place from where several metrics are measured using easy to use context managers.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To measure time:
+import time
+
+# To capture metrics for Prometheus:
+from prometheus_client import Counter, Summary, Gauge
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Define the Prometheus metrics.
+# FOR THE MICROSERVICE AS A WHOLE:
+WORKER_COUNT = Counter(
+    name = "ms_workers_active_total",
+    documentation = "The number of threads for the microservice being monitored.",
+    labelnames = ["project_name", "service_name", "host_name"]
+)
+
+# Define the Prometheus metrics.
+# FOR INDIVIDUAL API ENDPOINTS:
+REQUEST_LATENCY = Summary(
+    name = "http_request_latency_seconds",
+    documentation = "Latency of HTTP requests in seconds.",
+    labelnames = ["method", "endpoint", "http_status"]
+)
+TOTAL_REQUEST_COUNT = Counter(
+    name = "http_requests_total",
+    documentation = "Total HTTP requests.",
+    labelnames = ["method", "endpoint", "http_status"]
+)
+LIVE_REQUEST_COUNT = Gauge(
+    name = "http_requests_live_total",
+    documentation = "To check if an API endpoint is being served right now.",
+    labelnames = ["endpoint"]
+)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class MetricsAPI:
+
+    """
+    Use this class through its context manager to automatically measure all the metrics in one place.
+    This was originally created to measure the performance of API endpoints made in Quart, but it should work with
+    other frameworks as well.
+    """
+
+    def __init__(
+            self,
+            method = None,
+            endpoint = None,
+            raise_exception = False
+    ):
+
+        # Make provisions for things to note.
+        # NOTE: THESE MUST BE SET FROM OUTSIDE:
+        self.method = method
+        self.endpoint = endpoint
+        self.http_code = None
+        self.__raise_exception = raise_exception
+
+    async def __aenter__(self):
+
+        # Note down the start time immediately:
+        self.__start_ts = time.perf_counter()
+        self.__cpu_start_ts = time.process_time()
+
+        # Note down the metrics:
+        LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1)
+
+        # Setup done:
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+
+        # Note down the metrics:
+        LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1)
+
+        TOTAL_REQUEST_COUNT.labels(
+            self.method,
+            self.endpoint,
+            self.http_code
+        ).inc()
+
+        REQUEST_LATENCY.labels(
+            self.method,
+            self.endpoint,
+            self.http_code
+        ).observe(time.perf_counter() - self.__start_ts)
+
+        # Handle the exception as per the user's preference:
+        return False if self.__raise_exception else True
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    import random
+    from prometheus_client import generate_latest
+
+    async def simulate_endpoint():
+
+        async with MetricsAPI(
+                method = random.choice(["GET", "POST"]),
+                endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}"
+        ) as metrics:
+
+            # Simulate some action on some endpoint:
+            await asyncio.sleep(1.0)
+
+            # Note down the values:
+            metrics.http_code = 200
+
+    async def main():
+
+        print("Simulating endpoints...")
+        tasks = [simulate_endpoint() for _ in range(250)]
+        await asyncio.gather(*tasks)
+        print("Done!")
+
+        print("METRICS:")
+        print(generate_latest().decode())
+
+    asyncio.run(main())

api/response.py

@@ -0,0 +1,121 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 12th Sept., 2024
+
+    OBJECTIVE:
+
+        To have a structure to the response sent from the API calls.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+from pydantic import BaseModel
+from typing import Any, Optional, List
+
+from utils_v2.api.codes import StatusCodes, HttpCodes
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class ResponseModel(BaseModel):
+
+    status_code: StatusCodes
+    message: Optional[str | List] = None
+    data: Optional[Any] = None
+    seconds: Optional[float | int] = None
+    log_id: Optional[str] = None
+    http_code: Optional[HttpCodes] = None
+    api_version: Optional[str] = None
+
+    def for_quart(self):
+
+        # Construct the basic structure:
+        response_dict = {
+            "status": 1 if self.status_code.value[0] else 0,
+            "code": self.status_code.value[1],
+            "message": self.message or self.status_code.name.replace("_", " ").lower(),
+            "data": self.data,
+            "apiVer": self.api_version
+        }
+
+        # Now add the additional fields:
+        if self.seconds is not None: response_dict["seconds"] = self.seconds
+        if self.log_id is not None: response_dict["logId"] = self.log_id
+
+        # Figure out the HTTP code:
+        response_http_code = self.http_code.value if self.http_code is not None else self.status_code.value[2]
+
+        # Done here:
+        return response_dict, response_http_code
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    my_response = ResponseModel(
+        status_code = StatusCodes.RATE_LIMIT_EXCEEDED
+    )
+    my_response.log_id = "abc123"
+
+    print(my_response.for_quart())

cache/async_redis_cache.py

@@ -0,0 +1,451 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 11th April, 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to work with '.json' files in an async manner.
+
+    REFERENCES:
+
+        1) https://www.w3schools.com/python/python_json.asp
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To use redis:
+import redis.asyncio as redis
+
+# Other utils:
+from utils_v2.string import json
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+# To make a decorator:
+from functools import wraps
+
+# For hashing and shortening the hash:
+import hashlib
+import base64
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   WRAPPERS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def cache_it(cache = None, expiry = 120):
+
+    """
+    This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
+    If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
+    cache instead of going through the whole function again.
+    :param cache: The instance of 'AsyncRedisCache'.
+    :param expiry: The time in seconds after which the cached data must be cleared.
+    :return: The decorator that automatically caches your data.
+    """
+
+    def decorator(func):
+
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+
+            # We first use the name of the function and the inputs given to it to generate a key for Redis with the
+            # simple hashing and shortening by way of base64 strings:
+            inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
+            sha256_hash = hashlib.sha256()
+            sha256_hash.update(inputs_given.encode("utf-8"))
+            hashed_key = sha256_hash.digest()
+            base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+            # Now we check if we have the value in cache:
+            try: response = await cache.get(base64_key, raise_exception = True)
+
+            # If the key doesn't exist, we pass through the function and store the results.
+            except:
+                response = await func(*args, **kwargs)
+                await cache.set(key = base64_key, value = response, expiry = expiry)
+
+            # Return the response from the wrapped function.
+            return response
+
+        return wrapper
+
+    return decorator
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncRedisCache:
+
+    # Handle datatypes when setting values to cache:
+    __set_converters = {
+        "set": lambda x: list(x),
+        "tuple": lambda x: list(x),
+        "complex": lambda x: {"r": x.real, "i": x.imag}
+    }
+
+    # Handle datatypes when getting from cache:
+    __get_converters = {
+        "set": lambda x: set(x),
+        "tuple": lambda x: tuple(x),
+        "complex": lambda x: complex(x["r"], x["i"])
+    }
+
+    def __init__(
+            self,
+            connection_string,
+            ping_counter = 1_000,
+            debug = False,
+            debug_prefix = "R-Cache | "
+    ):
+
+        """
+        Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools,
+        strings, dicts, lists, sets, and tuples :)
+        :param connection_string: The connection URL for connecting to Redis.
+        :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection.
+        :param debug: Whether, or not, you want to show debugging messages from the start.
+        :param debug_prefix: The prefix text to show with the debugging messages.
+        """
+
+        # Note down the configuration:
+        self.__client = None
+        self.__ping_counter = ping_counter
+        self.__requests_since_last_ping = 0
+        self.__connection_string = connection_string
+
+        # For debugging:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    @staticmethod
+    def make_key(*args, **kwargs):
+
+        """
+        Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the
+        inputs stay the same, the output stays the same.
+        :param args: Any number of args that you would like to use to generate the key.
+        :param kwargs: Any number of kwargs that you would like to use to generate the key.
+        :return: A string that can be used as a key to store values on Redis.
+        """
+
+        # Take everything into one plain text string:
+        plain_text = "".join(str(a) for a in args)
+        plain_text += json.to_string(kwargs, no_space = True)
+
+        # Hash the plain text value, and create a key from it:
+        sha256_hash = hashlib.sha256()
+        sha256_hash.update(plain_text.encode("utf-8"))
+        hashed_key = sha256_hash.digest()
+        base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+        # Done here:
+        return base64_key
+
+    def add_converter(
+            self,
+            type_name,
+            set_converter_func,
+            get_converter_func,
+    ):
+
+        """
+        To add custom data converters to handle new values.
+        RULE: Each of the converter functions must take in exactly on argument.
+        :param type_name: The name of the type of the object to handle. HINT: type(obj).__name__.
+        :param set_converter_func: A converter function that converts from the custom datatype to a datatype that Redis
+            can work with. Try converting to an bool, int, float, str, list or dict.
+        :param get_converter_func: The inverse of the set converter. This function will be used to convert from a
+            datatype that Redis can work with to the custom datatype.
+        :return: None.
+        """
+
+        self.__set_converters[type_name] = lambda x: set_converter_func(x)
+        self.__get_converters[type_name] = lambda x: get_converter_func(x)
+
+    async def connect(self):
+
+        """
+        To make an asynchronous connection request to the Redis server to establish a connection.
+        :return: True if connected, else False.
+        """
+
+        try:
+            self.__client = redis.from_url(
+                self.__connection_string
+            )
+            return True
+
+        except Exception as exception:
+            self.__printer(exception)
+            return False
+
+    async def disconnect(self):
+
+        """
+        Close the connection to the Redis server.
+        :return: True if successful, else False.
+        """
+
+        if self.__client is not None:
+            try: await self.__client.close()
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+            return True
+        return True
+
+    async def ensure_connection(self):
+
+        """
+        To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected.
+        :return: True if connected, else False.
+        """
+
+        # If we are not connected, we try to establish a connection:
+        if self.__client is None: return await self.connect()
+
+        # Else we check if we are connected. If not, we try to connect.
+        # But we check only once in a while. In the meantime, we assume that we are connected.
+        elif self.__requests_since_last_ping > self.__ping_counter:
+            try:
+                await self.__client.ping()
+                self.__requests_since_last_ping = 0
+                return True
+            except Exception as exception:
+                self.__printer(exception)
+                return await self.connect()
+        else:
+            self.__requests_since_last_ping += 1
+            return True
+
+    async def set(self, key, value, expiry: float = 120, raise_exception = False):
+
+        """
+        Saves some value to the cache.
+        :param key: The key with which the data will be stored and retrieved.
+        :param value: The value to store.
+        :param expiry: The time in seconds after which the data will expire. Must be a positive number.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: True if cached, else False.
+        """
+
+        await self.ensure_connection()
+
+        try:
+
+            # We handle conversion for unsupported Pythonic datatypes:
+            original_type = type(value).__name__
+            converter = self.__set_converters.get(original_type)
+            if converter is not None: value = converter(value)
+
+            # Here we actually try to store the data:
+            response = await self.__client.setex(
+                key,
+                int(expiry),
+                json.to_string({
+                    "value": value,
+                    "type": original_type
+                }, indent = 0, no_space = False)
+            )
+            return response
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return False
+
+    async def get(self, key, raise_exception = False, on_fail = None):
+
+        """
+        Retrieve the cached value.
+        :param key: The key with which the data was saved.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :param on_fail: What to return if the process fails due to an exception.
+        :return: The retrieved data or null if not found.
+        """
+
+        await self.ensure_connection()
+
+        try:
+            data = await self.__client.get(key)
+            data = data.decode("utf-8")
+            data = json.from_string(data)
+            converter = self.__get_converters.get(data["type"])
+            if converter is not None: return converter(data["value"])
+            return data["value"]
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return on_fail
+
+    async def delete(self, key, raise_exception = False):
+
+        """
+        Prematurely delete the value from the cache before it expires.
+        :param key: The key with which the data was saved.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: True if deleted, else False.
+        """
+
+        await self.ensure_connection()
+
+        try:
+            response = await self.__client.delete(key)
+            return True if response else False
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return False
+
+    async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False):
+
+        """
+        To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the
+        counter will be created and the value will be incremented from 0.
+        :param key: The name of the counter.
+        :param value: The amount to increment the value by. Send negative values to count backwards.
+        :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created.
+            This value has to be an integer. If a float is passed, the value will be rounded off.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed.
+        """
+
+        await self.ensure_connection()
+
+        try:
+
+            already_existed = await self.__client.exists(key)
+            new_value = await self.__client.incrby(key, value)
+            if expiry and not already_existed: await self.__client.expire(key, int(expiry))
+            return new_value
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+    def set_complex(x):
+        return {"r": x.real, "i": x.imag}
+
+    def get_complex(x):
+        return complex(x["r"], x["i"])
+
+    async def main():
+
+        my_cache = AsyncRedisCache(
+            connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0",
+            ping_counter = 100,
+            debug = False
+        )
+
+        my_cache.add_converter(
+            type_name = type(2j).__name__,
+            set_converter_func = set_complex,
+            get_converter_func = get_complex
+        )
+
+        value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624")
+        print("GET:", value)
+        print("TYP:", type(value), end = "\n\n")
+
+        # success = await my_cache.set(
+        #     key = "name",
+        #     value = {"first": "John", "last": "Doe"},
+        #     expiry = 10
+        # )
+        # print("SET:", success, end = "\n\n")
+        #
+        # value = await my_cache.get(key = "name")
+        # print("GET:", value)
+        # print("TYP:", type(value), end = "\n\n")
+        #
+        # success = await my_cache.delete(key = "name")
+        # print("DEL:", success, end = "\n\n")
+        #
+        # value = await my_cache.get(key = "cnt")
+        # print("GET:", value)
+        # print("TYP:", type(value), end = "\n\n")
+        #
+        # await my_cache.delete(key = "cnt")
+        # for _ in range(50):
+        #     await asyncio.sleep(1.0)
+        #     counter = await my_cache.count(key = "cnt", value = 1, expiry = 10)
+        #     print("COUNTER:", counter)
+
+    asyncio.run(main())

database/async_firebase.py

@@ -0,0 +1,504 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+        Bhushan Thakkar
+
+    DATE:
+
+        Friday, 19th April, 2024
+
+    OBJECTIVE:
+
+        To provide an easy interface to work with Firebase.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+# ---
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For async operations:
+# ---
+import asyncio
+
+# For system-level activities:
+# ---
+import os
+
+# My async utils:
+# ---
+import async_json_utils
+
+# Firebase:
+# ---
+import firebase_admin
+import firebase_admin.firestore_async
+import firebase_admin.auth
+import firebase_admin.db
+
+# For debugging and logging:
+# ---
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncFirebase:
+
+    def __init__(
+            self,
+            credentials_json_path,
+            app_name,
+            max_connections = 5,
+            debug = True
+    ):
+
+        # Debugging print:
+        # ---
+        self.__printer = IceCreamDebugger(prefix = f"FBase ({app_name[:8]}) | ", includeContext = True)
+        if not debug: self.__printer.disable()
+        alert = f"Firebase session starting with max. {max_connections} connections."
+        self.__printer(alert)
+
+        # Initialize the instance:
+        # ---
+        self.__semaphore = asyncio.Semaphore(max_connections)
+        credentials = firebase_admin.credentials.Certificate(credentials_json_path)
+        self.__firebase_app = firebase_admin.initialize_app(credentials, name = app_name)
+        self.__firestore = firebase_admin.firestore_async.client(self.__firebase_app)
+
+    def __del__(self):
+        firebase_admin.delete_app(self.__firebase_app)
+        self.__firestore.close()
+        alert = "Firebase session ended."
+        self.__printer(alert)
+
+    def __user_to_json(self, firebase_user):
+
+        user_json = {
+            "uid": firebase_user.uid,
+            "email": firebase_user.email,
+            "emailVerified": firebase_user.email_verified,
+            "displayName": firebase_user.display_name,
+            "phoneNo": firebase_user.phone_number,
+            "photoUrl": firebase_user.photo_url,
+            "customClaims": firebase_user.custom_claims,
+            "disabled": firebase_user.disabled,
+            "providerId": firebase_user.provider_id,
+            "providerData": firebase_user.provider_data,
+            "tenantId": firebase_user.tenant_id
+        }
+
+        return user_json
+
+    async def create_custom_token(self, uid):
+
+        async with self.__semaphore:
+
+            try:
+                custom_token = firebase_admin.auth.create_custom_token(
+                    uid = uid,
+                    app = self.__firebase_app
+                ).decode("utf-8")
+                return custom_token
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def authenticate_token(self, token):
+
+        """
+        To authenticate the given session token.
+        :param token: The session token generated by Firebase on a successful sign-in.
+        :return: Either the retrieved user information or a blank dictionary.
+        """
+
+        try:
+            user_info = firebase_admin.auth.verify_id_token(token, app = self.__firebase_app)
+            return user_info
+        except Exception as exception:
+            self.__printer(exception)
+            return {}
+
+    async def create_user(
+            self,
+            uid,
+            display_name = None,
+            password = None,
+            email = None,
+            phone_number = None
+    ):
+
+        """
+        Create a new user.
+        :param uid: The id to identify the user by.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                firebase_admin.auth.create_user(
+                    uid = uid,
+                    app = self.__firebase_app
+                )
+                return True
+
+            except firebase_admin.auth.UidAlreadyExistsError as excp:
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def delete_user(self, uid):
+
+        async with self.__semaphore:
+
+            try:
+                firebase_admin.auth.delete_user(
+                    uid = uid,
+                    app = self.__firebase_app
+                )
+                return True
+
+            except firebase_admin.auth.UserNotFoundError as excp:
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def get_user(self, uid):
+
+        async with self.__semaphore:
+
+            try:
+                user = firebase_admin.auth.get_user(
+                    uid = uid,
+                    app = self.__firebase_app
+                )
+                return self.__user_to_json(user)
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def list_users(self, users_per_page, page_token = None):
+
+        """
+        To get a list of users. Firebase has a limit of 1000 per call of this API. So we use page-tokens to fetch next
+        pages of users.
+        :param users_per_page: How many users you want to list in this API call (Max. 1,000).
+        :param page_token: To be used in case of pagination.
+        :return: The list of users and the page token to be used for the next call. Will be None in case of failure.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                users = firebase_admin.auth.list_users(
+                    page_token = page_token,
+                    max_results = min(users_per_page, 1000),
+                    app = self.__firebase_app
+                )
+                next_page_token = users.next_page_token if users.has_next_page else None
+                users = users.users
+                users_json = [self.__user_to_json(user) for user in users]
+                return users_json, next_page_token
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None, None
+
+    async def create_document(self, collection_path, document_name, document_data = None):
+
+        """
+        To create a new document with the specified data in an existing collection.
+        :param collection_path: The path of the collection (can be a sub-collection).
+        :param document_name: The name of the document you want to create.
+        :param document_data: The data that you want to populate in the document.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.collection(
+                    collection_path
+                ).add(
+                    document_id = document_name,
+                    document_data = document_data or {}
+                )
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def delete_document(self, path):
+
+        """
+        To delete a document.
+        :param path: The path of the document.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).delete()
+                return True
+
+            except Exception as excp:
+                print("FIRESTORE DOCUMENT DELETION EXCEPTION:", excp)
+                return False
+
+    async def set_document(self, path, data):
+
+        """
+        To overwrite the data in a document.
+        :param path: The path of the document.
+        :param data: The data you want to update as a dictionary.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).set(data)
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def update_document(self, path, data):
+
+        """
+        To update the data in a document. Provide only the fields that you want to update.
+        :param path: The path of the document.
+        :param data: The data you want to update as a dictionary.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).update(data)
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def get_document(self, path):
+
+        """
+        To get the data in a document.
+        :param path: The path of the document.
+        :return: The dictionary of data as found in the path specified or None if the operation failed.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).get()
+                return snapshot.to_dict()
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def get_document_fields(self, path, fields):
+
+        """
+        To get only specific fields (keys) in a document. Like how projections are used in Mongo.
+        :param path: The path of the document.
+        :param fields: The list of fields (keys) of the document that you want.
+        :return: The dictionary of data as found in the path specified.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                if type(fields) is not list: fields = [fields]
+                snapshot = await self.__firestore.document(path).get(fields)
+                return snapshot.to_dict()
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def create_collection(self, collection_name, document_name, document_data):
+
+        """
+        Creates a new collection in the root of the database. Note that Firebase doesn't allow creating new empty
+        collections, so we must add one first document in it.
+        :param collection_name: The name of the collection you want to create.
+        :param document_name: The name of the first document you want to put in the collection.
+        :param document_data: The data that you want to put in the first document of the new collection.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.collection(
+                    collection_name
+                ).add(
+                    document_id = document_name,
+                    document_data = document_data
+                )
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def create_sub_collection(
+            self,
+            document_path,
+            sub_collection_name,
+            sub_document_name = None,
+            sub_document_data = None
+    ):
+
+        """
+        Firebase allows you to create collections inside documents. This method is built for that. Note that Firebase
+        doesn't allow creating new empty collections, so we must add one first document in it.
+        :param document_path: The path of the document in which you want to create a new collection.
+        :param sub_collection_name: The name of the collection you want to create.
+        :param sub_document_name: The name of the first document you want to put in the collection.
+        :param sub_document_data: The data that you want to put in the first document of the new collection.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(
+                    document_path
+                ).collection(
+                    sub_collection_name
+                ).add(
+                    document_id = sub_document_name,
+                    document_data = sub_document_data or {}
+                )
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def get_collection(self, path):
+
+        """
+        Get a whole collection's data.
+        :param path: The path of the collection.
+        :return: The collection.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshots = await self.__firestore.collection(path).get()
+                return {snapshot.id: snapshot.to_dict() for snapshot in snapshots}
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import time
+    from shared import constants
+
+    user_data = {
+        "firebaseUid": None,
+        "roles": "ClientAdmin",
+        "sessionToken": "siddhesh_20240419"
+    }
+
+    async def test():
+        my_firebase = AsyncFirebase(
+            f"{constants.PROJECT_DIRECTORY}/utils/cred/firebase_certs.json",
+            "myFire",
+            max_connections = 5
+        )
+        print("\n\n---\n\n")
+        results = await my_firebase.get_document("testCollection/testDoc")
+        print(async_json_utils.to_json_string(results))
+        print("\n\n---\n\n")
+        results = await my_firebase.get_document_fields("testCollection/testDoc", ["sampleMap"])
+        print(async_json_utils.to_json_string(results))
+        # await my_firebase.create_sub_collection(
+        #     "myDeepCollection/deepDocId",
+        #     "subCollection3",
+        #     "subDoc",
+        #     {"sub_key": "sub_val"}
+        # )
+        # await my_firebase.create_collection(
+        #     "rootCollection",
+        #     "subDoc",
+        #     {"sub_key": "sub_val"}
+        # )
+        # await my_firebase.delete_document("activeSessions/9d402f1502dfd55a4326fa7fc8e6cb7d")
+        # print(async_json_utils.to_json_string(await my_firebase.get_collection("myDeepCollection")))
+
+    start_time = time.time()
+    asyncio.run(test())
+    print(f"FINISHED IN {time.time() - start_time} SECONDS.")

database/async_mongo.py

@@ -0,0 +1,843 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 28th May, 2024
+
+    OBJECTIVE:
+
+        To have one central place from where all async database connectivity happens.
+
+    REFERENCES:
+
+        01. https://motor.readthedocs.io/en/stable/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For async behaviour:
+import asyncio
+
+# For datetime handling:
+import pytz
+import datetime
+
+# MongoDB:
+from motor.motor_asyncio import AsyncIOMotorClient
+from bson.objectid import ObjectId
+from bson.json_util import dumps, loads
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                EXCEPTIONS                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class MongoFindException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo find operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoInsertException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo insert operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoUpdateException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo update operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoReplaceException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo replace operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoDeleteException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo delete operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMongo:
+
+    __db = None
+    __db_name = None
+    __client = None
+
+    def __init__(
+            self,
+            database_name = "myDb",
+            max_connections = 5,
+            debug = True,
+            debug_only_errors = True,
+            host = "localhost",
+            port = 27017,
+            connection_string = None
+    ):
+
+        # Basic config:
+        self.__max_connections = max_connections
+        self.__db_name = database_name
+        self.__host = host,
+        self.__port = port
+        self.__connection_string = connection_string
+
+        # For debugging:
+        self.__debug_only_errors = debug_only_errors
+        self.__printer = IceCreamDebugger(prefix = f"Mongo ({self.__db_name}) | ", includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # Rate/access control:
+        self.__exclusive_semaphore = asyncio.Semaphore(1)
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    @staticmethod
+    def generate_id():
+
+        """
+        Just generates an '_id' in MongoDB style.
+        :return: The '_id' in MongoDB style.
+        """
+
+        return str(ObjectId())
+
+    async def connect(self):
+        try:
+            if self.__connection_string is None:
+                self.__client = AsyncIOMotorClient(
+                    self.__host,
+                    self.__port,
+                    maxPoolSize = self.__max_connections,
+                    minPoolSize = self.__max_connections
+                )
+            else:
+                self.__client = AsyncIOMotorClient(
+                    self.__connection_string,
+                    maxPoolSize = self.__max_connections,
+                    minPoolSize = self.__max_connections
+                )
+            self.__db = self.__client.get_database(self.__db_name)
+            if not self.__debug_only_errors:
+                server_info = await self.__client.server_info()
+                self.__printer(server_info)
+        except Exception as exception: self.__printer(exception)
+
+    async def ensure_connection(self):
+        if self.__db is None:
+            async with self.__exclusive_semaphore:
+                await self.connect()
+
+    @property
+    async def client(self):
+        await self.ensure_connection()
+        return self.__client
+
+    @staticmethod
+    def dict_to_dot_notation(input_dict, parent_key = "", separator = "."):
+        items = []
+        for k, v in input_dict.items():
+            new_key = f"{parent_key}{separator}{k}" if parent_key else k
+            if isinstance(v, dict) and v:
+                items.extend(AsyncMongo.dict_to_dot_notation(v, new_key, separator = separator).items())
+            else:
+                items.append((new_key, v))
+        return dict(items)
+
+    @staticmethod
+    def normalize_date_time(document):
+
+        """
+        MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it.
+        This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was
+        timezone naive, UTC timezone will be applied to it without changing the time value.
+        :param document: The document that you want to normalize the date-time in.
+        :return: The document with normalized datetime.
+        """
+
+        if isinstance(document, datetime.datetime):
+            utc_tz = pytz.timezone("UTC")
+            if document.tzinfo is None: document = utc_tz.localize(document)
+            else: document = document.astimezone(utc_tz)
+
+        if type(document) is list:
+            document = [AsyncMongo.normalize_date_time(item) for item in document]
+
+        if type(document) is dict:
+            document = {
+                AsyncMongo.normalize_date_time(k): AsyncMongo.normalize_date_time(v)
+                for k, v in document.items()
+            }
+
+        return document
+
+    @staticmethod
+    def __from_json_string(json_data):
+        return loads(json_data)
+
+    @staticmethod
+    def __to_json_string(python_data, indent = 4, default = None):
+        return dumps(python_data, indent = indent, default = default)
+
+    async def list_indexes(
+            self,
+            collection,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Lists out the indexes of a collection.
+        :param collection: The collection whose indexes you want to list out,
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The list of indexes or None if the action fails.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        indexes = None
+
+        # Try to list the indexes:
+        try:
+            responses = await self.__db[collection].list_indexes(session = session).to_list(None)
+            indexes = [{key: value for key, value in response.items()} for response in responses]
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not indexes: raise MongoException(hint = "list indexes")
+        return indexes
+
+    async def create_index(
+            self,
+            collection,
+            keys,
+            options = None,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Creates an index on a collection.
+        :param collection: The collection to create the index on.
+        :param keys: The keys (and sorting) to implement the index on.
+        :param options: Additional config.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the execution.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        success = False
+
+        # Try to make the insertion:
+        try:
+            options = options or {}
+            keys = [(k, v) for k, v in keys.items()]
+            response = await self.__db[collection].create_index(keys, session = session, **options)
+            if response: success = True
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not success: raise MongoException(hint = "create index")
+        return success
+
+    async def insert_one(
+            self,
+            collection,
+            document,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Insert data into a collection.
+        :param collection: The collection you want to feed the data into.
+        :param document: The data to be stored.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The id of the inserted data, or null if the action fails.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        inserted_id = None
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].insert_one(document.copy(), session = session)
+            inserted_id = response.inserted_id
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not inserted_id: raise MongoInsertException(hint = f"{collection}")
+        return inserted_id
+
+    async def insert_many(
+            self,
+            collection,
+            documents,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Insert a lot of data into a collection.
+        :param collection: The collection you want to feed the data into.
+        :param documents: The data to be stored.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The id of the inserted data, or null if the action fails.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        inserted_ids = []
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].insert_many(documents, session = session)
+            inserted_ids = response.inserted_ids
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not inserted_ids: raise MongoInsertException(hint = f"{collection}")
+        return inserted_ids
+
+    async def update_one(
+            self,
+            collection,
+            filter_json = None,
+            update_json = None,
+            upsert = False,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Update one document.
+        :param collection: The collection you want to update.
+        :param filter_json: The selection criteria to locate the document to update.
+        :param update_json: The values you want to update.
+        :param upsert: If you want to insert if the document doesn't already exist.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        success = False
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].update_one(
+                filter_json,
+                update_json,
+                upsert = upsert,
+                session = session
+            )
+            success = False if response.modified_count == 0 else True
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not success: raise MongoUpdateException(hint = f"{collection}")
+        return success
+
+    async def update_many(
+            self,
+            collection,
+            filter_json = None,
+            update_json = None,
+            upsert = False,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Update many documents.
+        :param collection: The collection you want to update.
+        :param filter_json: The selection criteria to locate the document to update.
+        :param update_json: The values you want to update.
+        :param upsert: If you want to insert if the document doesn't already exist.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        update_count = 0
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].update_many(
+                filter_json,
+                update_json,
+                upsert = upsert,
+                session = session
+            )
+            update_count = response.modified_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not update_count: raise MongoUpdateException(hint = f"{collection}")
+        return update_count
+
+    async def count(
+            self,
+            collection,
+            filter_json = None
+    ):
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            if filter_json is None: filter_json = {}
+            count = await self.__db[collection].count_documents(filter_json)
+        except Exception as exception: self.__printer(exception)
+
+        # Done here:
+        return count
+
+    async def bulk_write(
+            self,
+            collection,
+            requests,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To perform various individual operations in one go. You will have to import individual actions like "UpdateOne"
+        and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method.
+        :param collection: The collection you want to run the requests on.
+        :param requests: The array of requests (operations) to be performed.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return:
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            response = await self.__db[collection].bulk_write(requests, session = session)
+            count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not count: raise MongoException(hint = "bulk write")
+        return count
+
+    async def find_many(
+            self,
+            collection,
+            filter_json,
+            projections = None,
+            skip = 0,
+            limit = None,
+            sort = None,
+            session = None,
+            as_json_string = False,
+            indent = 4,
+            raise_exception = False
+    ):
+
+        """
+        Finds one or more records that match the given conditions.
+        :param collection: The name of the collection to perform the search in.
+        :param filter_json: The filter criteria.
+        :param projections: What parts of the matching data you want to fetch.
+        :param skip: The no. of records to skip before picking next ones. Needed for pagination.
+        :param limit: The max. no. of records you want to fetch.
+        :param sort: The sorting rules to apply.
+        :param session: The session if you need to do this in a transaction.
+        :param as_json_string: Whether you want it as a JSON string or a Python dict/list.
+        :param indent: The indentation to use if you want it as a JSON string.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        results = None
+
+        # Assume defaults:
+        if sort is None: sort = {"_id": -1}
+        if limit is None: limit = 10
+
+        # try to query the data:
+        try:
+            results = await self.__db[collection].find(
+                filter_json,
+                projections,
+                session = session
+            ).sort(sort).skip(skip).limit(limit).to_list(None)
+            if as_json_string: results = self.__to_json_string(results, indent = indent, default = str)
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not results: raise MongoFindException(hint = f"{collection}")
+        return results
+
+    async def find_one(
+            self,
+            collection,
+            filter_json,
+            projections = None,
+            session = None,
+            as_json_string = False,
+            indent = 4,
+            raise_exception = False
+    ):
+
+        """
+        Finds one record that matches the given conditions.
+        :param collection: The name of the collection to perform the search in.
+        :param filter_json: The filter criteria.
+        :param projections: What parts of the matching data you want to fetch.
+        :param session: The session if you need to do this in a transaction.
+        :param as_json_string: Whether you want it as a JSON string or a Python dict/list.
+        :param indent: The indentation to use if you want it as a JSON string.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        results = None
+
+        # try to query the data:
+        try:
+            results = await self.__db[collection].find_one(filter_json, projections, session = session)
+            if as_json_string: results = self.__to_json_string(results, indent = indent, default = str)
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not results: raise MongoFindException(hint = f"{collection}")
+        return results
+
+    async def replace_one(
+            self,
+            collection,
+            filter_json,
+            replacement_json,
+            upsert = False,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To delete one document from a collection.
+        :param collection: The collection from which you want to delete many records.
+        :param filter_json: The filter criteria.
+        :param replacement_json: The data to put in place of the existing document.
+        :param upsert: If you want to insert if the document doesn't already exist.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        success = False
+
+        # try to query the data:
+        try:
+            result = await self.__db[collection].replace_one(
+                filter_json,
+                replacement_json,
+                upsert = upsert,
+                session = session
+            )
+            if result.modified_count or result.upserted_id: success = True
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not success: raise MongoReplaceException(hint = f"{collection}")
+        return success
+
+    async def delete_one(
+            self,
+            collection,
+            filter_json,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To delete one document from a collection.
+        :param collection: The collection from which you want to delete many records.
+        :param filter_json: The filter criteria.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            result = await self.__db[collection].delete_one(filter_json, session = session)
+            count = result.deleted_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}")
+        return count
+
+    async def delete_many(
+            self,
+            collection,
+            filter_json,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To delete many documents from a collection.
+        WARNING: sending {} in the filter would mean deleting ALL the documents.
+        :param collection: The collection from which you want to delete many records.
+        :param filter_json: The filter criteria.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            result = await self.__db[collection].delete_many(filter_json, session = session)
+            count = result.deleted_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}")
+        return count
+
+    async def aggregate(
+            self,
+            collection,
+            pipeline,
+            limit = None,
+            raise_exception = False
+    ):
+
+        """
+        Perform an advance query on the data.
+        :param collection: The collection to perform the query on.
+        :param pipeline: The pipeline of actions to take. Must be a list.
+        :param limit: The max. no. of records to retrieve.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        results = None
+
+        # try to perform the aggregation action:
+        try: results = await self.__db[collection].aggregate(pipeline).to_list(limit)
+        except Exception as exception: self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not results: raise MongoException(hint = "aggregation")
+        return results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

database/async_mongo_storage.py

@@ -0,0 +1,358 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 28th May, 2024
+
+    OBJECTIVE:
+
+        To have one central place from where all async database connectivity happens.
+
+    REFERENCES:
+
+        01. https://motor.readthedocs.io/en/stable/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+# ---
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For system-level activity:
+import io
+
+# For async behaviour:
+import asyncio
+
+# MongoDB for File Storage:
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
+from bson.objectid import ObjectId
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMongoStorage:
+
+    __db = None
+    __db_name = None
+    __client = None
+    __store = None
+
+    def __init__(
+            self,
+            connection_string = None,
+            max_connections = 5,
+            host_name = "localhost",
+            port = 27017,
+            database_name = "fileStore",
+            debug = True,
+            debug_prefix = "GridFS (M) | ",
+            debug_only_errors = True
+    ):
+
+        # Database Initialization:
+        self.__host_name = host_name
+        self.__port = port
+        self.__db_name = database_name
+        self.__connection_string = connection_string
+        self.__max_connections = max_connections
+
+        # Debugging:
+        self.__debug_only_errors = debug_only_errors
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+    @staticmethod
+    def generate_id():
+
+        """
+        Just generates an '_id' in MongoDB style.
+        :return: The '_id' in MongoDB style.
+        """
+
+        return str(ObjectId())
+
+    async def connect(self):
+
+        """
+        Initialize the database connection.
+        :return: Nothing.
+        """
+
+        if self.__connection_string is None:
+            self.__client = AsyncIOMotorClient(
+                self.__host_name,
+                self.__port,
+                maxPoolSize = self.__max_connections,
+                minPoolSize = self.__max_connections
+            )
+        else:
+            self.__client = AsyncIOMotorClient(
+                self.__connection_string,
+                maxPoolSize = self.__max_connections,
+                minPoolSize = self.__max_connections
+            )
+        self.__db = self.__client.get_database(self.__db_name)
+        self.__store = AsyncIOMotorGridFSBucket(self.__db)
+
+    @property
+    def fs(self):
+
+        """
+        To access the features that have not been wrapped in this reportlab directly.
+        This could include things like streaming files chunk-by-chunk.
+        :return: The file-store instance.
+        """
+
+        return self.__store
+
+    async def write_from_memory(self, file_name, file_data, metadata_json = None):
+
+        """
+        Save a file (from RAM) to Mongo. Suitable for smaller files (a few MBs max.).
+        :param file_name: The name of the file.
+        :param file_data: The data of the file (held in RAM).
+        :param metadata_json: A JSON of metadata information that can later be used to search files (RECOMMENDED).
+        :return: The file's id as a string (if it gets saved) or None.
+        """
+
+        if self.__store is None: await self.connect()
+
+        file_data.seek(0)
+        file_size = file_data.__sizeof__()
+
+        file_id = None
+        try: file_id = await self.__store.upload_from_stream(file_name, file_data, metadata = metadata_json)
+        except Exception as exception: self.__printer(exception, file_name, file_size, file_id)
+        if not self.__debug_only_errors: self.__printer(file_name, file_size, file_id)
+        return str(file_id)
+
+    async def read_to_memory(self, file_identifier, by_id = True):
+
+        """
+        To retrieve a file (in RAM) based on the provided identifier.
+        Suitable for smaller files (a few MBs max.).
+        :param file_identifier: Either the name or the "_id" of the file.
+        :param by_id: Set to True if you are fetching by the "_id" of the file.
+        :return: Either the file (in RAM) or None.
+        """
+
+        if self.__store is None: await self.connect()
+
+        file_data = None
+
+        try:
+            if by_id: grid_out = await self.__store.open_download_stream(ObjectId(file_identifier))
+            else: grid_out = await self.__store.open_download_stream_by_name(file_identifier)
+            file_data = io.BytesIO(await grid_out.read())
+            file_data.seek(0)
+
+        except Exception as exception:
+            file_data = None
+            self.__printer(exception, file_identifier, by_id)
+
+        if not self.__debug_only_errors: self.__printer(file_identifier, by_id)
+        return file_data
+
+    async def delete_file_by_id(self, file_id):
+
+        """
+        Tries to delete one file by the id.
+        :param file_id: The id of the file in the database.
+        :return: True or False based on the success of the operation.
+        """
+
+        if self.__store is None: await self.connect()
+
+        deleted = False
+
+        try:
+            response = await self.__store.delete(file_id = ObjectId(file_id))
+            deleted = True
+        except Exception as exception:
+            self.__printer(exception, file_id, deleted)
+
+        return deleted
+
+    def __format_metadata_json(self, metadata_json):
+
+        """
+        NOTE: ONLY USE WHEN SEARCHING FILES BY METADATA.
+        MongoDB expects dot-notation while searching for files by the metadata. We are making a function to search
+        files assuming that the conditions are to be applied to the metadata itself. So this function add the
+        dot-notation to the right places to conduct a successful search.
+        :param metadata_json: The JSON to format.
+        :return: The formatted JSON that has the right dot-notation.
+        """
+
+        formatted_metadata_json = {}
+
+        for key, value in metadata_json.items():
+            if not key.startswith("$"): key = f"metadata.{key}"
+            else:
+                if type(value) is dict: value = self.__format_metadata_json(value)
+                if type(value) is list: value = [self.__format_metadata_json(item) for item in value]
+            formatted_metadata_json[key] = value
+
+        return formatted_metadata_json
+
+    async def find_file_by_metadata(self, metadata_json, limit = None, skip = None, sort = None):
+
+        """
+        This method only lists the files that match the criteria mentioned in the metadata JSON.
+        :param metadata_json: The JSON that describes what you want to find.
+        :param limit: Max. no. of records to retrieve.
+        :param skip: No. of starting results to skip. Useful for pagination.
+        :param sort: The sorting conditions to follow.
+        :return: A list of (JSONs of) files that match the conditions. The list can be empty.
+        """
+
+        if self.__store is None: await self.connect()
+
+        files_list = []
+
+        try:
+            limit = limit or 10
+            skip = skip or 0
+            sort = {"_id": -1} if not isinstance(sort, dict) else sort
+            formatted_metadata_json = self.__format_metadata_json(metadata_json)
+            return await self.__store.find(
+                formatted_metadata_json
+            ).sort(sort).skip(skip).limit(limit).to_list(None)
+
+        except Exception as exception: self.__printer(exception, metadata_json, len(files_list))
+
+        if not self.__debug_only_errors: self.__printer(metadata_json, len(files_list))
+        return files_list
+
+    async def find_file_by_id(self, file_id):
+
+        """
+        This method allows you to get the file's info from the id of the file.
+        :param file_id: The id that was assigned by Mongo during upload.
+        :return: The file's info or None if the file doesn't exist.
+        """
+
+        if self.__store is None: await self.connect()
+
+        file_info = None
+
+        try:
+            formatted_metadata_json = {"_id": ObjectId(file_id)}
+            file_info = (await self.__store.find(formatted_metadata_json).to_list(1))[0]
+
+        except Exception as exception: self.__printer(exception, file_id, file_info)
+
+        return file_info
+
+    async def get_file_name(self, file_id):
+
+        """
+        Returns the file name if the id of the file is known.
+        :param file_id: The id of the file as assigned by MongoDB when the file was stored.
+        :return: The file's name (if it exists), or None.
+        """
+
+        # Ensure that we are connected:
+        if self.__store is None: await self.connect()
+
+        # Ensure that the input given is of 'ObjectId' type:
+        if type(file_id) is not ObjectId: file_id = ObjectId(str(file_id))
+
+        # Fetch and return the file name:
+        files_list = await self.__store.find(
+            {"_id": file_id},
+            {"filename": True}
+        ).sort({"_id": -1}).limit(1).to_list(None)
+        try: file_name = files_list[0]["filename"]
+        except: file_name = None
+        return file_name
+
+    async def aggregate(
+            self,
+            collection,
+            pipeline,
+            limit = None
+    ):
+
+        """
+        Perform an advance query on the data.
+        :param collection: The collection to perform the query on.
+        :param pipeline: The pipeline of actions to take. Must be a list.
+        :param limit: The max. no. of records to retrieve.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        if self.__store is None: await self.connect()
+
+        results = None
+        try: results = await self.__db[collection].aggregate(pipeline).to_list(limit)
+        except Exception as exception: self.__printer(exception)
+        return results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

database/async_mysql_v2.py

@@ -0,0 +1,311 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 30th Aug., 2024
+
+    OBJECTIVE:
+
+        To be able to access SQL-based databases from python in a simple way.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# MySQL Database:
+import aiomysql
+
+# For data-crunching:
+import pandas as pd
+
+# For time-keeping:
+import time
+
+# OS-level operations:
+import os
+
+# My utils:
+from utils_v2.string import json
+
+# For async activities:
+import asyncio
+
+# For debugging:
+from icecream import IceCreamDebugger
+import traceback
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMySQL:
+
+    def __init__(
+            self,
+            pool_size,
+            *args,
+            **kwargs
+    ):
+
+        """
+        A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as
+        JSON-like structures (list or dict).
+        :param pool_size: The number of connections to maintain n a pool.
+        :param args: Any arguments to pass. Not used.
+        :param kwargs: Pass the connection configuration from here.
+        """
+
+        # Set up the variables:
+        self.__args = args
+        self.__kwargs = kwargs
+        self.__min_pool_size = 10
+        self.__max_pool_size = max(pool_size, self.__min_pool_size)
+        self.__pool = None
+
+        # Set up the debugging tools:
+        self.__printer = IceCreamDebugger(prefix = "MySQL | ", includeContext = True)
+
+    def __del__(self):
+        pass
+
+    async def connect(self):
+
+        """
+        Establish a connection and create a pool of connections to call from.
+        :return: None.
+        """
+
+        try:
+
+            self.__kwargs["db"] = self.__kwargs.pop("database")
+            self.__pool = await aiomysql.create_pool(
+                minsize = self.__min_pool_size,
+                maxsize = self.__max_pool_size,
+                loop = asyncio.get_event_loop(),
+                **self.__kwargs
+            )
+
+        except Exception as exception:
+            self.__printer(exception)
+            self.__pool = None
+
+    async def ensure_connection(self):
+
+        """
+        Tries to ensure that a connection is present.
+        Can be called before every function to make sure that our pool is established.
+        :return: None.
+        """
+
+        if self.__pool is None: await self.connect()
+
+    async def call_procedure(self, procedure_name, procedure_args):
+
+        """
+        To call stored procedures and retrieve all the responses.
+        :param procedure_name:
+        :param procedure_args:
+        :return:
+        """
+
+        # Make sure we have a connection:
+        await self.ensure_connection()
+
+        # Make a variable to hold all the result sets.
+        # Needed for when the procedure responds with many "tables":
+        all_result_sets = []
+
+        # Call the procedure and get the results:
+        async with self.__pool.acquire() as connection:
+            async with connection.cursor() as cursor:
+                await cursor.callproc(procedure_name, procedure_args)
+
+                # Iterate over all result sets,
+                # and process them one-by-one:
+                while True:
+                    this_result_set = []
+                    result = await cursor.fetchall()
+                    if not result: break
+                    columns = [desc[0] for desc in cursor.description]
+                    for row in result: this_result_set.append(dict(zip(columns, row)))
+                    all_result_sets.append(this_result_set)
+                    await cursor.nextset()
+
+        # Done here:
+        return all_result_sets
+
+    async def call_procedure_and_get_json(
+            self,
+            procedure_name,
+            procedure_args,
+            retry_count = 1,
+            backoff_seconds = 0.5,
+            backoff_multiplier = 1.1,
+            return_exception = False
+    ):
+
+        # Note down the start time:
+        start_ts = time.perf_counter()
+
+        # Try to get the data from the database:
+        results = []
+        exception = None
+        for _ in range(retry_count):
+            try: results = await self.call_procedure(
+                procedure_name = procedure_name,
+                procedure_args = procedure_args,
+            )
+            except Exception as exc: exception = exc
+            if exception is None: break
+            await asyncio.sleep(backoff_seconds)
+            backoff_seconds = backoff_seconds * backoff_multiplier
+
+        print("LEN:", len(results))
+
+        # If the results are blank:
+        if len(results) == 0: return {
+            "status": results[0][0]["status"],
+            "message": results[0][0].get("message", "please contact admin (E)"),
+            "seconds": None,
+            "data": {}
+        }
+
+        # Extract the very basic success or failure indicators:
+        formatted_results = {
+            "status": results[0][0]["status"],
+            "message": results[0][0].get("message", "ok"),
+            "seconds": 0.0,
+            "data": {}
+        }
+
+        # Handle the remaining keys of the zeroth result set:
+        for key, value in results[0][0].items():
+            if key not in formatted_results.keys():
+                formatted_results["data"][key] = value
+
+        # Format
+        for index in range(len(results)):
+            if index > 0: formatted_results["data"][f"rs{index-1}"] = results[index]
+
+        # Note down the time taken:
+        formatted_results["seconds"] = time.perf_counter() - start_ts
+
+        # Done here:
+        print(f"{procedure_name}:")
+        print(json.to_string(formatted_results))
+        print("\n")
+        if return_exception: return formatted_results, exception
+        else: return formatted_results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def test(connector):
+        pass
+
+    async def multi_test(count = 1):
+
+        """
+        Test asynchronous behaviour.
+        :return: None.
+        """
+
+        cred_json = {
+            "host": "del.ditscentre.in",
+            "user": "bicree",
+            "port": 3306,
+            "password": "9c3b2808a4aa281129d399fe09e69b53",
+            "database": "bicree"
+        }
+
+        db_conn = AsyncMySQL(
+            pool_size = 25,
+            **cred_json
+        )
+
+        await db_conn.connect()
+
+        # result = await db_conn.call_procedure_and_get_json(
+        #     procedure_name = "login",
+        #     procedure_args = (
+        #         "shree",
+        #         "shree",
+        #         "aiomysql",
+        #         "127.0.0.1"
+        #     )
+        # )
+
+        result = await db_conn.call_procedure_and_get_json(
+            procedure_name = "listSummary",
+            procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",)
+        )
+
+
+    start_time = time.time()
+    asyncio.run(multi_test(count = 1))
+    print(f"ASYNC HITS DONE IN: {time.time() - start_time} seconds.")
+

date_time/date_time.py

@@ -0,0 +1,270 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 21st jun, 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to work with time.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For date and time handling:
+import pytz
+from datetime import datetime, timedelta
+import dateparser
+
+# To handle date-time objects from a Pandas Dataframe:
+import pandas as pd
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Date-Time Formats:
+DATE_TIME_FORMATS = (
+    "%d/%m/%y",
+    "%d-%b-%y",
+    "%d-%m-%y",
+    "%d.%m.%y",
+    "%d/%m/%Y",
+    "%d-%b-%Y",
+    "%d-%m-%Y",
+    "%d.%m.%Y",
+    "%d/%b",
+    "%d%m%Y",
+    "%Y%m%d",
+    "%Y-%m-%d %H:%M:%S"
+)
+
+# Useful Timezones:
+TIMEZONE_UTC = pytz.timezone("UTC")
+TIMEZONE_IST = pytz.timezone("Asia/Kolkata")
+TIMEZONE_ET = pytz.timezone("America/New_York")
+TIMEZONE_CT = pytz.timezone("America/Chicago")
+TIMEZONE_MT = pytz.timezone("America/Denver")
+TIMEZONE_PT = pytz.timezone("America/Los_Angeles")
+TIMEZONE_JST = pytz.timezone("Asia/Tokyo")
+TIMEZONE_CET = pytz.timezone("Europe/Paris")
+TIMEZONE_GMT = pytz.timezone("GMT")
+TIMEZONE_AEST = pytz.timezone("Australia/Sydney")
+TIMEZONE_NZST = pytz.timezone("Pacific/Auckland")
+TIMEZONE_CST = pytz.timezone("Asia/Shanghai")
+TIMEZONE_KST = pytz.timezone("Asia/Seoul")
+TIMEZONE_MSK = pytz.timezone("Europe/Moscow")
+TIMEZONE_BRT = pytz.timezone("America/Sao_Paulo")
+TIMEZONE_GST = pytz.timezone("Asia/Dubai")
+TIMEZONE_SAST = pytz.timezone("Africa/Johannesburg")
+TIMEZONE_AST = pytz.timezone("Asia/Riyadh")
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def translate_date_time_string(
+        datetime_string,
+        source_format = None,
+        destination_format = "%Y-%m-%dT%H:%M:%S"
+):
+
+    """
+    To convert an input datetime string to a different format.
+    :param datetime_string: The datetime string to translate.
+    :param source_format: The current format of the string. If not provided, dateparser will be used.
+    :param destination_format: The format to convert to.
+    :return: The converted datetime string.
+    """
+
+    try:
+        if source_format is None: datetime_obj = dateparser.parse(datetime_string)
+        else: datetime_obj = datetime.strptime(datetime_string, source_format)
+        return datetime_obj.strftime(destination_format)
+    except Exception as exception:
+        return None
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def parse_date_time(input_value, timezone = None, date_formats = None):
+
+    """
+    To take any kind of input and interpret the datetime from it.
+    :param input_value: Either a string or an integer or some form of datetime representation.
+    :param timezone: The timezone to apply to the interpreted datetime. EXISTING TIMEZONE INFO WILL BE OVERWRITTEN.
+    :param date_formats: The string formats to consider when parsing a string input.
+    :return: The parsed datetime or null.
+    """
+
+    datetime_object = None
+    date_formats = date_formats or DATE_TIME_FORMATS
+
+    # It could either be in seconds or milliseconds from epoch time's base date (January 1, 1970),
+    # or it could be days since Microsoft Excel's base date (December 31, 1899).
+    if type(input_value) in [int, float]:
+        if input_value > 9999999999.0: datetime_object = datetime.fromtimestamp(input_value / 1000.0)
+        if input_value > 999999.0: datetime_object = datetime.fromtimestamp(input_value)
+        else: datetime_object = datetime.fromtimestamp(input_value * 24 * 60 * 60.0) - timedelta(days = 25569)
+
+    # The input can even be a pre-formatted date:
+    if type(input_value) is str:
+        datetime_object = dateparser.parse(input_value, date_formats = date_formats)
+
+    # If the type is a datetime object, then return it as it is:
+    if type(input_value) is datetime:
+        datetime_object = input_value
+
+    # If the type is the native datetime format of pandas:
+    if type(input_value) is pd._libs.tslibs.timestamps.Timestamp:
+        datetime_object = input_value.to_pydatetime()
+
+    # Process the timezone:
+    if datetime_object is not None and timezone is not None:
+        datetime_object = as_if_timezone(datetime_object, timezone)
+
+    # Done here:
+    return datetime_object
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_current_date_time(timezone = None, as_string = False):
+
+    """
+    Returns the current time as a datetime object.
+    :param timezone: The timezone to apply to the returned datetime.
+    :param as_string: Whether, or not, you want the output as a string.
+    :return: The datetime object/string representing the current time.
+    """
+
+    if timezone is not None and isinstance(timezone, str): timezone = pytz.timezone(timezone)
+    now = datetime.now(timezone)
+    return now.isoformat() if as_string else now
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_current_ist_date_time(as_string = False):
+
+    """
+    Gives out the current time in IST timezone.
+    :param as_string: Whether, or not, you want the output as a string.
+    :return: The datetime object or string representing the current time.
+    """
+
+    return get_current_date_time(
+        timezone = TIMEZONE_IST,
+        as_string = as_string
+    )
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_current_utc_date_time(as_string = False):
+
+    """
+    Gives out the current time in UTC timezone.
+    :param as_string: Whether, or not, you want the output as a string.
+    :return: The datetime object or string representing the current time.
+    """
+
+    return get_current_date_time(
+        timezone = TIMEZONE_UTC,
+        as_string = as_string
+    )
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def as_if_timezone(datetime_object, timezone):
+
+    """
+    Ignores existing timezone info and applies the intended timezone.
+    The time stays the same, only the timezone marker changes.
+    e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-09 00:00:00+00:00
+    HINT: IT PRETENDS "AS IF" THE TIMEZONE WAS THE INPUT TIMEZONE.
+    :param datetime_object: The datetime object on which the timezone needs to be applied.
+    :param timezone: The timezone that needs to be applied.
+    :return: A timezone-aware datetime object.
+    """
+
+    tz_object = pytz.timezone(timezone) if isinstance(timezone, str) else timezone
+    return tz_object.localize(datetime_object.replace(tzinfo = None))
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def to_timezone(datetime_object, timezone):
+
+    """
+    Converts from one timezone to another. The time is adjusted by computing the difference between the two timezones.
+    NOTE: THIS FUNCTION ASSUMES THE INPUT WAS IN UTC IF THE INPUT WAS TIMEZONE-NAIVE.
+    e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-08 18:30:00+00:00
+    :param datetime_object: The datetime object on which the timezone needs to be applied.
+    :param timezone: The timezone that needs to be applied.
+    :return: A timezone-aware datetime object.
+    """
+
+    if isinstance(timezone, str): timezone = pytz.timezone(timezone)
+    if datetime_object.tzinfo is None: return datetime_object.replace(tzinfo = TIMEZONE_UTC)
+    return datetime_object.astimezone(timezone)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

image/scanner/scanner.py

@@ -0,0 +1,410 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 19th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a way to easily scan documents like printouts and visiting cards.
+
+    REFERENCES:
+
+        01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085
+        02. https://github.com/JaidedAI/EasyOCR
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+
+from diffusers.utils.import_utils import candidates
+
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To work with image processing:
+import cv2
+import imutils
+import numpy as np
+
+# To run OCR:
+import easyocr
+
+# To download images from the web:
+import requests
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.ai.object_detection.yolo import YoloDetect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DocumentScanner:
+
+    def __init__(
+            self,
+            layout_detection_yolo = None,
+            whitelisted_yolo_classes = None,
+            ocr_languages = None
+    ):
+
+        self.__yolo = YoloDetect(
+            model_file = layout_detection_yolo,
+            debug = False
+        )
+        self.__whitelisted_yolo_classes = whitelisted_yolo_classes or []
+        self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"])
+        pass
+
+    @staticmethod
+    def open_image(image):
+
+        """
+        Opens an image from various sources.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :return: The image as a CV2 object.
+        """
+
+        # When the image is provided as a string,
+        # it could be either as a URL or a path to a local file:
+        if isinstance(image, str):
+            if image.startswith("https://") or image.startswith("http://"):
+                image = io.BytesIO(requests.get(image).content)
+            else: image = cv2.imread(image)
+
+        # If the image is provided as a io.BytesIO buffer:
+        if isinstance(image, io.BytesIO):
+            image.seek(0)
+            image = np.asarray(bytearray(image.read()), dtype = np.uint8)
+            image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+
+        # Done here:
+        return image
+
+    @staticmethod
+    def polygon_area(points):
+
+        """
+        Computes the area occupied by a shape described by the array of points.
+        Example input: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+        :return: The area of the shape.
+        """
+
+        n = len(points)
+        area = 0.0
+        for i in range(n):
+            j = (i + 1) % n  # next vertex
+            area += points[i][0] * points[j][1]
+            area -= points[j][0] * points[i][1]
+        return abs(area) / 2.0
+
+    @staticmethod
+    def show(image, title = "Preview", wait = True):
+
+        """
+        Just a quick wrapper to show the image in a window.
+        :param image: The image that you want to show.
+        :param title: The title of the window.
+        :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many
+            windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the
+            very last one.
+        :return: None.
+        """
+
+        cv2.imshow(title, image)
+        if wait:
+            cv2.waitKey()
+            cv2.destroyAllWindows()
+
+    @staticmethod
+    def enhance(image):
+
+        """
+        Applies a form of contrast boost to make the edges more easily visible.
+        :param image: The OpenCV image that needs to be enhanced.
+        :return: The enhanced CV2 image.
+        """
+
+        return cv2.detailEnhance(
+            src = image,
+            sigma_s = 20,
+            sigma_r = 0.15
+        )
+
+    @staticmethod
+    def get_edges(image):
+
+        """
+        Gets the edges in an image.
+        :param image: A CV2 image.
+        :return: The image with the edges detected.
+        """
+
+        # Denoise the image:
+        processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        processed_image = cv2.GaussianBlur(
+            processed_image,
+            ksize = (5, 5),
+            sigmaX = 0
+        )
+
+        # Detect edges in the image:
+        processed_image = cv2.Canny(
+            processed_image,
+            threshold1 = 50,
+            threshold2 = 200
+        )
+
+        # Close small gaps in the edges:
+        kernel = np.ones((5, 5), np.uint8)
+        processed_image = cv2.dilate(processed_image, kernel, iterations = 1)
+        # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel)
+        # processed_image = cv2.erode(processed_image, kernel, iterations = 1)
+
+        # Done here:
+        return processed_image
+
+    def scan(self, image, min_area = 0.125):
+
+        """
+        Looks for a rectangle in an image and flattens it out. No colour filters are applied here.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_area: The minimum area occupied by the document in the image.
+        :return: The flattened image if any, else None.
+        """
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        if image is None: return None
+        height, width = image.shape[:2]
+        image_area = height * width
+
+        # Enhance the image to have better visibility of edges:
+        # processed_image = self.enhance(image.copy())
+
+        # Detect edges in the image:
+        processed_image = self.get_edges(image.copy())
+
+        # Find the contours in the edges,
+        # and sort them in ascending order:
+        contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+        contours = imutils.grab_contours(contours)
+        contours = sorted(contours, key = cv2.contourArea, reverse = True)
+
+        # We try to get clean and passable contours.
+        # For our purposes, 'clean' is when it can be approximated to exactly 4 sides,
+        # and 'passable' is when the approximation comes close to it:
+        clean_rects = []
+        passable_rects = []
+        for contour in contours:
+            perimeter = cv2.arcLength(contour, closed = True)
+            approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True)
+            if len(approximation) == 4: clean_rects.append(approximation)
+            elif len(approximation) <= 32:
+                candidate_contour = cv2.minAreaRect(approximation)
+                candidate_contour = cv2.boxPoints(candidate_contour)
+                candidate_contour = np.intp(candidate_contour)
+                passable_rects.append(candidate_contour)
+        clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True)
+        passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True)
+
+        # Now we decide between the best candidate:
+        document_outline = None
+        if len(clean_rects) > 0:
+            candidate_rect = np.array([p[0] for p in clean_rects[0]])
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+        if document_outline is None and len(passable_rects) > 0:
+            candidate_rect = cv2.minAreaRect(passable_rects[0])
+            candidate_rect = cv2.boxPoints(candidate_rect)
+            candidate_rect = np.intp(candidate_rect)
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+
+        # If there is no rectangular contour,
+        # we exit with failure:
+        if document_outline is None: return None
+
+        # Since we did get the best candidate for the document,
+        # we figure out which point n the contour is which corner of the document:
+        document_rectangle = np.zeros((4, 2), dtype = "float32")
+        sum_points = document_outline.sum(axis = 1)
+        document_rectangle[0] = document_outline[np.argmin(sum_points)]
+        document_rectangle[2] = document_outline[np.argmax(sum_points)]
+        diff_points = np.diff(document_outline, axis = 1)
+        document_rectangle[1] = document_outline[np.argmin(diff_points)]
+        document_rectangle[3] = document_outline[np.argmax(diff_points)]
+        (top_left, top_right, bottom_right, bottom_left) = document_rectangle
+
+        # let's compute the dimensions of the document in the image:
+        left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
+        right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
+        top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
+        bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
+        max_height = max(int(left_height), int(right_height))
+        max_width = max(int(top_width), int(bottom_width))
+
+        # We compute the destination of the transform:
+        destination = np.array(
+            object = [
+                [0, 0],
+                [max_width - 1, 0],
+                [max_width - 1, max_height - 1],
+                [0, max_height - 1]
+            ],
+            dtype = "float32"
+        )
+
+        # We apply the transform that flattens out the document:
+        matrix = cv2.getPerspectiveTransform(document_rectangle, destination)
+        flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height))
+
+        # Done here:
+        return flattened_image
+
+    def extract_text(
+            self,
+            image,
+            min_confidence = 0.5,
+            margin = 0.05
+    ):
+
+        """
+        To run OCR on the input image.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_confidence: The minimum amount of confidence for detected text to be considered.
+        :param margin: The extra pixels to include when cropping into a section for OCR.
+        :return: The extracted text.
+        """
+
+        # Make a variable that will hold the final result:
+        extracted_text = {}
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        height, width = image.shape[:2]
+
+        # Run OCR on the entire canvas:
+        ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence]
+        extracted_text["fullDoc"] = " ".join(ocr_result)
+
+        # Get the doc layout in the image:
+        document_sections = self.__yolo.predict(image)
+        document_sections = document_sections["boxes"]
+
+        # For each section, we run the OCR process individually:
+        section_results = []
+        for section in document_sections:
+            if section["class"] in self.__whitelisted_yolo_classes:
+                section_width = section["x2"] - section["x1"]
+                section_height = section["y2"] - section["y1"]
+                x1 = max(int(section["x1"] - (margin * section_width)), 0)
+                y1 = max(int(section["y1"] - (margin * section_height)), 0)
+                x2 = min(int(section["x2"] + (margin * section_width)), width)
+                y2 = min(int(section["y2"] + (margin * section_height)), height)
+                sub_image = image[y1:y2, x1:x2]
+                ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence]
+                section_results.append({
+                    "text": " ".join(ocr_result),
+                    "class": section["class"],
+                    "className": section["className"],
+                    "x1": x1,
+                    "y1": y1,
+                    "x2": x2,
+                    "y2": y2
+                })
+        extracted_text["bySection"] = section_results
+
+        # Done here:
+        return extracted_text
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import time
+
+    my_scanner = DocumentScanner(
+        layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
+        whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
+        # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
+        ocr_languages = ["en"]
+    )
+
+    image = my_scanner.open_image(r"/home/developer/Downloads/talkaholics_card.jpg")
+    document_image = my_scanner.scan(image)
+    if document_image is not None:
+        cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image)
+        start_time = time.time()
+        document_text = my_scanner.extract_text(
+            document_image,
+            min_confidence = 0.5
+        )
+        print("OCR RESULT:")
+        print(document_text)
+
+        print(f"FINISHED IN {time.time() - start_time} SECONDS!")
+
+    else: print("No image")

image/scanner/scanner_v2.py

@@ -0,0 +1,379 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 24th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a way to easily scan documents like printouts and visiting cards.
+
+    REFERENCES:
+
+        01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085
+        02. https://github.com/JaidedAI/
+        03. https://github.com/criistian14/flutter_document_scanner/blob/master/flutter_document_scanner_android/android/src/main/kotlin/com/christian/flutterDocumentScanner/OpenCVPlugin.kt
+        04. https://www.geeksforgeeks.org/python-bilateral-filtering/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To work with image processing:
+import cv2
+import imutils
+import numpy as np
+
+# To run OCR:
+import easyocr
+
+# To download images from the web:
+import requests
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.ai.object_detection.yolo import YoloDetect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DocumentScanner:
+
+    def __init__(
+            self,
+            layout_detection_yolo = None,
+            whitelisted_yolo_classes = None,
+            ocr_languages = None
+    ):
+
+        self.__yolo = YoloDetect(
+            model_file = layout_detection_yolo,
+            debug = False
+        )
+        self.__whitelisted_yolo_classes = whitelisted_yolo_classes or []
+        self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"])
+        pass
+
+    @staticmethod
+    def open_image(image):
+
+        """
+        Opens an image from various sources.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :return: The image as a CV2 object.
+        """
+
+        # When the image is provided as a string,
+        # it could be either as a URL or a path to a local file:
+        if isinstance(image, str):
+            if image.startswith("https://") or image.startswith("http://"):
+                image = io.BytesIO(requests.get(image).content)
+            else: image = cv2.imread(image)
+
+        # If the image is provided as a io.BytesIO buffer:
+        if isinstance(image, io.BytesIO):
+            image.seek(0)
+            image = np.asarray(bytearray(image.read()), dtype = np.uint8)
+            image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+
+        # Done here:
+        return image
+
+    @staticmethod
+    def show(image, title = "Preview", wait = True):
+
+        """
+        Just a quick wrapper to show the image in a window.
+        :param image: The image that you want to show.
+        :param title: The title of the window.
+        :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many
+            windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the
+            very last one.
+        :return: None.
+        """
+
+        cv2.imshow(title, image)
+        if wait:
+            cv2.waitKey()
+            cv2.destroyAllWindows()
+
+    @staticmethod
+    def get_edges(image):
+
+        """
+        Apply all the preprocessing filters on the image before sending it off for contour-finding.
+        :param image: The image to pre-process.
+        :return: The pre-processed image.
+        """
+
+        # Denoise the image:
+        processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        processed_image = cv2.bilateralFilter(
+            processed_image,
+            d = 9,
+            sigmaColor = 75,
+            sigmaSpace = 75
+        )
+
+        # Detect edges in the image:
+        processed_image = cv2.Canny(
+            processed_image,
+            threshold1 = 75,
+            threshold2 = 200
+        )
+
+        # Close small gaps in the edges:
+        kernel = np.ones((5, 5), np.uint8)
+        processed_image = cv2.dilate(processed_image, kernel, iterations = 1)
+        # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel)
+        # processed_image = cv2.erode(processed_image, kernel, iterations = 1)
+
+        # Done here:
+        return processed_image
+
+    def scan(self, image, min_area = 0.125):
+
+        """
+        Looks for a rectangle in an image and flattens it out. No colour filters are applied here.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_area: The minimum area occupied by the document in the image.
+        :return: The flattened image if any, else None.
+        """
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        if image is None: return None
+        height, width = image.shape[:2]
+        image_area = height * width
+
+        # image = self.enhance(image)
+        processed_image = self.get_edges(image)
+        # self.show(processed_image)
+
+        # Find the contours in the edges,
+        # and sort them in ascending order:
+        contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+        contours = imutils.grab_contours(contours)
+        contours = sorted(contours, key = cv2.contourArea, reverse = True)
+
+        # We try to get clean and passable contours.
+        # For our purposes, 'clean' is when it can be approximated to exactly 4 sides,
+        # and 'passable' is when the approximation comes close to it:
+        clean_rects = []
+        passable_rects = []
+        for contour in contours:
+            perimeter = cv2.arcLength(contour, closed = True)
+            approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True)
+            if len(approximation) == 4: clean_rects.append(approximation)
+            elif len(approximation) <= 32:
+                candidate_contour = cv2.minAreaRect(approximation)
+                candidate_contour = cv2.boxPoints(candidate_contour)
+                candidate_contour = np.intp(candidate_contour)
+                passable_rects.append(candidate_contour)
+        clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True)
+        passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True)
+
+        # Now we decide between the best candidate:
+        document_outline = None
+        if len(clean_rects) > 0:
+            candidate_rect = np.array([p[0] for p in clean_rects[0]])
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+        if document_outline is None and len(passable_rects) > 0:
+            candidate_rect = cv2.minAreaRect(passable_rects[0])
+            candidate_rect = cv2.boxPoints(candidate_rect)
+            candidate_rect = np.intp(candidate_rect)
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+
+        # If there is no rectangular contour,
+        # we exit with failure:
+        if document_outline is None: return None
+
+        # Since we did get the best candidate for the document,
+        # we figure out which point n the contour is which corner of the document:
+        document_rectangle = np.zeros((4, 2), dtype = "float32")
+        sum_points = document_outline.sum(axis = 1)
+        document_rectangle[0] = document_outline[np.argmin(sum_points)]
+        document_rectangle[2] = document_outline[np.argmax(sum_points)]
+        diff_points = np.diff(document_outline, axis = 1)
+        document_rectangle[1] = document_outline[np.argmin(diff_points)]
+        document_rectangle[3] = document_outline[np.argmax(diff_points)]
+        (top_left, top_right, bottom_right, bottom_left) = document_rectangle
+
+        # let's compute the dimensions of the document in the image:
+        left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
+        right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
+        top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
+        bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
+        max_height = max(int(left_height), int(right_height))
+        max_width = max(int(top_width), int(bottom_width))
+
+        # We compute the destination of the transform:
+        destination = np.array(
+            object = [
+                [0, 0],
+                [max_width - 1, 0],
+                [max_width - 1, max_height - 1],
+                [0, max_height - 1]
+            ],
+            dtype = "float32"
+        )
+
+        # We apply the transform that flattens out the document:
+        matrix = cv2.getPerspectiveTransform(document_rectangle, destination)
+        flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height))
+
+        # Done here:
+        return flattened_image
+
+    def extract_text(
+            self,
+            image,
+            min_confidence = 0.5,
+            margin = 0.05
+    ):
+
+        """
+        To run OCR on the input image.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_confidence: The minimum amount of confidence for detected text to be considered.
+        :param margin: The extra pixels to include when cropping into a section for OCR.
+        :return: The extracted text.
+        """
+
+        # Make a variable that will hold the final result:
+        extracted_text = {}
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        height, width = image.shape[:2]
+
+        # Run OCR on the entire canvas:
+        ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence]
+        extracted_text["fullDoc"] = " ".join(ocr_result)
+
+        # Get the doc layout in the image:
+        document_sections = self.__yolo.predict(image)
+        document_sections = document_sections["boxes"]
+
+        # For each section, we run the OCR process individually:
+        section_results = []
+        for section in document_sections:
+            if section["class"] in self.__whitelisted_yolo_classes:
+                section_width = section["x2"] - section["x1"]
+                section_height = section["y2"] - section["y1"]
+                x1 = max(int(section["x1"] - (margin * section_width)), 0)
+                y1 = max(int(section["y1"] - (margin * section_height)), 0)
+                x2 = min(int(section["x2"] + (margin * section_width)), width)
+                y2 = min(int(section["y2"] + (margin * section_height)), height)
+                sub_image = image[y1:y2, x1:x2]
+                ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence]
+                section_results.append({
+                    "text": " ".join(ocr_result),
+                    "class": section["class"],
+                    "className": section["className"],
+                    "x1": x1,
+                    "y1": y1,
+                    "x2": x2,
+                    "y2": y2
+                })
+        extracted_text["bySection"] = section_results
+
+        # Done here:
+        return extracted_text
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import time
+
+    my_scanner = DocumentScanner(
+        layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
+        whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
+        # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
+        ocr_languages = ["en"]
+    )
+
+    image = my_scanner.open_image(r"/home/developer/Downloads/sushmita_card.jpg")
+    # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card.jpg")
+    # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card_2.jpg")
+    # image = my_scanner.open_image(r"/home/developer/Downloads/card_square.jpg")
+    document_image = my_scanner.scan(image)
+    if document_image is not None:
+        cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image)
+        start_time = time.time()
+        document_text = my_scanner.extract_text(
+            document_image,
+            min_confidence = 0.5
+        )
+        print("OCR RESULT:")
+        print(document_text)
+
+        print(f"FINISHED IN {time.time() - start_time} SECONDS!")
+
+    else: print("No image")

mail/async_mail.py

@@ -0,0 +1,440 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 16th Jul, 2024
+
+    OBJECTIVE:
+
+        To be able to send out mails from code.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For working with mails:
+import aiosmtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.image import MIMEImage
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email import encoders
+
+# My utils:
+from utils import rate_limit_utils
+
+# Common:
+from shared.statuses import StatusCodes
+
+# For random strings:
+import string
+import random
+
+# For system-level activities:
+import os
+
+# For working with files in RAM:
+import io
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class MailMessage:
+
+    def __init__(self, to_email, subject):
+
+        """
+        Create an instance of the message that you would like to send.
+        :param to_email: The EMail ID of th recipient.
+        :param subject: The subject of the mail.
+        """
+
+        self.message = MIMEMultipart()
+        self.message["To"] = to_email
+        self.message["Subject"] = subject
+
+    def add_text(self, text):
+
+        """
+        Add plain-text to the mail body.
+        :param text: The text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(text, "plain"))
+
+    def add_html(self, html_text):
+
+        """
+        Add HTML text to the mail body.
+        :param html_text: The HTML text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(html_text, "html"))
+
+    def add_inline_image(self, image_file, content_id = None):
+
+        """
+        Add an inline image to the body of the mail.
+        NOTE: This is NOT the same as sending an image as an attachment.
+        :param image_file: The image data to attach to the mail body.
+        :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not
+            specified, I will generate a random string. You may write a custom value here if you know what you are
+            doing. For most use cases, please ignore this field.
+        :return: None.
+        """
+
+        # Read the image as bytes:
+        image_bytes = None
+        if type(image_file) is str:
+            with open(image_file, "rb") as opened_image_file:
+                image_bytes = opened_image_file.read()
+        if type(image_file) is io.BytesIO:
+            image_file.seek(0)
+            image_bytes = image_file.getvalue()
+
+        # Declare the part to be attached to the multipart message:
+        if image_bytes is not None:
+
+            # Create the HTML block if the image pointer is blank:
+            if content_id is None:
+                content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+                self.add_html(f"""
+                <html>
+                    <body>
+                        <p><img src="cid:{content_id}"></p>
+                    </body>
+                </html>
+                """)
+
+            # Then add the image:
+            image_part = MIMEImage(image_bytes)
+            image_part.add_header("Content-ID", f"<{content_id}>")
+            self.message.attach(image_part)
+
+    def add_attachment(self, attachment_file, file_name = None):
+
+        # Declare the part to be attached to the multipart message:
+        part = MIMEBase("application", "octet-stream")
+
+        # If the attachment is a file stored in the local disk:
+        if type(attachment_file) is str:
+            file_name = file_name or os.path.split(attachment_file)[-1]
+            with open(attachment_file, "rb") as attachment:
+                part.set_payload(attachment.read())
+
+        # If the file is held in RAM:
+        if type(attachment_file) is io.BytesIO():
+            attachment_file.seek(0)
+            part.set_payload(attachment_file.read())
+
+        # Encode and attach the file:
+        encoders.encode_base64(part)
+        part.add_header(
+            "Content-Disposition",
+            f"attachment; filename= {file_name}",
+        )
+        self.message.attach(part)
+
+    def get_message(self):
+        return self.message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class AsyncMailClient:
+
+    # Constants:
+    SMTP_TLS_PORT = 587
+    SMTP_SSL_PORT = 465
+
+    # variables:
+    __smtp = None
+
+    def __init__(
+            self,
+            email,
+            password,
+            server,
+            port = 587,
+            rate_limiters = None,
+            wait_for_turn = True,
+            debug = True,
+            debug_prefix = "Mail (C) | "
+    ):
+
+        """
+        Set up the mail client.
+        :param email: The Email ID to use when sending out mails.
+        :param password: The password of the EMail ID that is being used.
+        :param server: The EMail server.
+        :param port: The port number to connect to the host.
+        :param rate_limiters: The rate limiters to use. Must have "get_turn" and "has_turn" methods. "get_turn" method
+            must wait for the turn, and "has_turn" method must only check if a turn is available.
+        :param wait_for_turn: To wait for turn if the rate limit has been exceeded, or to return with failure.
+        :param debug: Whether, or not, you want to print debugging messages.
+        :param debug_prefix: The prefix to identify the debugging messages.
+        """
+
+        # Initialize the debugger:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # Note down the credentials and other details:
+        self.__email = email
+        self.__password = password
+        self.__server = server
+        self.__port = port
+        self.__rate_limiters = rate_limiters if type(rate_limiters) is list else ([rate_limiters] if rate_limiters is not None else [])
+        self.__wait_for_turn = wait_for_turn
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    async def login(self):
+
+        """
+        To connect to the mail server and authenticate the user.
+        :return: True if authenticated, else False.
+        """
+
+        # Initialize the SMTP connection,
+        # and return with success if all goes well:
+        try:
+            self.__smtp = aiosmtplib.SMTP(
+                hostname = self.__server,
+                port = self.__port,
+                use_tls = False,
+                start_tls = False
+            )
+            await self.__smtp.connect()
+            await self.__smtp.starttls()
+            await self.__smtp.login(self.__email, self.__password)
+            return True
+
+        # Return with failure if something goes wrong:
+        except Exception as exception:
+            self.__printer(exception)
+            try: await self.__smtp.quit()
+            except Exception as exception: self.__printer(exception)
+            self.__smtp = None
+            return False
+
+    async def ensure_connection(self):
+
+        """
+        Can be run before the sending operation to confirm that we are yet connected to the server.
+        If not connected, this code will reattempt to connect and log-in.
+        :return: True if connected, else False.
+        """
+
+        # If the login had failed initially, the object will be set to null.
+        # In such a case, we make an attempt to login:
+        if self.__smtp is None:
+            return await self.login()
+
+        # If the login was successful, we check if the connection is active.
+        # If not, we try to re-login:
+        if self.__smtp.is_connected:
+            return True
+        else:
+            try:
+                await self.__smtp.connect()
+                await self.__smtp.starttls()
+                await self.__smtp.login(self.__email, self.__password)
+                return True
+            except Exception as exception:
+                self.__printer(exception)
+                try: await self.__smtp.quit()
+                except Exception as exception: self.__printer(exception)
+                self.__smtp = None
+                return False
+
+    async def logout(self):
+
+        """
+        Closes the connection to the SMTP client.
+        :return: True by default.
+        """
+
+        if self.__smtp is not None:
+            try: await self.__smtp.quit()
+            except Exception as exception: self.__printer(exception)
+        self.__smtp = None
+        return True
+
+    async def send(self, mail: MailMessage):
+
+        """
+        Send out the mail.
+        :param mail: The instance of 'MailMessage' with all the content populated.
+        :return: A dict with 'success' and 'message'.
+        """
+
+        # Return with failure if we aren't connected,
+        # and our attempt to (re)connect fails:
+        if not await self.ensure_connection():
+            return {
+                "success": False,
+                "message": "login failed"
+            }
+
+        # Comply with the rate-limit:
+        for rate_limiter in self.__rate_limiters:
+            if not self.__wait_for_turn:
+                if not await rate_limiter.has_turn(): return False
+            got_turn = await rate_limiter.get_turn()
+            if not got_turn:
+                return {
+                    "success": False,
+                    "message": "rate-limit wait timeout"
+                }
+
+        # Try to send the message:
+        try:
+            mail.message["From"] = self.__email
+            await self.__smtp.send_message(mail.message)
+            return {
+                "success": True,
+                "message": "mail sent"
+            }
+
+        # If something goes wrong:
+        except Exception as exception:
+            self.__printer(exception)
+            return {
+                "success": False,
+                "message": str(exception)
+            }
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    from utils_v2.string import json
+    from utils_v2.mail.mail_message import MailMessage
+
+    async def test():
+
+        rate_lim = rate_limit_utils.TokenBucket(
+            rate_limit = 1,
+            seconds = 60.0,
+        )
+
+        mail_client = AsyncMailClient(
+            email = "sender@gmail.com",
+            password = "secret_password",
+            server = "smtp.gmail.com",
+            rate_limiters = rate_lim
+        )
+
+        my_mail = MailMessage(
+            to_email = "recipient@gmail.com",
+            subject = "Bhopli is the best!"
+        )
+        my_mail.add_html(
+            """
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <meta charset="UTF-8">
+                <title>Sample HTML String</title>
+                <style>
+                    .heading {
+                        color: #ff9025;
+                    }
+                    .sub-heading {
+                        color: #000000;
+                    }
+                </style>
+            </head>
+            <body>
+                <h1 class="heading">Hello, Bhopli!</h1>
+                <h2 class="sub-heading">Bhopli is the best, most well-behaved cat in the known universe.</h2>
+            </body>
+            </html>
+            """
+        )
+        my_mail.add_text("This is how you should pet her 👇")
+        my_mail.add_inline_image(r"/path/to/image/cat_petting.png")
+        my_mail.add_attachment(r"/path/to/file/sample_label.pdf")
+
+        await mail_client.login()
+        result = await mail_client.send(my_mail)
+        print("MAIL RESULT:", json.to_json_string(result))
+        await mail_client.logout()
+
+    asyncio.run(test())

mail/mail_message.py

@@ -0,0 +1,268 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 16th Jul, 2024
+
+    OBJECTIVE:
+
+        To be able to send out mails from code.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For working with mails:
+import aiosmtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.image import MIMEImage
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email import encoders
+
+# My utils:
+from utils import rate_limit_utils
+
+# Common:
+from shared.statuses import StatusCodes
+
+# For random strings:
+import string
+import random
+
+# For system-level activities:
+import os
+
+# For working with files in RAM:
+import io
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class MailMessage:
+
+    def __init__(self, to_email, subject):
+
+        """
+        Create an instance of the message that you would like to send.
+        :param to_email: The EMail ID of th recipient.
+        :param subject: The subject of the mail.
+        """
+
+        self.message = MIMEMultipart()
+        self.message["To"] = to_email
+        self.message["Subject"] = subject
+
+    def add_text(self, text):
+
+        """
+        Add plain-text to the mail body.
+        :param text: The text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(text, "plain"))
+
+    def add_html(self, html_text):
+
+        """
+        Add HTML text to the mail body.
+        :param html_text: The HTML text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(html_text, "html"))
+
+    def add_inline_image(self, image_file, content_id = None):
+
+        """
+        Add an inline image to the body of the mail.
+        NOTE: This is NOT the same as sending an image as an attachment.
+        :param image_file: The image data to attach to the mail body.
+        :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not
+            specified, I will generate a random string. You may write a custom value here if you know what you are
+            doing. For most use cases, please ignore this field.
+        :return: None.
+        """
+
+        # Read the image as bytes:
+        image_bytes = None
+        if type(image_file) is str:
+            with open(image_file, "rb") as opened_image_file:
+                image_bytes = opened_image_file.read()
+        if type(image_file) is io.BytesIO:
+            image_file.seek(0)
+            image_bytes = image_file.getvalue()
+
+        # Declare the part to be attached to the multipart message:
+        if image_bytes is not None:
+
+            # Create the HTML block if the image pointer is blank:
+            if content_id is None:
+                content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+                self.add_html(f"""
+                <html>
+                    <body>
+                        <p><img src="cid:{content_id}"></p>
+                    </body>
+                </html>
+                """)
+
+            # Then add the image:
+            image_part = MIMEImage(image_bytes)
+            image_part.add_header("Content-ID", f"<{content_id}>")
+            self.message.attach(image_part)
+
+    def add_attachment(self, attachment_file, file_name = None):
+
+        # Declare the part to be attached to the multipart message:
+        part = MIMEBase("application", "octet-stream")
+
+        # If the attachment is a file stored in the local disk:
+        if type(attachment_file) is str:
+            file_name = file_name or os.path.split(attachment_file)[-1]
+            with open(attachment_file, "rb") as attachment:
+                part.set_payload(attachment.read())
+
+        # If the file is held in RAM:
+        if type(attachment_file) is io.BytesIO():
+            attachment_file.seek(0)
+            part.set_payload(attachment_file.read())
+
+        # Encode and attach the file:
+        encoders.encode_base64(part)
+        part.add_header(
+            "Content-Disposition",
+            f"attachment; filename= {file_name}",
+        )
+        self.message.attach(part)
+
+    def get_message(self):
+        return self.message
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    from utils import json_utils
+    from utils_v2.mail.async_mail import AsyncMailClient
+
+    async def test():
+
+        rate_lim = rate_limit_utils.TokenBucket(
+            rate_limit = 1,
+            seconds = 60.0,
+        )
+
+        mail_client = AsyncMailClient(
+            email = "sender@gmail.com",
+            password = "secret_password",
+            server = "smtp.gmail.com",
+            rate_limiters = rate_lim
+        )
+
+        my_mail = MailMessage(
+            to_email = "recipient@gmail.com",
+            subject = "Bhopli is the best!"
+        )
+        my_mail.add_html(
+            """
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <meta charset="UTF-8">
+                <title>Sample HTML String</title>
+                <style>
+                    .heading {
+                        color: #ff9025;
+                    }
+                    .sub-heading {
+                        color: #000000;
+                    }
+                </style>
+            </head>
+            <body>
+                <h1 class="heading">Hello, Bhopli!</h1>
+                <h2 class="sub-heading">Bhopli is the best, most well-behaved cat in the known universe.</h2>
+            </body>
+            </html>
+            """
+        )
+        my_mail.add_text("This is how you should pet her 👇")
+        my_mail.add_inline_image(r"/path/to/image/cat_petting.png")
+        my_mail.add_attachment(r"/path/to/file/sample_label.pdf")
+
+        await mail_client.login()
+        result = await mail_client.send(my_mail)
+        print("MAIL RESULT:", json_utils.to_json_string(result))
+        await mail_client.logout()
+
+    asyncio.run(test())

network/ip.py

@@ -0,0 +1,173 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 12th Jul., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to get geolocation information of an IP address.
+
+    REFERENCES:
+
+        1) https://medium.com/@tubelwj/how-to-retrieve-ip-geolocation-information-in-python-929e15041e3e
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For working with IP Addresses:
+import ipaddress
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def ipv4_to_int(ip_string):
+
+    """
+    Converts an IP (v4) string to an integer value.
+    :param ip_string: The IP address (v4) that you want to convert to integer format.
+    :return: An integer representation of the IP (v4) address.
+    """
+
+    ip_numerical = int(ipaddress.IPv4Address(ip_string))
+    return ip_numerical
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def int_to_ipv4(ip_numerical):
+
+    """
+    Interprets the IP (v4) value from the given integer value.
+    :param ip_numerical: The integer value that represents an IP (v4) address.
+    :return:
+    """
+
+    ip_string = str(ipaddress.IPv4Address(ip_numerical))
+    return ip_string
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def ipv4_to_bin(ip_string):
+
+    """
+    Converts an input IP (v4) address to the binary string that represents the 32 bits.
+    :param ip_string: The IP (v4) string in a format like "192.168.0.1"
+    :return: The binary representation (as a string) of the input IP address.
+    """
+
+    ip_binary = bin(int(ipaddress.IPv4Address(ip_string)))[2:].zfill(32)
+    return ip_binary
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def bin_to_ipv4(ip_binary):
+
+    """
+    Interprets the IP (v4) value from the given binary string.
+    :param ip_binary: The string of 1s and 0s that represents the IP (v4) address.
+    :return: The IP (v4) address as a string.
+    """
+
+    ip_string = str(ipaddress.IPv4Address(int(ip_binary, 2)))
+    return ip_string
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_ipv4_range(ip_string, as_string = True):
+
+    """
+    Given a network description in the format "88.95.100.128/25", this function tells you the first and last IP
+    addresses of that network. Useful for determining if an IP address lies in a network.
+    :param ip_string: The input network description in the format "88.95.100.128/25"
+    :param as_string: To select between integer and string formats for the IP range output.
+    :return: The first and last IP addresses of the input network, and the count.
+    """
+
+    # Extract the components of the string:
+    ip_components = ip_string.split("/")
+    ip_addr = ipv4_to_int(ip_components[0])
+    ip_bits = int(ip_components[1])
+
+    # Convert the mask number to binary representation:
+    ip_mask = (1 << ip_bits) - 1
+    ip_mask = ip_mask << (32 - ip_bits)
+    inv_ip_mask = (~ip_mask) & 0xFFFF
+
+    # Figure out the start and end IP addresses:
+    start_ip = ip_addr & ip_mask
+    end_ip = ip_addr | inv_ip_mask
+    count = end_ip - start_ip + 1
+
+    # If the IPs are needed as strings, we perform the conversion:
+    if as_string:
+        start_ip = int_to_ipv4(start_ip)
+        end_ip = int_to_ipv4(end_ip)
+
+    # Done here:
+    return start_ip, end_ip, count
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    print(ipv4_to_int("255.255.255.255"))
+    print(ipv4_to_int("x.x.x.x"))

network/ping.py

@@ -0,0 +1,218 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 28th Jun, 2024
+
+    OBJECTIVE:
+
+        To provide a way to ping a server and get the traceroute dump.
+
+    REFERENCES:
+
+        1) https://www.geeksforgeeks.org/traceroute-implementation-on-python/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+# ---
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# Utils:
+# ---
+from utils import json_utils
+from utils import time_utils
+from utils import regex_utils
+
+# For networking:
+# ---
+import socket
+from scapy.all import *
+
+# For running the script from the terminal:
+# ---
+import argparse
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def get_name_and_addr(destination):
+
+    # If the user provided the IP address:
+    # ---
+    if (
+            regex_utils.match(destination, regex_utils.REGEX_IPV4) or
+            regex_utils.match(destination, regex_utils.REGEX_IPV6)
+    ):
+        try: destination_name = socket.gethostbyaddr(destination)[0]
+        except Exception as exception: destination_name = "*"
+        destination_ip = destination
+
+    # If the provided destination was the domain name:
+    # ---
+    else:
+        destination_name = destination
+        try: destination_ip = socket.gethostbyname(destination)
+        except Exception as exception: destination_ip = "*"
+
+    # Done here:
+    # ---
+    return destination_name, destination_ip
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def tracert(
+        destination,
+        max_hops = 30,
+        timeout = 2.0,
+        port = 33434
+):
+
+    # Initialize the variables:
+    # ---
+    destination_name, destination_ip = get_name_and_addr(destination)
+    full_trace = []
+    ttl = 1
+
+    # Keep noting hops till the limit is reached:
+    # ---
+    while ttl <= max_hops:
+
+        # Create a JSON for this stage:
+        # ---
+        this_hop = {
+            "destAddr": destination_ip,
+            "destName": destination_name,
+            "hopNo": ttl - 1,
+            "isDest": False,
+            "hopAddr": None,
+            "hopName": None,
+            "ts": None
+        }
+
+        # Create the IP and UDP headers and combine them:
+        # ---
+        ip_packet = IP(dst = destination, ttl = ttl)
+        udp_packet = UDP(dport = port)
+        trace_packet = ip_packet / udp_packet
+
+        # Send the packet and receive a reply and note down the timestamp:
+        # ---
+        reply = sr1(trace_packet, timeout = timeout, verbose = 0)
+        this_hop["ts"] = time_utils.get_current_utc_datetime(as_string = True)
+
+        # No response:
+        # ---
+        if reply is None: this_hop["hopAddr"] = this_hop["hopName"] = "*"
+
+        # If some response was received, we note the values and break out if this was the destination hop:
+        # ---
+        else:
+            this_hop["hopName"], this_hop["hopAddr"] = get_name_and_addr(f"{reply.src}")
+            if reply.type == 3:
+                this_hop["isDest"] = True
+                full_trace.append(this_hop)
+                break
+
+        # Carry on to the next hop:
+        # ---
+        full_trace.append(this_hop)
+        ttl += 1
+
+    # Done here:
+    # ---
+    return full_trace
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description = "Traceroute Implementation in Python!")
+
+    parser.add_argument(
+        "dest",
+        help = "Destination (Name or IP address)."
+    )
+    parser.add_argument(
+        "-m",
+        "--max-hops",
+        type = int,
+        default = 30,
+        help =
+        "Maximum number of hops (default: 30)."
+    )
+    parser.add_argument(
+        "-t",
+        "--timeout",
+        type = float,
+        default = 2.0,
+        help = "Timeout for each packet in seconds (default: 2.0)."
+    )
+    parser.add_argument(
+        "-p",
+        "--port",
+        type = int,
+        default = 33434,
+        help = "Timeout for each packet in seconds (default: 33434)."
+    )
+
+    args = parser.parse_args()
+
+    trace = tracert(
+        destination = args.dest,
+        max_hops = args.max_hops,
+        timeout = args.timeout,
+        port = args.port
+    )
+    print("TRACE:")
+    print(json_utils.to_json_string(trace))

queue/async_kafka.py

@@ -0,0 +1,554 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 13th Jul, 2024
+
+    OBJECTIVE:
+
+        To be able to work with keys
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To use Kafka:
+from aiokafka import AIOKafkaProducer
+from aiokafka import AIOKafkaConsumer
+
+# For working with JSON strings:
+from utils_v2.string import json
+from utils_v2.serialization.json_serializer import JSONSerializer
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+# For SSL security:
+import ssl
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def get_ssl_context(
+        ca_file,
+        cert_file,
+        key_file
+):
+
+    """
+    Generate the SSL context to use with the Kafka instances.
+    :param ca_file: The Certificate Authority file as a path to a local file.
+    :param cert_file: The Certificate file as a path to a local file.
+    :param key_file: The Key file as a path to a local file.
+    :return: The SSL context instance as a path to a local file.
+    """
+
+    ssl_context = ssl.create_default_context()
+    ssl_context.load_verify_locations(ca_file)
+    ssl_context.load_cert_chain(certfile = cert_file, keyfile = key_file)
+    return ssl_context
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class ProducerKafka:
+
+    def __init__(
+            self,
+            topic,
+            serializer = None,
+            debug = True,
+            debug_prefix = "Kafka (P) | ",
+            **kwargs
+    ):
+
+        """
+        Create a Kafka Producer.
+        :param topic: The topic to produce on.
+        :param serializer: The serializer to use.
+        :param debug: Whether, or not, you want to print the debug strings.
+        :param debug_prefix: The prefix to use while debugging.
+        :param kwargs: Any configuration parameters for the Kafka instances.
+        """
+
+        # Initialize the debugger:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # initialize the Kafka producer:
+        self.__topic = topic
+        self.__kwargs = kwargs
+        self.__producer = None
+        self.__connected = False
+        self.__serializer = serializer or JSONSerializer()
+
+        # For establishing connection:
+        self.__exclusive_semaphore = asyncio.Semaphore(1)
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    async def connect(self):
+
+        """
+        Connects to the Kafka server if not connected.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__exclusive_semaphore:
+            if not self.__connected:
+                try:
+                    self.__producer = AIOKafkaProducer(**self.__kwargs)
+                    await self.__producer.start()
+                    self.__connected = True
+                except Exception as exception: self.__printer(exception)
+            return self.__connected
+
+    async def ensure_connection(self):
+
+        """
+        Connects to the Kafka server if not connected.
+        :return: True or False based on the success of the operation.
+        """
+
+        if not self.__connected: await self.connect()
+        return self.__connected
+
+    async def close(self):
+
+        """
+        Terminates the connection.
+        :return: None.
+        """
+
+        if self.__connected:
+            try:
+                await self.__producer.stop()
+                self.__printer("Producer closed!")
+                self.__connected = False
+            except Exception as exception: self.__printer(exception)
+
+    async def produce(self, value, key = None, topic = None, encoding = "utf-8"):
+
+        """
+        Sends one message to the Kafka server on the topic that has been set for this instance.
+        :param value: The message to send.
+        :param key: The key to use when you want the messages to follow an order.
+        :param topic: A custom topic for this message, else the topic defined during the creation of this instance will
+            be used by default.
+        :param encoding: The encoding format.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure connectivity to the server.
+        # If not connected, return with failure immediately.
+        if not await self.ensure_connection(): return False
+
+        try:
+
+            # Send the message:
+            await self.__producer.send_and_wait(
+                topic = topic or self.__topic,
+                value = self.__serializer.serialize(data = value, encoding = encoding),
+                key = key
+            )
+
+            # Return with success if no exception occurred:
+            return True
+
+        # Return with failure if something went wrong:
+        except Exception as exception:
+            self.__printer(exception, self.__topic, type(value), value)
+            return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class ConsumerKafka:
+
+    def __init__(
+            self,
+            topic,
+            serializer = None,
+            debug = True,
+            debug_prefix = "Kafka (C) | ",
+            **kwargs
+    ):
+
+        """
+        Create a Kafka Consumer.
+        :param topic: The topic to consumer on.
+        :param serializer: The serializer to use.
+        :param debug: Whether, or not, you want to print the debug strings.
+        :param debug_prefix: The prefix to use while debugging.
+        :param kwargs: Any configuration parameters for the Kafka instances.
+        """
+
+        # Initialize the debugger:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # initialize the Kafka producer:
+        self.__topic = topic
+        self.__kwargs = kwargs
+        self.__consumer = None
+        self.__connected = False
+        self.__serializer = serializer or JSONSerializer()
+
+        # For establishing connection:
+        self.__exclusive_semaphore = asyncio.Semaphore(1)
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    async def connect(self):
+
+        """
+        Connects to the Kafka server if not connected.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__exclusive_semaphore:
+            if not self.__connected:
+                try:
+                    self.__consumer = AIOKafkaConsumer(self.__topic, **self.__kwargs)
+                    await self.__consumer.start()
+                    self.__connected = True
+                except Exception as exception: self.__printer(exception)
+            return self.__connected
+
+    async def ensure_connection(self):
+
+        """
+        Connects to the Kafka server if not connected.
+        :return: True or False based on the success of the operation.
+        """
+
+        if not self.__connected: await self.connect()
+        return self.__connected
+
+    async def close(self):
+
+        """
+        Terminates the connection.
+        :return: None.
+        """
+
+        if self.__connected:
+            try:
+                await self.__consumer.stop()
+                self.__printer("Consumer closed!")
+                self.__connected = False
+            except Exception as exception: self.__printer(exception)
+
+    async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"):
+
+        """
+        Get messages from the Kafka server.
+        :param count: The number of messages to get from the Kafka server.
+        :param timeout: The time in seconds to wait for retrieval.
+        :param encoding: The encoding to use.
+        :return: The messages that were received. If no messages are available, an empty list will be returned.
+        """
+
+        # Ensure connectivity to the server.
+        # If not connected, return with failure immediately.
+        if not await self.ensure_connection(): return []
+
+        # Make a variable that will hold the final results:
+        messages = []
+
+        try:
+
+            # Read some messages:
+            results = await self.__consumer.getmany(
+                max_records = max(1, count),
+                timeout_ms = int(timeout * 1_000)
+            )
+
+            # Format the received messages:
+            if results:
+                for topic_partition, records in results.items():
+                    for record in records:
+                        record_dict = record.__dict__
+                        record_dict["value"] = self.__serializer.deserialize(
+                            data = record_dict["value"],
+                            encoding = encoding
+                        )
+                        messages.append(record_dict)
+
+        # Debugging print if something went wrong:
+        except Exception as exception: self.__printer(exception)
+
+        # Done here:
+        return messages
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class BidirectionalKafka:
+
+    # The 'roles' that the instance can take.
+    # The master talks on the channel (topic) that the slave listens on and vice versa.
+    # Master-Slave is only for deciding who talks on which channel and who listens on which.
+    # In a two-party system, one must be the master, the other must be the slave.
+    # There are no extra privileges that the master enjoys. The naming convention was borrowed from common protocols
+    # used in electronics (like I2C).
+    ROLE_MASTER = 1
+    ROLE_SLAVE = 0
+
+    def __init__(
+            self,
+            role,
+            topic,
+            ack_topic: str = None,
+            group: str = None,
+            serializer = None,
+            debug = True,
+            debug_prefix = "Kafka (B) | ",
+            **kwargs
+    ):
+
+        """
+        Creates a walkie-talkie type setup to use Kafka in a bidirectional manner. Fo more information on all the
+        individual methods, please read through the doc-strings of the component classes 'ProducerKafka', and
+        'ConsumerKafka'.
+        :param role: Select from "ROLE_MASTER" and "ROLE_SLAVE". Between the two parties that are talking, one will be
+            the master and the other will be the slave. The channel that the master uses to speak will the one the slave
+            uses to listen, and vice versa.
+        :param topic: The topic to communicate on. Will be the same between the master and the slave.
+        :param ack_topic: Explicitly provide this for the second channel, or it will be created from the name of the
+            topic itself. Will be the same between the master and the slave.
+        :param group: The group to assign the instance to.
+        :param debug: Whether, or not, you want to print the debug strings.
+        :param debug_prefix: The prefix to use while debugging.
+        :param kwargs: Any configuration parameters for the Kafka instances.
+        """
+
+        # Not down the basic variables:
+        self.__role = role
+        self.__topic = topic
+        self.__ack_topic = ack_topic or topic + "Ack"
+        self.__group = group
+
+        # In case the current instance is the master,
+        # it will talk on "topic", and listen on "ack_topic":
+        if self.__role == self.ROLE_MASTER:
+            self.__producer_kwargs = kwargs.copy()
+            self.__producer = ProducerKafka(
+                topic = self.__topic,
+                serializer = serializer,
+                debug = debug,
+                debug_prefix = debug_prefix.strip() + " (P) | ",
+                **self.__producer_kwargs
+            )
+            self.__consumer_kwargs = kwargs.copy()
+            self.__consumer_kwargs["group_id"] = self.__group
+            self.__consumer = ConsumerKafka(
+                topic = self.__ack_topic,
+                group = group,
+                serializer = serializer,
+                debug = debug,
+                debug_prefix = debug_prefix.strip() + " (C) | ",
+                **self.__consumer_kwargs
+            )
+
+        # On the other hand, if the current instance is a slave,
+        # It will listen on "topic", and talk on "ack_topic":
+        else:
+            self.__producer_kwargs = kwargs.copy()
+            self.__producer = ProducerKafka(
+                topic = self.__ack_topic,
+                serializer = serializer,
+                debug = debug,
+                debug_prefix = debug_prefix.strip() + " (P) | ",
+                **self.__producer_kwargs
+            )
+            self.__consumer_kwargs = kwargs.copy()
+            self.__consumer_kwargs["group_id"] = self.__group
+            self.__consumer = ConsumerKafka(
+                topic = self.__topic,
+                group = group,
+                serializer = serializer,
+                debug = debug,
+                debug_prefix = debug_prefix.strip() + " (C) | ",
+                **self.__consumer_kwargs
+            )
+
+    def enable_debug(self):
+        self.__producer.enable_debug()
+        self.__consumer.enable_debug()
+
+    def disable_debug(self):
+        self.__producer.disable_debug()
+        self.__consumer.disable_debug()
+
+    async def ensure_connection(self):
+        await self.__producer.ensure_connection()
+        await self.__consumer.ensure_connection()
+
+    async def close(self):
+        await self.__producer.close()
+        await self.__consumer.close()
+
+    async def produce(self, message, encoding = "utf-8"):
+        return await self.__producer.produce(message, encoding = encoding)
+
+    async def consume(self, count = 1, timeout = 0.05, encoding = "utf-8"):
+        return await self.__consumer.consume(count = count, timeout = timeout, encoding = encoding)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    import time
+    from data_models.kafka_message import KafkaMessage
+
+    ssl_ctx = get_ssl_context(
+        ca_file = r"/home/developer/PycharmProjects/utils/cred/kafka/cert_authority.pem",
+        cert_file = r"/home/developer/PycharmProjects/utils/cred/kafka/fullchain.pem",
+        key_file = r"/home/developer/PycharmProjects/utils/cred/kafka/privkey.pem"
+    )
+
+    async def consumer_test():
+
+        consumer = ConsumerKafka(
+            topic = "kft_file_upload",
+            # group_id = "assessImg",
+            group_id = "updateMedia",
+            bootstrap_servers = "wtt.ditscentre.in:9092",
+            security_protocol = "SSL",
+            ssl_context = ssl_ctx
+        )
+        await consumer.connect()
+        await asyncio.sleep(1.5)
+        print("READY!")
+
+        while True:
+            messages = await consumer.consume(count = 1)
+            if len(messages) > 0: print("MESSAGE:", json.to_string(messages[0], default=str))
+            await asyncio.sleep(1.0)
+
+    async def producer_test():
+
+        producer = ProducerKafka(
+            topic = "kft_file_upload",
+            bootstrap_servers = "del.ditscentre.in:9092",
+            security_protocol = "SSL",
+            ssl_context = ssl_ctx
+        )
+        await producer.connect()
+        print("READY!")
+
+        while True:
+            my_msg = KafkaMessage(
+                data = {
+                    "accepted": False,
+                    "reason": "low resolution"
+                },
+                media = {
+                    "name": "pikachu_poster.jpg",
+                    "ext": "jpg",
+                    "url": "https://nexcom.ditscentre.in/utils/files/small/download/66ded1c1c1c05139a618b5ff",
+                    "attr": {
+                        "user": "SarangKabir",
+                        "project": "ACE-PGP",
+                        "id": 173,
+                        "campaignActivityId": "25",
+                        "idCampaign": 49,
+                        "phoneNo": "7977821877"
+                    }
+                },
+                appId = "aceWockhardt",
+                proc = {
+                    "name": "_assessImg",
+                    "attr": {
+                        "blurThreshold": 0.25,
+                        "clarityThreshold": 0.65,
+                        "nsfwThreshold": 0.25,
+                        "minWidth": 512,
+                        "minHeight": 512
+                    }
+                },
+                ack = None
+            )
+            success = await producer.produce(my_msg.model_dump())
+            print("produced...")
+            time.sleep(1.0)
+            break
+
+        await producer.close()
+
+
+    asyncio.run(consumer_test())

rate_limit/async_token_bucket.py

@@ -0,0 +1,212 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Wednesday, 15th May, 2024
+
+    OBJECTIVE:
+
+        To provide a rate-limiting mechanism using the simple token bucket algorithm.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For date and time keeping:
+import datetime
+import time
+
+# For asynchronous activities:
+import asyncio
+
+# For mathematical operations:
+import math
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class TokenBucket:
+
+    """
+    MODES: "add" - Add 'rate_limit' no. of tokens to the bucket every interval.
+           "reset" - Reset the token count to 'rate_limit' every interval
+    """
+
+    MODE_RESET = 0
+    MODE_ADD = 1
+
+    ONE_YEAR = 3_15_36_000.0
+    ONE_MONTH = 26_78_400.0
+    ONE_WEEK = 6_04_800.0
+    ONE_DAY = 86_400.0
+    ONE_HOUR = 3_600.0
+    ONE_MINUTE = 60.0
+    ONE_SECOND = 1.0
+
+    def __init__(self, rate_limit, seconds = 1.0, mode = "reset", sleep = 0.1, in_sequence = False):
+
+        """
+        Initialize the rate controller.
+        :param rate_limit: The no. of operations allowed per unit of time.
+        :param seconds: The time period in seconds in which the tokens get reset or added.
+        :param mode: To select what happens when the period is over. 'reset' mode means that the remaining tokens from
+            the previous period are discarded and the counter is set back to the rate limit, and 'add' mode means that
+            new tokens are added on top of the exiting ones that were unused in the previous period.
+        :param sleep: The delay to add before checking back to see if tokens are available. Keep it longer for longer
+            periods (which is adjusted by the 'seconds' parameter).
+        :param in_sequence: Whether, or not, you want to maintain the sequence in which the requests were made
+            (experimental).
+        """
+
+        if rate_limit is not None:
+            rate_limit = int(rate_limit)
+            self.__semaphore = asyncio.Semaphore(1 if in_sequence else rate_limit)
+        mode = mode.lower()
+        if mode not in [self.MODE_RESET, self.MODE_ADD]: mode = self.MODE_RESET
+        self.__mode = mode
+        self.__rate_limit = rate_limit
+        self.__token_count = rate_limit
+        self.__last_token_generation_time = time.time()
+        self.__sleep = sleep
+        self.__seconds = seconds
+
+    async def has_turn(self):
+
+        """
+        Just to check if a token is available.
+        Should be used only if you don't want to wait for turn in case it is unavailable.
+        :return: True if the token is available, False if not.
+        """
+
+        # If the user doesn't want any rate-limits, we return immediately.
+        # Otherwise, we check if the user has any tokens available:
+        if self.__rate_limit is None: return True
+        if self.__token_count > 0: return True
+        else: return False
+
+    async def get_turn(self, timeout = None):
+
+        """
+        To wait till either you get your turn or the wait gets timed-out.
+        :param timeout: The time (in seconds) to wait to get a turn before exiting with failure.
+        :return: True if a turn was received, else False if timed-out.
+        """
+
+        # If the user doesn't want any rate-limits, we return immediately:
+        if self.__rate_limit is None: return
+
+        # Start by making variables:
+        got_turn = False
+        wait_start = time.time()
+
+        # Otherwise, we wait for the semaphore:
+        async with self.__semaphore:
+
+            # Wait till you get your turn or the attempt gets timed-out:
+            while True:
+
+                # If the timeout has been exceeded:
+                if (
+                        timeout is not None and
+                        time.time() - wait_start >= timeout
+                ): break
+
+                # If it is time to generate new tokens:
+                time_delta = time.time() - self.__last_token_generation_time
+                if time_delta >= self.__seconds:
+                    if self.__mode == self.MODE_RESET: self.__token_count = self.__rate_limit
+                    else: self.__token_count += int(self.__rate_limit * math.floor(time_delta / self.__seconds))
+                    self.__last_token_generation_time = time.time()
+
+                # If a token is available (or not):
+                if self.__token_count > 0:
+                    self.__token_count -= 1
+                    got_turn = True
+                    break
+                else: await asyncio.sleep(self.__sleep)
+
+        # Done here:
+        return got_turn
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import random
+
+    async def single_test(rate_controller, count):
+        has_turn = await rate_controller.has_turn()
+        got_turn = await rate_controller.get_turn()
+        print(f"TURN: {count:.<5} {'Y' if has_turn else '-'} / {'Y' if got_turn else '-'} ({datetime.datetime.now()})")
+        if got_turn: await asyncio.sleep(random.random())
+
+    async def multi_test(max_count):
+        rate_controller = TokenBucket(
+            rate_limit = 10,
+            seconds = 1,
+            in_sequence = False,
+            sleep = 0.025,
+            mode = "reset",
+            timeout = 3
+        )
+        tasks = [single_test(rate_controller, count + 1) for count in range(max_count)]
+        await asyncio.gather(*tasks)
+
+
+    start_time = time.time()
+    asyncio.run(multi_test(100))
+    print(f"FINISHED IN {time.time() - start_time} SECONDS!")

security/hash.py

@@ -0,0 +1,245 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 26th Aug., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to hash inputs.
+
+    REFERENCES:
+
+        1) Book: Full Stack Python Security - Dennis Byrne
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# For hashing:
+import hashlib
+import hmac
+import secrets
+from bcrypt import hashpw, gensalt
+
+# To work with buffers:
+import io
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class Hasher:
+
+    def __init__(
+            self,
+            algorithm = hashlib.sha256,
+            key = None
+    ):
+
+        """
+        hashes data or a message or a file. Uses HMAC if 'key' is specified, else performs simple hashing.
+        :param algorithm: The algorithm to use. SHA256 by default.
+        :param key: Specify this as either a string or as bytes to use HMAC. Leave as null for simple hashing.
+        """
+
+        # Initialize the hasher:
+        self.__hasher = None,
+        self.__algorithm = algorithm
+        self.__hmac_key = key
+        if self.__hmac_key is not None:
+            self.__hmac_key = self.__hmac_key.encode("utf-8") if isinstance(self.__hmac_key, str) else self.__hmac_key
+        self.reset()
+
+    @staticmethod
+    def generate_key(byte_count = 32, url_safe = False):
+
+        """
+        A mechanism to generate key/salt values.
+        NOTE: For a proper salt for passwords, I recommend using "generate_salt" method. It's far better.
+        :param byte_count: The number of bytes to have in the key. The hex output (as string) will have 2x the
+            characters.
+        :param url_safe: Set to True if you need the generated output to be a part of a URL.
+        :return: The generate key/salt.
+        """
+
+        return secrets.token_urlsafe(byte_count) if url_safe else secrets.token_hex(byte_count)
+
+    @staticmethod
+    def generate_salt():
+
+        """
+        Generate a salt to use while hashing things like passwords.
+        :return: The salt as bytes.
+        """
+
+        return gensalt()
+
+    @staticmethod
+    def hash_password(password, salt):
+
+        """
+        Hashes a password with the given salt.
+        :param password: The password to hash, either as a string or as bytes.
+        :param salt: The salt to hash the password with, either as a string or as bytes.
+        :return: The hashed string.
+        """
+
+        return hashpw(
+            password = password.encode("utf-8") if isinstance(password, str) else password,
+            salt = salt.encode("utf-8") if isinstance(salt, str) else salt
+        )
+
+    @staticmethod
+    def compare_hashes(hash_0, hash_1):
+
+        """
+        Compares two hashes to see if they match.
+        Comparison is done in constant time to avoid timing-based side-channel attacks.
+        :param hash_0: One of the hashes to compare.
+        :param hash_1: The other hash to compare.
+        :return: True if they match, else False.
+        """
+
+        return hmac.compare_digest(hash_0, hash_1)
+
+    def reset(self):
+
+        """
+        Resets the hasher by removing all the data that was fed into it.
+        :return: None.
+        """
+
+        if self.__hmac_key is not None:
+            self.__hasher = hmac.new(
+                key = self.__hmac_key,
+                digestmod = self.__algorithm
+            )
+        else: self.__hasher = self.__algorithm()
+
+    def update(self, data):
+
+        """
+        Adds data to the hash to update it.
+        :param data: The data to be hashed.
+        :return: None.
+        """
+
+        data = data.encode("utf-8") if isinstance(data, str) else data
+        self.__hasher.update(data)
+
+    def digest(self):
+
+        """
+        Returns the hexadecimal representation of the hash as a string.
+        :return: The hexadecimal representation of the hash as a string
+        """
+
+        return self.__hasher.digest()
+
+    def hexdigest(self):
+
+        """
+        Returns the hexadecimal representation of the hash as a string.
+        :return: The hexadecimal representation of the hash as a string
+        """
+
+        return self.__hasher.hexdigest()
+
+    def hash_message(self, message, as_hex = True):
+
+        """
+        Hashes one message and returns the result, and then resets the instance.
+        :param message: The data you want to hash.
+        :param as_hex: Invokes 'hexdigest' if True, else 'digest'.
+        :return: The hash of the message in either hexadecimal string form or binary form.
+        """
+
+        self.update(message)
+        hash_result = self.hexdigest() if as_hex else self.digest()
+        self.reset()
+        return hash_result
+
+    def hash_file(self, file, chunk_size = 4096, as_hex = True):
+
+        """
+        Hashes one file and returns the result, and then resets the instance.
+        :param file: The file you want to hash either as a path or as some buffer (like io.BytesIO).
+        :param chunk_size: The size of data (in bytes) that you would like to pick at one time.
+        :param as_hex: Invokes 'hexdigest' if True, else 'digest'.
+        :return: The hash of the file in either hexadecimal string form or binary form.
+        """
+
+        # In case the file was given as a io.BytesIO buffer:
+        if isinstance(file, io.BytesIO):
+            file.seek(0)
+            while True:
+                chunk = file.read(chunk_size)
+                if not chunk: break
+                self.update(chunk)
+
+        # In case the file was given as a path:
+        else:
+            with open(file, "rb") as f:
+                for chunk in iter(lambda: f.read(chunk_size), b""):
+                    self.update(chunk)
+
+        # Now we capture the results, reset the instance, and return the result:
+        hash_result = self.hexdigest() if as_hex else self.digest()
+        self.reset()
+        return hash_result
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    print(
+        Hasher.hash_password(
+            password = "mic test, mic test, 123",
+            salt = Hasher.generate_salt()
+        )
+    )

security/otp.py

@@ -0,0 +1,156 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 10th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a way to generate and verify OTPs.
+
+    REFERENCES:
+
+        01. https://pyauth.github.io/pyotp/#
+        02. https://en.wikipedia.org/wiki/Google_Authenticator
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To work with OTPs:
+import pyotp
+import base64
+import hashlib
+
+# To work with date and time:
+import time
+import datetime
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class HashedOTP:
+
+    def __init__(
+            self,
+            secret
+    ):
+
+        """
+        Used to generate and verify HMAC-based OTPs.
+        :param secret: The key to use to generate and verify OTPs.
+        """
+
+        # Note down the input variables:
+        self.__secret = secret
+        self.__otp_client = pyotp.HOTP(secret)
+
+    @staticmethod
+    def generate_secret(message = None):
+
+        """
+        Generate a secret key to then use to generate and verify the OTPs.
+        You may override the random generator by giving a "message" of any length.
+        :param message: A custom value to convert into a key. Avoid using this for better security, but this can be used
+            to generate keys based on user identifiers. THERE IS NO RANDOMNESS IF YOU USE THIS FEATURE. IT IS FOR
+            CONVENIENCE ONLY. NOT RECOMMENDED.
+        :return: The key (as a string) that can be used to generate and verify the OTPs.
+        """
+
+        # If the user wants to generate a key from a custom input:
+        if message:
+
+            # Ensure we have a bytes object:
+            if not isinstance(message, (str, bytes)): message = str(message)
+            if isinstance(message, str): message = message.encode("utf-8")
+
+            # Hash the bytes object:
+            sha256_hash = hashlib.sha256()
+            sha256_hash.update(message)
+            hashed_key = sha256_hash.digest()
+
+            # Convert to base-32:
+            return base64.b32encode(hashed_key).decode("utf-8")
+
+        # If the user wants a totally random key:
+        else: return pyotp.random_base32()
+
+    def generate_otp(self, count: int):
+
+        """
+        Generates the OTP at a particular step.
+        :param count: The step at which the OTP needs to be generated.
+        :return: The OTP string (6 digits).
+        """
+
+        return str(self.__otp_client.at(count))
+
+    def verify_otp(self, otp, count: int):
+
+        """
+        Verifies the claimed OTP.
+        :param otp: The OTP as claimed by the end user.
+        :param count: The step at which the OTP needs to be verified.
+        :return: True if the OTP is valid, else False.
+        """
+
+        return self.__otp_client.verify(otp, counter = count)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

security/sanitizers.py

@@ -0,0 +1,135 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 25th Jul., 2024
+
+    OBJECTIVE:
+
+        To provide a set of data cleaning functions for inputs like phone numbers, emails, etc.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# My utils:
+from utils_v2.string import regex
+
+# For random strings and tokens:
+import string
+import random
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def file_name(input_string: str):
+
+    """
+    Cleans up the string to allow it to safely become a filename.
+    :param input_string: The string that you want to make safe for using as a filename.
+    :return: The string that can safely be used as a filename.
+    """
+
+    return regex.replace(
+        text = input_string.replace("\n", " "),
+        pattern = r"[^a-zA-Z0-9 \-_\.]",
+        substitute_text = ""
+    )
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def for_mongo(input_document):
+
+    """
+    Sanitizes and disarms any JSON-like input that could be used for NoSQL-injection attacks.
+    :param input_document: The list or dict to be sanitized.
+    :return: The sanitized list or dict.
+    """
+
+    # A special function that disarms any input string by dealing with special characters
+    # that Mongo may consider to be instructions:
+    def disarm(input_string):
+        input_string = regex.replace(
+            text = input_string,
+            pattern = r"[^a-zA-Z0-9,_\-\.\\\/:;'\(\) ]",
+            substitute_text = ""
+        )
+        return input_string
+
+    # Initially we assign the value of the input to the output:
+    sanitized_document = input_document
+
+    # Handle the case where the input is an array:
+    if isinstance(input_document, list):
+        sanitized_document = [for_mongo(document) for document in input_document]
+
+    # Handle the case when the input is a document:
+    elif isinstance(input_document, dict):
+        sanitized_document = {}
+        for k, v in input_document.items():
+            sanitized_document[disarm(k)] = v if type(v) not in [list, dict] else for_mongo(v)
+
+    # Done here:
+    return sanitized_document
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

serialization/json_serializer.py

@@ -0,0 +1,129 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Sunday 1st Sept. 2024.
+
+    OBJECTIVE:
+
+        To provide a way to convert any input data to serialized bytes, and back.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To work with files:
+from utils_v2.string import json
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class JSONSerializer:
+
+    def __init__(self):
+
+        """
+        Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and
+        python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour.
+        """
+
+        pass
+
+    @staticmethod
+    def serialize(data, encoding = "utf-8"):
+
+        """
+        Serializes the data that is given to it.
+        The input has to be JSON-compatible.
+        :param data: The data to serialize.
+        :param encoding: The encoding to use.
+        :return: The bytes representing the data.
+        """
+
+        # If the data is not already a JSON string, parse it. Then return it as bytes:
+        data = data if isinstance(data, str) else json.to_string(data, no_space = True)
+        return data.encode(encoding)
+
+    @staticmethod
+    def deserialize(data, encoding = "utf-8"):
+
+        """
+        Deserializes the bytes that are given to it.
+        The input has to be JSON-compatible.
+        :param data: The bytes to deserialize.
+        :param encoding: The encoding to use.
+        :return: The data from the bytes that described it.
+        """
+
+        data = data.decode(encoding)
+        return json.from_string(data)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

serialization/universal_serializer.py

@@ -0,0 +1,265 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Sunday 1st Sept. 2024.
+
+    OBJECTIVE:
+
+        To provide a way to convert any input data to serialized bytes, and back.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To work with files:
+from utils_v2.string import json
+
+# To work with tabulated data:
+import pandas as pd
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class UniversalSerializer:
+
+    def __init__(self):
+
+        """
+        Use this when you are working with varied datatypes. You can add custom data-converters also using the
+        'add_converters' method. Otherwise, most default pythonic datatypes are supported out of the box. Note that this
+        is NOT recommended because of how large the serialized messages become. Try using 'JSONSerializer' when you know
+        you will be working specifically with JSOn-compatible inputs.
+        """
+
+        # These are the converters to use when serializing data:
+        self.__forward_converters = {
+            "set": lambda x: list(x),
+            "tuple": lambda x: list(x),
+            "complex": lambda x: {"r": x.real, "i": x.imag},
+            "DataFrame": lambda x: x.to_dict()
+        }
+
+        # These are the converters to use when deserializing data:
+        self.__reverse_converters = {
+            "set": lambda x: set(x),
+            "tuple": lambda x: tuple(x),
+            "complex": lambda x: complex(x["r"], x["i"]),
+            "DataFrame": lambda x: pd.DataFrame.from_dict(x)
+        }
+
+    def add_converters(
+            self,
+            type_name,
+            forward_converter_func,
+            reverse_converter_func
+    ):
+
+        """
+        Add custom datatype converters.
+        RULES:
+            01. Each of the converter functions must take in exactly on argument and return one output of native python
+                type. This is very important.
+            02. Each forward and reverse converters must give symmetric results.
+        :param type_name: The name of the datatype. HINT: type(obj).__name__
+        :param forward_converter_func: The function to handle conversion to bytes. Use when serializing.
+        :param reverse_converter_func: The function to handle conversion from bytes. Used when deserializing.
+        :return: None.
+        """
+
+        self.__forward_converters[type_name] = lambda x: forward_converter_func(x)
+        self.__reverse_converters[type_name] = lambda x: reverse_converter_func(x)
+
+    def __describe(self, data):
+
+        """
+        Notes down the input datatypes of everything.
+        Does everything upto conversion to byes.
+        :param data: The data to process.
+        :return: The description of the datatypes and values of what was given.
+        """
+
+        # Note down the type of data that was sent as the input:
+        data_type = type(data).__name__
+
+        # Handle iterables:
+        if isinstance(data, list): data = [self.__describe(item) for item in data]
+        elif isinstance(data, set): data = [self.__describe(item) for item in data]
+        elif isinstance(data, tuple): data = [self.__describe(item) for item in data]
+        elif isinstance(data, dict): data = [
+            {
+                "k": self.__describe(k),
+                "v": self.__describe(v)
+            } for k, v in data.items()
+        ]
+
+        # Convert here, and return:
+        conv = self.__forward_converters.get(data_type)
+        if conv is not None: data = conv(data)
+        return {"d": data, "t": data_type}
+
+    def serialize(self, data, encoding = "utf-8"):
+
+        """
+        Serializes the data that is given to it.
+        :param data: The data to serialize.
+        :param encoding: The encoding to use.
+        :return: The bytes representing the data.
+        """
+
+        data = self.__describe(data)
+        data = json.to_string(data, no_space = True)
+        return data.encode(encoding)
+
+    def __interpret(self, data):
+
+        """
+        Interprets the types of data that were serialized originally.
+        :param data: The data in the serialized form.
+        :return: Data where the appropriate datatypes have been applied.
+        """
+
+        # Handle iterables:
+        if data["t"] == "list": data = [self.__interpret(item) for item in data["d"]]
+        elif data["t"] == "set": data = set([self.__interpret(item) for item in data["d"]])
+        elif data["t"] == "tuple": data = tuple([self.__interpret(item) for item in data["d"]])
+        elif data["t"] == "dict": data = {
+            self.__interpret(item["k"]): self.__interpret(item["v"])
+            for item in data["d"]
+        }
+
+        # Handle custom types:
+        else:
+            conv = self.__reverse_converters.get(data["t"])
+            data = data["d"]
+            if conv is not None: data = conv(data)
+
+        # Done here
+        return data
+
+    def deserialize(self, data, encoding = "utf-8"):
+
+        """
+        Deserializes the bytes that are given to it.
+        :param data: The bytes to deserialize.
+        :param encoding: The encoding to use.
+        :return: The data from the bytes that described it.
+        """
+
+        data = data.decode(encoding)
+        data = json.from_string(data)
+        return self.__interpret(data)
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class JSONSerializer:
+
+    def __init__(self):
+
+        """
+        Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and
+        python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour.
+        """
+
+        pass
+
+    @staticmethod
+    def serialize(data, encoding = "utf-8"):
+
+        """
+        Serializes the data that is given to it.
+        The input has to be JSON-compatible.
+        :param data: The data to serialize.
+        :param encoding: The encoding to use.
+        :return: The bytes representing the data.
+        """
+
+        # If the data is not already a JSON string, parse it. Then return it as bytes:
+        data = data if isinstance(data, str) else json.to_string(data, no_space = True)
+        return data.encode(encoding)
+
+    @staticmethod
+    def deserialize(data, encoding = "utf-8"):
+
+        """
+        Deserializes the bytes that are given to it.
+        The input has to be JSON-compatible.
+        :param data: The bytes to deserialize.
+        :param encoding: The encoding to use.
+        :return: The data from the bytes that described it.
+        """
+
+        data = data.decode(encoding)
+        return json.from_string(data)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

sms/async_nimbus.py

@@ -0,0 +1,252 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 9th Sept., 2024
+
+    OBJECTIVE:
+
+        To be able to send SMSs from Nimbus's API and manage the templates and other things from one place.
+
+    REFERENCES:
+
+        01. https://github.com/innovativevijay/SmsHitApiSample
+        02. https://nimbusit.net/appforms/apimanual.php
+
+    DOWNLOADS:
+
+        N/A
+
+    WEB-PORTAL:
+
+        01. http://nimbusit.net/
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To make API Calls:
+import httpx
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncNimbusSMS:
+
+    MESSAGE_TYPE_REGULAR = 0
+    MESSAGE_TYPE_UNICODE = 1
+
+    def __init__(
+            self,
+            entity_id,
+            sender_id,
+            user_id,
+            api_key,
+            debug = True,
+            debug_prefix = "Nimbus SMS | "
+    ):
+
+        """
+        Sets up an instance of the SMS sender through Nimbus IT.
+        :param entity_id: The entity id as registered with DLT.
+        :param sender_id: The 6-char code like "HDFCBK", "NSESMS", "ZRODHA" that you see in your SMS inbox.
+        :param user_id: The 6-digit id that Nimbus has assigned to you.
+        :param api_key: The API key generated through Nimbus's portal.
+        :param debug: Whether, or not, you would like to show debugging messages (can be changed on the fly).
+        :param debug_prefix: The prefix text to show with the debug string.
+        """
+
+        # Create the debugging tools:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # Create an HTTP client to work with:
+        limits = httpx.Limits(
+            max_connections = 5,
+            max_keepalive_connections = 5,
+            keepalive_expiry = 3600
+        )
+        self.__http_client = httpx.AsyncClient(limits = limits, timeout = 120)
+
+        # Capture the input config:
+        self.__entity_id = entity_id
+        self.__sender_id = sender_id
+        self.__user_id = user_id,
+        self.__api_key = api_key
+
+    async def get_balance(self):
+
+        """
+        Checks the balance in the Nimbus wallet.
+        :return: The balance (float) if the request was successful, else None.
+        """
+
+        balance = None
+
+        try:
+
+            # Call the API:
+            response = await self.__http_client.get(
+                url = r"http://nimbusit.net/api/balance",
+                params = {"user": self.__user_id, "authkey": self.__api_key}
+            )
+
+            # The response of a successful API call looks like "BALANCE:599". We need just the number:
+            if response.status_code in [200]: balance = float(response.content.decode().split(":")[-1].strip())
+
+        except Exception as exception:
+            self.__printer(exception)
+
+        return balance
+
+    async def send_sms(
+            self,
+            template_id,
+            recipient_number,
+            message,
+            message_type = MESSAGE_TYPE_REGULAR
+    ):
+
+        """
+        Sends one SMS through Nimbus IT's system. The text of the message must match the template that had been
+        submitted. A mismatch may cause the message to fail at best, and raise troubles in the real-world with
+        government bodies at worst. Be careful.
+        :param template_id: The id of the SMS template as registered on Nimbus's portal.
+        :param recipient_number: The phone number of the recipient. You can send an array of numbers, too, BUT IT IS
+            STRONGLY RECOMMENDED TO NOT DO THAT TO AVOID BEING BLOCKED BY DLT.
+        :param message: The message to send to the recipient. Should match the template that is being sent.
+        :param message_type: Choose between 'AsyncNimbusSMS.MESSAGE_TYPE_REGULAR' (default) and
+            'AsyncNimbusSMS.MESSAGE_TYPE_UNICODE' based on the type of characters being sent. Both are class variables.
+        :return: The dict of all the details of the message that was sent including whether, or not, it was successfully
+            sent. Other details depend on the service provider (Nimbus IT in this case).
+        """
+
+        # Construct the basic structure of the response of this method:
+        summary = {
+            "success": False,
+            "info": None,
+            "sender": self.__sender_id,
+            "recipient": recipient_number,
+            "message": message,
+            "length": len(message),
+            "template_id": template_id,
+            "raw": None
+        }
+
+        try:
+
+            # Pre-process the recipient's number:
+            if not isinstance(recipient_number, (list, set, tuple)): recipient_number = [recipient_number]
+
+            # Call the API:
+            response = await self.__http_client.get(
+                url = r"http://nimbusit.net/api/pushsms",
+                params = {
+                    "user": self.__user_id,
+                    "authkey": self.__api_key,
+                    "sender": self.__sender_id,
+                    "mobile": ",".join([str(num) for num in recipient_number]),
+                    "text": message,
+                    "entityid": self.__entity_id,
+                    "templateid": template_id,
+                    "type": message_type
+                }
+            )
+
+            # For a successful API call:
+            if response.status_code == 200:
+                response_json = response.json()
+                summary["success"] = True if response_json.get("STATUS", "ERROR").lower() in ["ok"] else False
+                summary["info"] = response_json.get("RESPONSE", {}).get("INFO")
+                summary["raw"] = {
+                    "http_code": response.status_code,
+                    "response": response_json,
+                }
+
+            # For any other code that indicates some form of failure:
+            else: summary["raw"] = {
+                "http_code": response.status_code,
+                "response": response.content.decode()
+            }
+
+        except Exception as exception:
+            self.__printer(exception)
+
+        return summary
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+    async def main():
+
+        sender = AsyncNimbusSMS(
+            entity_id = "<your_entity_id>",
+            sender_id = "<your_sender_id>",
+            user_id = "<your_nimbus_user_id>",
+            api_key = "<your_nimbus_api_key>"
+        )
+
+        response = await sender.send_sms(
+            template_id = "<your_sms_template_id>",
+            recipient_number = "<the_number_you_want_to_send_the_message_to>",
+            message = "<your_sms_message>"
+        )
+        print("SMS API RESPONSE:", response)
+
+        my_balance = await sender.get_balance()
+        print("REMAINING BALANCE:", my_balance)
+
+
+    asyncio.run(main())

string/fuzzy.py

@@ -0,0 +1,168 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 18th May, 2024
+
+    OBJECTIVE:
+
+        To provide a quick set of functions to work with fuzzy logic.
+
+    REFERENCES:
+
+        1) https://www.w3schools.com/python/python_json.asp
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To apply fuzzy logic:
+from thefuzz import fuzz, process
+
+# To work with tabulated data:
+import pandas as pd
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def get_best_match(
+        target,
+        choices,
+        threshold = 0.70,
+        partial = False
+):
+
+    if partial: scorer = fuzz.partial_token_sort_ratio
+    else: scorer = fuzz.ratio
+
+    result = process.extractOne(
+        target,
+        choices,
+        score_cutoff = threshold * 100,
+        scorer = scorer
+    )
+
+    try: return result[0]
+    except: return None
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def rank(target, choices, partial = True):
+
+    if partial: scorer = fuzz.partial_token_sort_ratio
+    else: scorer = fuzz.ratio
+
+    result = process.extract(
+        target,
+        choices,
+        limit = len(choices),
+        scorer = scorer
+    )
+
+    result = pd.DataFrame(result, columns = ["choice", "closeness"])
+    result["closeness"] = result["closeness"] / 100.0
+
+    return result
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def match(targets, choices, threshold = 0.7, partial = False, allow_null = False):
+
+    all_matches_df = None
+    all_matches = {target: None for target in targets}
+    something_is_null = False
+
+    for target in targets:
+        match_df = rank(target, choices, partial = partial)
+        match_df["target"] = target
+        if all_matches_df is None: all_matches_df = match_df
+        else: all_matches_df = pd.concat([all_matches_df, match_df])
+
+    all_matches_df = all_matches_df.sort_values(by = ["closeness"], ascending = False).reset_index(drop = True)
+
+    for target in targets:
+        target_df = all_matches_df[all_matches_df["target"] == target].reset_index(drop = True)
+        if target_df.empty: continue
+        if target_df.at[0, "closeness"] >= threshold:
+            choice = target_df.at[0, "choice"]
+            all_matches[target] = choice
+            all_matches_df = all_matches_df[all_matches_df["choice"] != choice]
+        else:
+            all_matches[target] = None
+            something_is_null = True
+
+    # print(all_matches)
+    if something_is_null and not allow_null: return None
+    else: return all_matches
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import async_json_utils
+
+    awb_numbers = [
+        "SF1111BIC",
+        "SF2222BIC",
+        "SF3333BIC",
+        "SF4444BIC",
+    ]
+
+    chat_text = "SF1112BIC"
+
+    # print(chat_text == names[0])
+    best_match = get_best_match(chat_text, awb_numbers, threshold = 0.60, partial = False)
+    print(f"Best match for '{chat_text}' is '{best_match}'")

string/help.py

@@ -0,0 +1,177 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 24th Aug., 2024
+
+    OBJECTIVE:
+
+        To provide an overview of any function or class in a string.
+        The generated overview can then either be shown on the terminal, or transmitted over some other medium for
+        collaborative work.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# System-level activities:
+import io
+import inspect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def get_help_for_class(cls, skip_methods = None):
+
+    """
+    Returns the help documentation to use this class.
+    :param cls: The class whose help string is desired.
+    :param skip_methods: A list of methods to NOT include in the help text.
+    :return: This help documentation.
+    """
+
+    separator = "\n\n" + ("=" * 120) + "\n\n"
+    if skip_methods is None: skip_methods = []
+    elif not isinstance(skip_methods, list): skip_methods = [skip_methods]
+
+    # Get class name and docstring:
+    class_name = cls.__name__
+    docstring = inspect.getdoc(cls) or ""
+    help_string = "HELP FOR:\n\n"
+    help_string += class_name + "\n\n"
+    help_string += "This document has upto 120 chars per line.\n"
+    help_string += "Best viewed with monospaced font :)"
+    help_string += docstring + separator
+
+    # Get all methods and their docstrings.
+    # Then note the documentation of the methods while ignoring the blacklisted ones:
+    members = inspect.getmembers(cls, predicate = inspect.isfunction)
+    func_help = []
+    for name, method in members:
+
+        # Ignore if asked, or extract the details:
+        if name in skip_methods or name.startswith(f"_{class_name}__"): continue
+        else: func_help.append(get_help_for_function(method))
+
+    # Put all the things together:
+    func_help = separator.join(func_help)
+    help_string += func_help
+
+    # Done here:
+    return help_string
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_help_for_function(func):
+
+    """
+    Get the help string for one function.
+    It could be a standalone function, or a method of a class.
+    :param func: The function (or method) whose help string is needed.
+    :return: The help string of the function.
+    """
+
+    # Get the name and documentation:
+    func_name = func.__name__
+    async_indicator = "  (async)" if inspect.iscoroutinefunction(func) else ""
+    func_doc = inspect.getdoc(func) or ""
+
+    # Create the decorated header:
+    func_decorator = "-" * (len(func_name) + 2)
+    func_head = "." + func_decorator + f".\n| {func_name} |{async_indicator}\n`" + func_decorator + "`\n\n"
+
+    # Add the 'args' and 'kwargs':
+    func_args = []
+    for name, param in inspect.signature(func).parameters.items():
+        default = param.default
+        if isinstance(default, str): default = f"\"{default}\""
+        if default == inspect.Parameter.empty: func_args.append(f"{name}")
+        else: func_args.append(f"{name}: {type(default).__name__} = {default}")
+    if len(func_args) > 0: func_args = f"{func_name} (\n\t" + "\n\t".join(func_args) + "\n):\n\n"
+    else: func_args = f"{func_name} ():\n\n"
+
+    # Get the params and return value part from the doc:
+    params_start = func_doc.find(":param")
+    return_start = func_doc.find(":return")
+    func_params = "\n" + func_doc[params_start:return_start] if params_start >= 0 else ""
+    func_return = "\n" + func_doc[return_start:] if return_start >= 0 else ""
+
+    # Isolate the documentation part:
+    if params_start >= 0: func_doc = func_doc[:params_start]
+    elif return_start >= 0: func_doc = func_doc[:return_start]
+
+    # Done here:
+    return func_head + func_args + func_doc + func_params + func_return
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_help(entity, skip_methods = None):
+
+    """
+    Get the help documentation for anything from its docstring.
+    :param entity: The entity you want to get help for.
+    :param skip_methods: A list of methods to ignore if inspecting a class. Not valid for standalone functions.
+    :return: The help string.
+    """
+
+    if inspect.isclass(entity): return get_help_for_class(entity, skip_methods = skip_methods)
+    else: return get_help_for_function(entity)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

string/json.py

@@ -0,0 +1,202 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Create: Saturday, 18th May, 2022
+        Update: Thursday, 22nd Aug. 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to work with '.json' data and files.
+
+    REFERENCES:
+
+        1) https://www.w3schools.com/python/python_json.asp
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To work with the JSON standard:
+import json
+
+# To work with files:
+from utils_v2.system import files
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def from_string(json_data):
+
+    """
+    Decodes a JSON string to a pythonic variable like a dict.
+    :param json_data: The JSON string to decode.
+    :return: The decoded pythonic variable.
+    """
+
+    python_data = json.loads(json_data)
+    return python_data
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def to_string(
+        python_data,
+        indent = 4,
+        default = None,
+        separators = None,
+        no_space = False
+):
+
+    """
+    Converts the given pythonic data to a JSON string.
+    :param python_data: The input data like a dict.
+    :param indent: The tab-width for pretty presentation.
+    :param default: The function to use on something that cannot be directly parsed into a JSON string.
+    :param separators: Custom separators to use.
+    :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not
+        good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT
+        'default'.
+    :return: The JSON string representation of the input pythonic data.
+    """
+
+    if no_space:
+        json_data = json.dumps(
+            python_data,
+            default = default,
+            separators = (',', ':')
+        )
+
+    else:
+        json_data = json.dumps(
+            python_data,
+            indent = indent,
+            default = default,
+            separators = separators
+        )
+
+    return json_data
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def from_file(file):
+
+    """
+    Reads a JSON file and returns it as a pythonic variable like a dict.
+    :param file: The path to the file on the disk or a file held in RAM as a BytesIO object.
+    :return: The decoded pythonic variable.
+    """
+
+    if isinstance(file, io.BytesIO):
+        file.seek(0)
+        json_data = file.getvalue()
+    else: json_data = files.read_file(file)
+    python_data = from_string(json_data)
+    return python_data
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def to_file(
+        file,
+        python_data,
+        indent = 4,
+        default = None,
+        separators = None,
+        no_space = False
+):
+
+    """
+
+    :param file: Either a path to a file on disk, or a buffer in RAM in the form of a BytesIO object.
+    :param python_data: The pythonic data to be converted to the JSON string.
+    :param indent: The tab-width for pretty presentation.
+    :param default: The function to use on something that cannot be directly parsed into a JSON string.
+    :param separators: Custom separators to use.
+    :param no_space: If you want a dense JSON string that saves memory by not using spaces or tabs or line-breaks. Not
+        good for human readability, very good for saving memory. WARNING: THIS OVERRIDES EVERY OTHER PARAMETER EXCEPT
+        'default'.
+    :return: True/False if a path was given, else the same BytesIO object with the written JSON data.
+    """
+
+    json_data = to_string(
+        python_data,
+        indent = indent,
+        default = default,
+        separators = separators,
+        no_space = no_space
+    )
+
+    if isinstance(file, io.BytesIO):
+        file.write(json_data.encode("utf-8"))
+        file.seek(0)
+        return file
+
+    else:
+        try:
+            files.write_file(file, json_data, mode = "w")
+            return True
+        except: return False
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

string/regex.py

@@ -0,0 +1,263 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Sunday, 28th Apr., 2024
+
+    OBJECTIVE:
+
+        To provide a convenient way to perform RegEx operations like finding patterns and substituting them.
+
+    REFERENCES:
+
+        1) https://www.w3schools.com/python/python_regex.asp
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# To work with RegEx:
+import re
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Common RegEx patterns:
+REGEX_EMAIL_ID = r"[\d\w_.+]*@[\d\w_]*.[\d\w]{2,}"
+REGEX_PASSWORD = r"^(?=.*[a-z])(?=.*[A-Z])(?=.*[\d])(?=.*[!@#$%^&*()_+{}\[\]:;<>,.?~\\\/-]).{8,}$"
+REGEX_NAME = r"^[\d\w .\-]{1,30}$"
+REGEX_USERNAME = r"^[\d\w_]{8,25}$"
+REGEX_CONTACT_NUMBER = r"\+?\d{0,3}\s*\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}"
+REGEX_DATE = r"\b(?:\d{4}-\d{2}-\d{2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2},?\s+\d{4}|\d{1,2}\/\d{1,2}\/\d{4}|\d{1,2}-\d{1,2}-\d{2}|\d{1,2}(?:st|nd|rd|th)\s+(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?),?\s+\d{4})\b"
+REGEX_GSTIN = r"[0-9]{2}[A-Z]{5}[0-9]{4}[A-Z]{1}[1-9A-Z]{1}Z[0-9A-Z]{1}"
+REGEX_PAN = r"[A-Z]{5}[0-9]{4}[A-Z]{1}"
+REGEX_IPV4 = (r"[0-9]{1,3}\." * 3) + r"[0-9]{1,3}"
+REGEX_IPV6 = (r"[0-9a-fA-F]{1,4}:" * 7) + r"[0-9a-fA-F]{1,4}"
+REGEX_IFSC = r"[A-Z]{4}0[A-Z0-9]{6}"
+REGEX_UPI = r"[a-zA-Z0-9\.\-]{2,256}@[a-zA-Z][a-zA-Z]{2,64}"
+REGEX_MAC_ADDRESS = r"([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})|([0-9a-fA-F]{4}\\.[0-9a-fA-F]{4}\\.[0-9a-fA-F]{4})"
+REGEX_METRIC_WEIGHT = r"[\d\.]+[ ]?[k]?g"
+
+
+# RegEx chars (append them to the patterns if needed):
+REGEX_START = "^"
+REGEX_END = "$"
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def find(text, pattern, case_sensitive = True, dot_all = False):
+
+    """
+    Returns a list of substrings that match the given RegEx pattern in the input text.
+    :param text: The text in which the pattern needs to be found.
+    :param pattern: The RegEx pattern to look for.
+    :param case_sensitive: Whether, or not, you want the operation to be case-sensitive.
+    :param dot_all: Allow all characters to be matched in ".".
+    :return: An array (list) of substring that match the pattern. Can be an empty list as well.
+    """
+
+    # Prepare the flags:
+    flags = 0
+    if not case_sensitive: flags |= re.IGNORECASE
+    if dot_all: flags |= re.DOTALL
+
+    # Perform the RegEx operation, and clean the results:
+    matches = [match if type(match) is str else match[1] for match in re.findall(pattern, text, flags = flags)]
+    matches = [match for match in matches if len(match) > 0]
+
+    # Return the results:
+    return matches
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def replace(text, pattern, substitute_text, case_sensitive = True, dot_all = False):
+
+    """
+    Replaces any substring in the text that matches the RegEx pattern.
+    :param text: The text in which the substitutions need to be made.
+    :param pattern: The RegEx pattern that needs to be substituted.
+    :param substitute_text: The text that will replace the matches that were found.
+    :param case_sensitive: Whether, or not, you want the operation to be case-sensitive.
+    :param dot_all: Allow all characters to be matched in ".".
+    :return: The text with the substitutions. If no matches are found, the original string is returned.
+    """
+
+    # Prepare the flags:
+    flags = 0
+    if not case_sensitive: flags |= re.IGNORECASE
+    if dot_all: flags |= re.DOTALL
+
+    # Perform the RegEx operation, and return the results:
+    return re.sub(pattern, substitute_text, text, flags = flags)
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def search(text, pattern, case_sensitive = True, dot_all = False):
+
+    """
+    Checks if the given RegEx pattern occurs ANYWHERE in the text that was provided.
+    :param text: The text that needs to be matched against the pattern.
+    :param pattern: The RegEx pattern to look for.
+    :param case_sensitive: Whether, or not, you want the operation to be case-sensitive.
+    :param dot_all: Allow all characters to be matched in ".".
+    :return: True if the pattern matches, else False.
+    """
+
+    # Prepare the flags:
+    flags = 0
+    if not case_sensitive: flags |= re.IGNORECASE
+    if dot_all: flags |= re.DOTALL
+
+    # Perform the RegEx operation, and return the results:
+    if re.search(pattern, text, flags = flags): return True
+    else: return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def match(text, pattern, case_sensitive = True, dot_all = False):
+
+    """
+    Checks if the given text matches the RegEx pattern that was provided. The check is made only at the start of the
+    input string.
+    :param text: The text that needs to be matched against the pattern.
+    :param pattern: The RegEx pattern to look for.
+    :param case_sensitive: Whether, or not, you want the operation to be case-sensitive.
+    :param dot_all: Allow all characters to be matched in ".".
+    :return: True if the pattern matches, else False.
+    """
+
+    # Prepare the flags:
+    flags = 0
+    if not case_sensitive: flags |= re.IGNORECASE
+    if dot_all: flags |= re.DOTALL
+
+    # Perform the RegEx operation, and return the results:
+    if re.match(pattern, text, flags = flags): return True
+    else: return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def split(text, pattern, case_sensitive = True, dot_all = False):
+
+    """
+    Splits an input string based on the pattern that is being matched.
+    :param text: The text that needs to be matched against the pattern.
+    :param pattern: The RegEx pattern to look for.
+    :param case_sensitive: Whether, or not, you want the operation to be case-sensitive.
+    :param dot_all: Allow all characters to be matched in ".".
+    :return: True if the pattern matches, else False.
+    """
+
+    # Prepare the flags:
+    flags = 0
+    if not case_sensitive: flags |= re.IGNORECASE
+    if dot_all: flags |= re.DOTALL
+
+    # Perform the RegEx operation, and return the results:
+    substrings = re.split(pattern, text, flags = flags)
+    if len(substrings) > 0 and substrings[0] == "": substrings.pop(0)
+    return substrings
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def to_json(text, pattern, case_sensitive = True, dot_all = False):
+
+    """
+    Gives out a dict from the extracted features in a string. It is based on the concept of Named Groups.
+    Consider the following example (assuming the search is case-insensitive):
+    TEXT: "UPI/309258561479/14:17:35/UPI/omsainurses@okhdfc"
+    PATTERN: "upi/.*/(?P<time>.*)/.*/(?P<ref>.*)"
+    RESULT: {'time': '14:17:35', 'ref': 'omsainurses@okhdfc'}
+    :param text: The text that needs to be matched against the pattern.
+    :param pattern: The RegEx pattern to look for.
+    :param case_sensitive: Whether, or not, you want the operation to be case-sensitive.
+    :param dot_all: Allow all characters to be matched in ".".
+    :return: A dict with all the extracted features.
+    """
+
+    # Prepare the flags:
+    flags = 0
+    if not case_sensitive: flags |= re.IGNORECASE
+    if dot_all: flags |= re.DOTALL
+
+    # Perform the RegEx operation, and return the results:
+    matches = re.search(pattern, text, flags = flags)
+    regex_json = matches.groupdict() if matches else {}
+    return regex_json
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import json_utils
+
+    for narr in [
+        # r"UPI/309258561479/14:17:35/UPI/omsainurses@okhdf cb",
+        # r"UPI/309258561479/14:17:35/omsainurses@okhdf cb",
+        # r"NEFT-N095232403538009-RELIGARE BROKING LIMITED MAI",
+        r"Product listing - My product - 75g - Super combo pack",
+        r"Product listing - My product - 750 g",
+        r"0.5 kgs mini pack"
+    ]:
+
+        result = to_json(
+            narr,
+            r"upi/.*/(?P<time>.*)/.*/(?P<ref>.*)",
+            case_sensitive = False
+        )
+        print(result)

system/files.py

@@ -0,0 +1,308 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 26th April, 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to work with files and directories in an synchronous way.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# Other system-level dependencies:
+import os
+import sys
+import pathlib
+import platform
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def get_cwd():
+
+    if platform.system().lower().find("windows") > -1: return os.getcwd()
+    else: return os.path.split(os.path.realpath(__file__))[0]
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_parent_directory(path):
+
+    return pathlib.Path(path).parent.absolute()
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def make_directory(path):
+
+    if not os.path.exists(path):
+        try:
+            os.makedirs(path)
+        except Exception as excp:
+            print("FILE UTILS EXCEPTION (make dir):", excp)
+            return False
+
+    if os.path.exists(path):
+        return True
+
+    else:
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_directory_for_file_path(file_path):
+
+    if os.path.exists(file_path):
+        path_components = os.path.split(file_path)
+        file_name = path_components[-1]
+        if file_name.find(".") > -1: file_path = os.path.join("", *path_components[:-1])
+        return file_path
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def write_file(file_path, file_data, mode = "w"):
+
+    try:
+
+        file = open(file_path, mode)
+        file.write(file_data)
+        file.close()
+        return True
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (write file):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def append_file(file_path, file_data):
+
+    return write_file(file_path, file_data, mode = "a")
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def read_file(file_path, mode = "r", encoding = "utf8"):
+
+    try:
+        file = open(file_path, mode, encoding = encoding)
+        file_contents = file.read()
+        file.close()
+        return file_contents
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (read file):", excp)
+        return None
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def rename_file(current, new):
+
+    try:
+        if os.path.isfile(current):
+            os.rename(current, new)
+            return True
+        return False
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (rename file):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def rename_directory(current, new):
+
+    try:
+        if os.path.isdir(current):
+            os.rename(current, new)
+            return True
+        return False
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (rename dir):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def delete_file(file_path):
+
+    try:
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+            return True
+        return False
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (delete file):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def delete_directory(directory_path):
+
+    try:
+        if os.path.exists(directory_path):
+            for root, directories, files in os.walk(directory_path, topdown = False):
+                for file in files:
+                    os.remove(os.path.join(root, file))
+                for directory in directories:
+                    os.rmdir(os.path.join(root, directory))
+            os.rmdir(directory_path)
+            return True
+        return False
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (delete dir):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def move_file(file_path, new_dir):
+
+    try:
+        destination = os.path.join(new_dir, os.path.split(file_path)[-1])
+        os.replace(file_path, destination)
+        return True
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (move file):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def copy_file(file_path, new_dir):
+
+    try:
+        destination = os.path.join(new_dir, os.path.split(file_path)[-1])
+        handle_src = open(file_path, mode = "r")
+        handle_dst = open(destination, mode = "w")
+        stat_src = os.stat(file_path)
+        n_bytes = stat_src.st_size
+        fd_src = handle_src.fileno()
+        fd_dst = handle_dst.fileno()
+        os.sendfile(fd_dst, fd_src, 0, n_bytes)
+        return True
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (copy file):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def list_files(directory_path, full_path = False):
+
+    try:
+        path = pathlib.Path(directory_path)
+        files = [entry.name for entry in path.iterdir() if entry.is_file()]
+        if full_path: files = [os.path.join(directory_path, file) for file in files]
+        return files
+
+    except Exception as excp:
+        print("FILE UTILS EXCEPTION (list files):", excp)
+        return False
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def list_dirs(
+        directory_path,
+        full_path = False,
+        raise_exception = False
+):
+
+    try:
+        path = pathlib.Path(directory_path)
+        files = [entry.name for entry in path.iterdir() if entry.is_dir()]
+        if full_path: files = [os.path.join(directory_path, file) for file in files]
+        return files
+
+    except Exception as excp:
+        if raise_exception: raise
+        print("FILE UTILS EXCEPTION (list dirs):", excp)
+        return []
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass