Squashed 'utils_v2/' content from commit e7073350

git-subtree-dir: utils_v2
git-subtree-split: e7073350338ed6f28a4fdfab2e48c0bf458561b5

.gitignore

@@ -0,0 +1,8 @@
+/.venv/
+/.idea/
+**/__pycache__/
+__pycache__/
+
+*.pem
+*.pyc
+*.pyd

ai/image_classification/async_blur.py

@@ -0,0 +1,202 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 2nd Sept., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to assess images for blurriness.
+        Tried and implemented using HuggingFace behaviour_models.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# For using the AI model:
+import torch
+from transformers import pipeline
+
+# To download images from URLs:
+import requests
+
+# To read images:
+from PIL import Image
+
+# Common:
+from shared import variables
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AssessImageBlur:
+
+    def __init__(
+            self,
+            model,
+            blur_label = "Blur",
+            clarity_label = "Normal",
+            blur_threshold = 0.25,
+            clarity_threshold = 0.65
+    ):
+
+        """
+        This class assesses the input image and tells if it is blurry or clear.
+        :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is
+            stored.
+        :param blur_threshold: The max allowed blurriness (0 to 1 range).
+        :param clarity_threshold: The minimum needed clarity (0 to 1 range).
+        """
+
+        # Note down the config:
+        self.__model = model
+        self.__device = "gpu" if torch.cuda.is_available() else "cpu"
+        self.__blur_label = blur_label
+        self.__clarity_label = clarity_label
+        self.__blur_threshold = blur_threshold
+        self.__clarity_threshold = clarity_threshold
+
+        # Initialize the classifier:
+        self.__classifier = pipeline(
+            task = "image-classification",
+            model = self.__model,
+            device = self.__device
+        )
+
+    @staticmethod
+    async def read_image(source):
+
+        """
+        Reads an image in whichever format it is provided and returns it as a PIL object.
+        :param source: The image as either a path or a URL or a io.BytesIO object.
+        :return: The image opened as a PIL object.
+        """
+
+        # If the input image is already a PIL image:
+        if isinstance(source, Image.Image): return source
+
+        # If a buffer is provided:
+        elif isinstance(source, io.BytesIO):
+            source.seek(0)
+            return Image.open(source)
+
+        # If a string is provided (local path or URL):
+        elif isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://"):
+                response = await variables.http_client.get(source)
+                return Image.open(io.BytesIO(response.content))
+            else: return Image.open(source)
+
+    async def classify(self, image):
+
+        """
+        To get the prediction of the model from the given input image.
+        :param image: The image (as a PIL object or file path or a URL).
+        :return: The dictionary of classes with their respective predictions.
+        """
+
+        image = await self.read_image(image)
+        predictions = self.__classifier(image)
+        classes = {p["label"]: p["score"] for p in predictions}
+        return classes
+
+    async def is_ok(
+            self,
+            image,
+            blur_threshold = None,
+            clarity_threshold = None
+    ):
+
+        """
+        Checks if the image given to it can be used, or should be rejected.
+        :param image: The image (as a PIL object or file path or a URL).
+        :param blur_threshold: A custom threshold to test against. If not provided, the default will be taken that was
+            provided when the instance was created.
+        :param clarity_threshold: A custom threshold to test against. If not provided, the default will be taken that
+            was provided when the instance was created.
+        :return: True if the image is okay, else False.
+        """
+
+        image = await self.read_image(image)
+        blur_threshold = blur_threshold or self.__blur_threshold
+        clarity_threshold = clarity_threshold or self.__clarity_threshold
+        classes = await self.classify(image)
+        if (
+                classes[self.__blur_label] <= blur_threshold and
+                classes[self.__clarity_label] >= clarity_threshold
+        ): return True
+        return False
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+
+    async def main():
+        image = r"/home/developer/Downloads/low-res-check.png"
+        assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh")
+        usable = await assessor.is_ok(image)
+        classes = await assessor.classify(image)
+        print("IS OKAY:", usable)
+        print("CLASSES:", classes)
+
+    asyncio.run(main())

ai/image_classification/async_nsfw.py

@@ -0,0 +1,188 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 2nd Sept., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to assess images for adult content.
+        Tried and implemented using HuggingFace behaviour_models.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# For using the AI model:
+import torch
+from transformers import pipeline
+
+# To download images from URLs:
+import requests
+
+# To read images:
+from PIL import Image
+
+# Common:
+from shared import variables
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AssessImageNSFW:
+
+    def __init__(
+            self,
+            model,
+            nsfw_label = "nsfw",
+            nsfw_threshold = 0.25,
+    ):
+
+        """
+        This class assesses the input image and tells if it is blurry or clear.
+        :param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is
+            stored.
+        :param nsfw_threshold: The max allowed blurriness (0 to 1 range).
+        """
+
+        # Note down the config:
+        self.__model = model
+        self.__device = "gpu" if torch.cuda.is_available() else "cpu"
+        self.__nsfw_label = nsfw_label
+        self.__nsfw_threshold = nsfw_threshold
+
+        # Initialize the classifier:
+        self.__classifier = pipeline(
+            task = "image-classification",
+            model = self.__model,
+            device = self.__device
+        )
+
+    @staticmethod
+    async def read_image(source):
+
+        """
+        Reads an image in whichever format it is provided and returns it as a PIL object.
+        :param source: The image as either a path or a URL or a io.BytesIO object.
+        :return: The image opened as a PIL object.
+        """
+
+        # If the input image is already a PIL image:
+        if isinstance(source, Image.Image): return source
+
+        # If a buffer is provided:
+        elif isinstance(source, io.BytesIO):
+            source.seek(0)
+            return Image.open(source)
+
+        # If a string is provided (local path or URL):
+        elif isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://"):
+                response = await variables.http_client.get(source)
+                return Image.open(io.BytesIO(response.content))
+            else: return Image.open(source)
+
+    async def classify(self, image):
+
+        """
+        To get the prediction of the model from the given input image.
+        :param image: The image (as a PIL object or file path or a URL).
+        :return: The dictionary of classes with their respective predictions.
+        """
+
+        image = await self.read_image(image)
+        predictions = self.__classifier(image)
+        classes = {p["label"]: p["score"] for p in predictions}
+        return classes
+
+    async def is_ok(
+            self,
+            image,
+            nsfw_threshold = None
+    ):
+
+        """
+        Checks if the image given to it can be used, or should be rejected.
+        :param image: The image (as a PIL object or file path or a URL).
+        :param nsfw_threshold: A custom threshold to test against. If not provided, the default will be taken that was
+            provided when the instance was created.
+        :return: True if the image is okay, else False.
+        """
+
+        image = await self.read_image(image)
+        nsfw_threshold = nsfw_threshold or self.__nsfw_threshold
+        classes = await self.classify(image)
+        return True if classes[self.__nsfw_label] <= nsfw_threshold else False
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+
+    async def main():
+
+        assessor = AssessImageNSFW(model = r"/path/to/model")
+        usable = await assessor.is_ok(r"/path/to/image/img.jpg")
+        classes = await assessor.classify(r"https://...")
+
+
+    asyncio.run(main())

ai/image_segmentation/async_dichotomous_segmentation.py

@@ -0,0 +1,223 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 14th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to get masks from dichotomous image segmentation.
+
+        This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait".
+        You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This
+        means that the model should have only two classes like "foreground", and "background". The specified model was
+        trained for implementing portrait mode style blurring of backgrounds.
+
+        The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not
+        support drop-in replacement for other behaviour_models, please be aware about this.
+
+    REFERENCES:
+
+        01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait
+        02. https://github.com/ZhengPeng7/BiRefNet
+
+    DOWNLOADS:
+
+        01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait/tree/main
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For system-level activities:
+import io
+
+# To work with PIL images:
+from PIL import Image
+
+# To make asynchronous HTTP calls:
+from shared.variables import http_client
+
+# For using the AI model:
+import torch
+from torchvision import transforms
+from transformers import AutoModelForImageSegmentation
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DichotomousSegmenter:
+
+    def __init__(
+            self,
+            model = r"zhengpeng7/BiRefNet-portrait"
+    ):
+
+        """
+        This class uses Dichotomous Image Segmentation to produce a mask of what is in the foreground (or what is of
+        interest in a given image).
+        :param model: The path of the model.
+        """
+
+        # Check if GPU is available for faster predictions:
+        self.__cuda_is_available = True if torch.cuda.is_available() else False
+
+        # Initialize the model:
+        self.__model = AutoModelForImageSegmentation.from_pretrained(
+            pretrained_model_name_or_path = model,
+            trust_remote_code = True
+        )
+        torch.set_float32_matmul_precision(["high", "highest"][0])
+        self.__model.to("cuda" if self.__cuda_is_available else "cpu")
+
+    @staticmethod
+    async def read_image(source):
+
+        """
+        Reads an image in whichever format it is provided and returns it as a PIL object.
+        :param source: The image as either a path or a URL or a io.BytesIO object.
+        :return: The image opened as a PIL object.
+        """
+
+        # If the input image is already a PIL image:
+        if isinstance(source, Image.Image): return source
+
+        # If a buffer is provided:
+        elif isinstance(source, io.BytesIO):
+            source.seek(0)
+            return Image.open(source)
+
+        # If a string is provided (local path or URL):
+        elif isinstance(source, str):
+            if source.startswith("http://") or source.startswith("https://"):
+                response = await http_client.get(source)
+                return Image.open(io.BytesIO(response.content))
+            else: return Image.open(source)
+
+    async def get_mask(self, image):
+
+        """
+        Process the image to figure out the mask. In some cases (like the image of a sunset) you will have no white
+        pixels in the mask. White pixels represent the areas that the AI considered to be the foreground (area of
+        interest) and the black parts are the background.
+        :param image: The image data as either a PIL object, or a path to a file on the local disk, or a URL.
+        :return: The mask as a PIL object.
+        """
+
+        # Prepare the image transformer:
+        image_size = (1024, 1024)
+        transform_image = transforms.Compose([
+            transforms.Resize(image_size),
+            transforms.ToTensor(),
+            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+        ])
+
+        # Read and transform the image:
+        image = await self.read_image(image)
+        input_image = transform_image(image).unsqueeze(0).to("cuda" if self.__cuda_is_available else "cpu")
+
+        # Make the prediction:
+        with torch.no_grad():
+            if self.__cuda_is_available: mask = self.__model(input_image)[-1].sigmoid().cuda()
+            else: mask = self.__model(input_image)[-1].sigmoid().cpu()
+
+        # Create a PIL object of the mask data:
+        mask = mask[0].squeeze()
+        mask = transforms.ToPILImage()(mask)
+        mask = mask.resize(image.size)
+
+        # Done here:
+        return mask
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    import numpy as np
+
+    async def main():
+
+        model = DichotomousSegmenter()
+
+        images = [
+            # r"/home/developer/Downloads/kate.jpg",
+            # r"/home/developer/Downloads/IMG-20240730-WA0001.jpg",
+            # r"/home/developer/Downloads/pexels-pixabay-57416.jpg",
+            # r"/home/developer/Downloads/summer_clothes.jpg",
+            # r"/home/developer/Downloads/low_res_cat - upscaled.png",
+            # r"/home/developer/Downloads/blurry_traffic.jpg",
+            r"/home/developer/Downloads/card_0.jpg",
+            r"/home/developer/Downloads/card_1.jpg",
+            r"/home/developer/Downloads/card_2.jpg",
+            r"/home/developer/Downloads/card_3.jpg",
+            r"/home/developer/Downloads/sushmita_card.jpg",
+            r"/home/developer/Downloads/niranjan_card.jpg",
+            r"/home/developer/Downloads/niranjan_card_2.jpg",
+        ]
+
+        for image in images:
+            print(image.split("/")[-1])
+            image = await DichotomousSegmenter.read_image(image)
+            mask = await model.get_mask(image)
+            image.show()
+            mask.show()
+            print("MASK!")
+            array = np.array(mask)
+            print("Dimensions of the array:", array.shape)
+            masked_count = np.sum(array > 125)
+            print(masked_count, type(masked_count), np.size, type(np.size))
+            percentage = masked_count / np.prod(array.shape)
+            print("Area:", round(percentage * 100, 4))
+            # image.putalpha(mask)
+            # image.show()
+            # mask.show()
+
+
+    asyncio.run(main())

ai/object_detection/yolo.py

@@ -0,0 +1,176 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 4th Jul, 2024
+
+    OBJECTIVE:
+
+        To provide a class to detect objects in images using YOLO behaviour_models.
+
+    REFERENCES:
+
+        1) Code Examples: https://docs.ultralytics.com/usage/python/
+        2) Models: https://docs.ultralytics.com/models/yolov8/#supported-tasks-and-modes
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import os
+
+# To use YOLO architecture:
+from ultralytics import YOLO
+
+# To work with images:
+from PIL import Image
+
+# utils:
+from utils_v2.string import json
+
+# Common:
+from shared import constants
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class YoloDetect:
+
+    def __init__(
+            self,
+            model_file,
+            debug = True,
+            debug_prefix = "YOLO | "
+    ):
+
+        # Load the model:
+        self._model = YOLO(model = model_file)
+
+        # Initialize the dbugging tool:
+        self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self._printer.disable()
+
+    def disable_debug(self):
+        self._printer.disable()
+
+    def enable_debug(self):
+        self._printer.enable()
+
+    @staticmethod
+    def _process_result(raw_result):
+
+        # Extract the various types of results available from the inference:
+        speed = {k: (v / 100.0) for k, v in raw_result.speed.items()}
+        class_mapping = raw_result.names
+        probabilities = raw_result.probs
+        boxes = raw_result.boxes
+        masks = raw_result.masks
+
+        # Construct a default response:
+        response_json = {
+            "speed": speed,
+            "classes": class_mapping,
+            "boxes": None
+        }
+
+        # Object-Detection results:
+        formatted_boxes = []
+        for box in boxes:
+            detected_class = int(box.cls.numpy()[0])
+            x1, y1, x2, y2 = box.xyxy[0]
+            formatted_boxes.append(
+                {
+                    "class": detected_class,
+                    "className": class_mapping[detected_class],
+                    "confidence": float(box.conf.numpy()[0]),
+                    "x1": int(x1.numpy()),
+                    "y1": int(y1.numpy()),
+                    "x2": int(x2.numpy()),
+                    "y2": int(y2.numpy())
+                }
+            )
+        response_json["boxes"] = formatted_boxes
+
+        # Done here:
+        return response_json
+
+    def predict(self, image, show = False, verbose = False):
+
+        # Process the input image with the given task:
+        results = self._model(
+            source = image,
+            show = show,
+            verbose = verbose
+        )
+
+        # Based on the task, interpret the results:
+        results_json = self._process_result(results[0])
+
+        # Done here:
+        return results_json
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt")
+    model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt"
+    # sample_image_path = r"/home/developer/Downloads/2_cats.jpg"
+    sample_image_path = r"/home/developer/Downloads/flattened_image.jpg"
+    my_yolo = YoloDetect(model_file = model_file_path)
+    results = my_yolo.predict(image = Image.open(sample_image_path))
+    print("FINAL RESULTS:", json.to_string(results))
+

ai/tts/easy_tts.py

@@ -0,0 +1,175 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 26th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a fast way to make audio from TTS engines.
+
+    REFERENCES:
+
+        01. Usage: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#python-api
+        02. Installation: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#linux-and-macos-install
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To use the AI:
+from melo.api import TTS
+import numpy as np
+import soundfile
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class EasyTTS:
+
+    __sampling_rate = 44_100
+
+    def __init__(
+            self,
+            language = "EN",
+            speed = 1.0
+    ):
+
+        self.__audio = np.zeros(1)
+        self.__language = language
+        self.__speed = speed
+        self.__model = TTS(language = language, device = "auto")
+        self.__speakers = self.__model.hps.data.spk2id
+
+    def list_speakers(self):
+        return list(self.__speakers.keys())
+
+    def speak(
+            self,
+            text,
+            speaker
+    ):
+
+        this_audio = self.__model.tts_to_file(
+            text,
+            self.__speakers[speaker],
+            speed = self.__speed,
+            quiet = True
+        )
+
+        self.__audio = np.concatenate((self.__audio, this_audio))
+
+    def pause(self, seconds):
+
+        self.__audio = np.concatenate((
+            self.__audio,
+            np.zeros(int(self.__sampling_rate * seconds))
+        ))
+
+    def to_wav(self, path = None):
+
+        if path is None:
+            audio_buffer = io.BytesIO()
+            soundfile.write(audio_buffer, self.__audio, self.__sampling_rate)
+            return audio_buffer
+
+        else: soundfile.write(path, self.__audio, self.__sampling_rate)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    speaker = "EN-BR"
+    tts_maker = EasyTTS(language = "EN", speed = 0.9)
+
+    tts_maker.speak(
+        text = """
+            Imagine delighting your doctors with a personalized calendar,  
+            crafted from their own cherished memories.
+            """,
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = """
+            Every day, as they turn the page,  
+            they’ll not only relive those special moments but also remember you, 
+            the one who made it happen.
+            """,
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.75)
+    tts_maker.speak(
+        text = "STEP 1:",
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = "Start by identifying the doctors you’d like to engage with, and add them to our app.",
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = "No rush, you can add their photographs later as well.",
+        speaker = speaker
+    )
+    tts_maker.pause(seconds = 0.3)
+    tts_maker.speak(
+        text = "With this, your engagement funnel is created.",
+        speaker = speaker
+    )
+
+    tts_maker.to_wav(r"/home/developer/Downloads/voiceover.wav")

api/codes.py

@@ -0,0 +1,164 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 12th Sept., 2024
+
+    OBJECTIVE:
+
+        To maintain all status codes in one place.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+from enum import Enum, unique
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+@unique
+class HttpCodes(Enum):
+
+    """
+    Commonly used standard HTTP status codes.
+    Can be sent after an API Call is processed.
+    Refer to: https://http.dev/status
+    NOTE: THIS LIST IS NOT EXHAUSTIVE!
+    """
+
+    # 1XX - Informational:
+    CONTINUE = 100  # .............. https://http.dev/102
+    SWITCHING_PROTOCOLS = 101  # ... https://http.dev/101
+    PROCESSING = 102  # ............ https://http.dev/102
+    EARLY_HINTS = 103  # ........... https://http.dev/103
+
+    # 2XX - Success:
+    SUCCESS = 200  # .................. https://http.dev/200
+    CREATED = 201  # .................. https://http.dev/201
+    ACCEPTED = 202  # ................. https://http.dev/202
+    NON_AUTHORITATIVE_INFO = 203  # ... https://http.dev/203
+    NO_CONTENT = 204  # ............... https://http.dev/204
+    RESET_CONTENT = 205  # ............ https://http.dev/205
+    PARTIAL_CONTENT = 206  # .......... https://http.dev/206
+    MULTI_STATUS = 207  # ............. https://http.dev/207
+    ALREADY_REPORTED = 208  # ......... https://http.dev/208
+    THIS_IS_FINE = 218  # ............. https://http.dev/218
+    IM_USED = 226  # .................. https://http.dev/226
+
+    # 3XX - Redirection:
+    MULTIPLE_CHOICES = 300  # ..... https://http.dev/300
+    MOVED_PERMANENTLY = 301  # .... https://http.dev/301
+    MOVED_TEMPORARILY = 302  # .... https://http.dev/302
+    SEE_OTHER = 303  # ............ https://http.dev/303
+    NOT_MODIFIED = 304  # ......... https://http.dev/304
+    USE_PROXY = 305  # ............ https://http.dev/305
+    SWITCH_PROXY = 306  # ......... https://http.dev/306
+    TEMPORARY_REDIRECT = 307  # ... https://http.dev/307
+    PERMANENT_REDIRECT = 308  # ... https://http.dev/308
+
+    # 4XX - Client Errors:
+    BAD_REQUEST = 400  # ..................... https://http.dev/401
+    UNAUTHORIZED = 401  # .................... https://http.dev/401
+    PAYMENT_REQUIRED = 402  # ................ https://http.dev/402
+    FORBIDDEN = 403  # ....................... https://http.dev/403
+    NOT_FOUND = 404  # ....................... https://http.dev/404
+    METHOD_NOT_ALLOWED = 405  # .............. https://http.dev/405
+    NOT_ACCEPTABLE = 406  # .................. https://http.dev/406
+    PROXY_AUTH_REQUIRED = 407  # ............. https://http.dev/407
+    REQUEST_TIMEOUT = 408  # ................. https://http.dev/408
+    CONFLICT = 409  # ........................ https://http.dev/409
+    GONE = 410  # ............................ https://http.dev/410
+    LENGTH_REQUIRED = 411  # ................. https://http.dev/411
+    PRECONDITION_FAILED = 412  # ............. https://http.dev/412
+    PAYLOAD_TOO_LARGE = 413  # ............... https://http.dev/413
+    URI_TOO_LONG = 414  # .................... https://http.dev/414
+    UNSUPPORTED_MEDIA_TYPE = 415  # .......... https://http.dev/415
+    PAGE_EXPIRED = 419  # .................... https://http.dev/419
+    TOO_MANY_REQUESTS = 429  # ............... https://http.dev/429
+    UNAVAILABLE_FOR_LEGAL_REASONS = 451  # ... https://http.dev/451
+    INVALID_TOKEN = 498  # ................... https://http.dev/498
+    CLIENT_CLOSED_REQUEST = 499  # ........... https://http.dev/499
+
+    # 5XX - Server Errors:
+    INTERNAL_SERVER_ERROR = 500  # ........... https://http.dev/500
+    NOT_IMPLEMENTED = 501  # ................. https://http.dev/501
+    BAD_GATEWAY = 502  # ..................... https://http.dev/502
+    SERVICE_UNAVAILABLE = 503  # ............. https://http.dev/503
+    GATEWAY_TIMEOUT = 504  # ................. https://http.dev/504
+    HTTP_VERSION_NOT_SUPPORTED = 505  # ...... https://http.dev/505
+    VARIANT_ALSO_NEGOTIATES = 506  # ......... https://http.dev/506
+    INSUFFICIENT_STORAGE = 507  # ............ https://http.dev/507
+    LOOP_DETECTED = 508  # ................... https://http.dev/508
+    BANDWIDTH_LIMIT_EXCEEDED = 509  # ........ https://http.dev/509
+    WEB_SERVER_DOWN = 521  # ................. https://http.dev/521
+    ORIGIN_IS_UNREACHABLE = 523  # ........... https://http.dev/523
+    SERVICE_IS_OVERLOADED = 529  # ........... https://http.dev/529
+    NETWORK_READ_TIMEOUT_ERROR = 598  # ...... https://http.dev/598
+    NETWORK_CONNECT_TIMEOUT_ERROR = 599  # ... https://http.dev/599
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+@unique
+class StatusCodes(Enum):
+
+    """
+    To be used internally within the context of your service. Customize these to match your service.
+    The format is: (SUCCESS_INDICATOR, INTERNAL_NUMERIC_CODE, HTTP_CODE)
+    Example: (True, 1, 200)
+    """
+
+    # Legacy Codes:
+    OK = (True, 1, HttpCodes.SUCCESS.value)
+    FAILED = (False, 0, HttpCodes.INTERNAL_SERVER_ERROR.value)
+    PARTIAL_SUCCESS = (True, 2, HttpCodes.PARTIAL_CONTENT.value)
+    PARTIAL_FAILURE = (False, 3, HttpCodes.PARTIAL_CONTENT.value)
+
+    # Authentication Codes:
+    LOGGED_IN_SUCCESSFULLY = (True, 200, HttpCodes.SUCCESS.value)
+    LOGIN_FAILED = (False, 201, HttpCodes.UNAUTHORIZED.value)
+    INVALID_SESSION_TOKEN = (False, 202, HttpCodes.UNAUTHORIZED.value)
+    AUTHENTICATION_DETAILS_INCOMPLETE = (False, 203, HttpCodes.BAD_REQUEST.value)
+
+    # Authorization Codes:
+    AUTHORIZED_SUCCESSFULLY = (True, 300, HttpCodes.SUCCESS.value)
+    NOT_ALLOWED = (False, 301, HttpCodes.FORBIDDEN.value)
+    AUTHORIZATION_DETAILS_INCOMPLETE = (False, 302, HttpCodes.BAD_REQUEST.value)
+    AUTHORIZATION_FAILED = (False, 303, HttpCodes.UNAUTHORIZED.value)
+
+    # General failures:
+    DOWN_FOR_MAINTENANCE = (False, 800, HttpCodes.SERVICE_UNAVAILABLE.value)
+    UNKNOWN_ERROR = (False, 801, HttpCodes.INTERNAL_SERVER_ERROR.value)
+    DATA_INCOMPLETE = (False, 802, HttpCodes.BAD_REQUEST.value)
+    HEADERS_INCOMPLETE = (False, 803, HttpCodes.BAD_REQUEST.value)
+    FILES_MISSING = (False, 804, HttpCodes.BAD_REQUEST.value)
+    CLIENT_CLOSED_REQUEST = (False, 805, HttpCodes.CLIENT_CLOSED_REQUEST.value)
+
+    # Validation failure:
+    DATA_VALIDATION_FAILURE = (False, 900, HttpCodes.BAD_REQUEST.value)
+    RATE_LIMIT_EXCEEDED = (False, 901, HttpCodes.TOO_MANY_REQUESTS.value)

api/log.py

@@ -0,0 +1,139 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 12th Sept., 2024
+
+    OBJECTIVE:
+
+        To have a structure to the response sent from the API calls.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# System-level activities:
+import distro
+import socket
+import platform
+
+# For data-modelling:
+from pydantic import BaseModel, Field
+from typing import Any, Optional, List, Literal
+
+# To work with date and time:
+import datetime
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Info for logging that will stay constant during runtime:
+SERVER_HOSTNAME = str(socket.gethostname())
+PLATFORM_INFO = platform.uname()
+HOST_OS = str(distro.name(True))
+HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})"
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class APILogModel(BaseModel):
+
+    # To identify the machine the code is running on.
+    # DO NOT MODIFY THESE:
+    hostname: str = SERVER_HOSTNAME
+    os: str = HOST_OS
+    cpu: str = HOST_CPU
+    # Can modify these:
+    pid: Optional[Any] = None
+    ppid: Optional[Any] = None
+
+    # To identify the project and actions:
+    project: Optional[str] = None
+    log: str
+    operation: str
+    apiVer: Optional[str] = None
+    logId: Optional[str] = None
+    logChain: Optional[str] = None
+
+    # Timing metrics:
+    ts: datetime.datetime
+    tat: float
+    cpuTime: float
+
+    # To understand the request that came in:
+    sessionInfo: Optional[Any] = None
+    method: Optional[str] = None
+    url: Optional[str] = None
+    route: Optional[str] = None
+    headers: Optional[Any] = None
+    data: Optional[Any] = None
+    files: Optional[Any] = None
+
+    # To understand the output that went out:
+    exception: Optional[Any] = None
+    response: Optional[Any] = None
+    httpCode: Optional[int] = None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    my_log = APILogModel(
+        log = "internal"
+    )
+
+    print(my_log)

api/response.py

@@ -0,0 +1,133 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 12th Sept., 2024
+
+    OBJECTIVE:
+
+        To have a structure to the response sent from the API calls.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# For data-modelling:
+from pydantic import BaseModel
+from typing import Any, Optional, List
+
+# My utils:
+from utils_v2.api.codes import StatusCodes, HttpCodes
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class ResponseModel(BaseModel):
+
+    """
+    A model for how the response should be when developing API endpoints.
+    """
+
+    # The fields that you want in your response:
+    status_code: StatusCodes
+    message: Optional[str | List] = None
+    data: Optional[Any] = None
+    seconds: Optional[float | int] = None
+    log_id: Optional[str] = None
+    http_code: Optional[HttpCodes] = None
+    api_version: Optional[str] = None
+
+    def for_quart(self):
+
+        """
+        Call this when you are using either Flask or Quart as your framework.
+        :return: The output as expected by Flask and Quart.
+        """
+
+        # Figure out the HTTP code:
+        response_http_code = self.http_code.value if self.http_code is not None else self.status_code.value[2]
+
+        # Construct the basic structure:
+        response_dict = {
+            "status": 1 if self.status_code.value[0] else 0,
+            "code": response_http_code,
+            "message": self.message or self.status_code.name.replace("_", " ").lower(),
+            "data": self.data,
+            "apiVer": self.api_version
+        }
+
+        # Now add the additional fields:
+        if self.seconds is not None: response_dict["seconds"] = self.seconds
+        if self.log_id is not None: response_dict["logId"] = self.log_id
+
+        # Done here:
+        return response_dict, response_http_code
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    my_response = ResponseModel(
+        status_code = StatusCodes.RATE_LIMIT_EXCEEDED
+    )
+    my_response.log_id = "abc123"
+
+    print(my_response.for_quart())

cache/async_redis_cache.py

@@ -0,0 +1,497 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 11th April, 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to work with '.json' files in an async manner.
+
+    REFERENCES:
+
+        1) https://www.w3schools.com/python/python_json.asp
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To use redis:
+import redis.asyncio as redis
+
+# Other utils:
+from utils_v2.string import json
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+# To make a decorator:
+from functools import wraps
+
+# For hashing and shortening the hash:
+import hashlib
+import base64
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   WRAPPERS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def cache_it(cache = None, expiry = 120):
+
+    """
+    This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
+    If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
+    cache instead of going through the whole function again.
+    :param cache: The instance of 'AsyncRedisCache'.
+    :param expiry: The time in seconds after which the cached data must be cleared.
+    :return: The decorator that automatically caches your data.
+    """
+
+    def decorator(func):
+
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+
+            # We first use the name of the function and the inputs given to it to generate a key for Redis with the
+            # simple hashing and shortening by way of base64 strings:
+            inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
+            sha256_hash = hashlib.sha256()
+            sha256_hash.update(inputs_given.encode("utf-8"))
+            hashed_key = sha256_hash.digest()
+            base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+            # Now we check if we have the value in cache:
+            try: response = await cache.get(base64_key, raise_exception = True)
+
+            # If the key doesn't exist, we pass through the function and store the results.
+            except:
+                response = await func(*args, **kwargs)
+                await cache.set(key = base64_key, value = response, expiry = expiry)
+
+            # Return the response from the wrapped function.
+            return response
+
+        return wrapper
+
+    return decorator
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def cache_class_methods(attr_name, expiry = 120):
+
+    """
+    This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
+    If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
+    cache instead of going through the whole function again.
+    :param attr_name: The name of the variable that has an instance of "AsyncRedisCache".
+    :param expiry: The time in seconds after which the cached data must be cleared.
+    :return: The decorator that automatically caches your data.
+    """
+
+    def decorator(func):
+
+        @wraps(func)
+        async def wrapper(self, *args, **kwargs):
+
+            # Get the cache object first:
+            cache_obj = getattr(self, attr_name)
+
+            # We first use the name of the function and the inputs given to it to generate a key for Redis with the
+            # simple hashing and shortening by way of base64 strings:
+            inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
+            sha256_hash = hashlib.sha256()
+            sha256_hash.update(inputs_given.encode("utf-8"))
+            hashed_key = sha256_hash.digest()
+            base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+            # Now we check if we have the value in cache:
+            try: response = await cache_obj.get(base64_key, raise_exception = True)
+
+            # If the key doesn't exist, we pass through the function and store the results.
+            except:
+                response = await func(self, *args, **kwargs)
+                await cache_obj.set(key = base64_key, value = response, expiry = expiry)
+
+            # Return the response from the wrapped function.
+            return response
+
+        return wrapper
+
+    return decorator
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncRedisCache:
+
+    # Handle datatypes when setting values to cache:
+    __set_converters = {
+        "set": lambda x: list(x),
+        "tuple": lambda x: list(x),
+        "complex": lambda x: {"r": x.real, "i": x.imag}
+    }
+
+    # Handle datatypes when getting from cache:
+    __get_converters = {
+        "set": lambda x: set(x),
+        "tuple": lambda x: tuple(x),
+        "complex": lambda x: complex(x["r"], x["i"])
+    }
+
+    def __init__(
+            self,
+            connection_string,
+            ping_counter = 1_000,
+            debug = False,
+            debug_prefix = "R-Cache | "
+    ):
+
+        """
+        Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools,
+        strings, dicts, lists, sets, and tuples :)
+        :param connection_string: The connection URL for connecting to Redis.
+        :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection.
+        :param debug: Whether, or not, you want to show debugging messages from the start.
+        :param debug_prefix: The prefix text to show with the debugging messages.
+        """
+
+        # Note down the configuration:
+        self.__client = None
+        self.__ping_counter = ping_counter
+        self.__requests_since_last_ping = 0
+        self.__connection_string = connection_string
+
+        # For debugging:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    @staticmethod
+    def make_key(*args, **kwargs):
+
+        """
+        Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the
+        inputs stay the same, the output stays the same.
+        :param args: Any number of args that you would like to use to generate the key.
+        :param kwargs: Any number of kwargs that you would like to use to generate the key.
+        :return: A string that can be used as a key to store values on Redis.
+        """
+
+        # Take everything into one plain text string:
+        plain_text = "".join(str(a) for a in args)
+        plain_text += json.to_string(kwargs, no_space = True)
+
+        # Hash the plain text value, and create a key from it:
+        sha256_hash = hashlib.sha256()
+        sha256_hash.update(plain_text.encode("utf-8"))
+        hashed_key = sha256_hash.digest()
+        base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+        # Done here:
+        return base64_key
+
+    def add_converter(
+            self,
+            type_name,
+            set_converter_func,
+            get_converter_func,
+    ):
+
+        """
+        To add custom data converters to handle new values.
+        RULE: Each of the converter functions must take in exactly on argument.
+        :param type_name: The name of the type of the object to handle. HINT: type(obj).__name__.
+        :param set_converter_func: A converter function that converts from the custom datatype to a datatype that Redis
+            can work with. Try converting to an bool, int, float, str, list or dict.
+        :param get_converter_func: The inverse of the set converter. This function will be used to convert from a
+            datatype that Redis can work with to the custom datatype.
+        :return: None.
+        """
+
+        self.__set_converters[type_name] = lambda x: set_converter_func(x)
+        self.__get_converters[type_name] = lambda x: get_converter_func(x)
+
+    async def connect(self):
+
+        """
+        To make an asynchronous connection request to the Redis server to establish a connection.
+        :return: True if connected, else False.
+        """
+
+        try:
+            self.__client = redis.from_url(
+                self.__connection_string
+            )
+            return True
+
+        except Exception as exception:
+            self.__printer(exception)
+            return False
+
+    async def disconnect(self):
+
+        """
+        Close the connection to the Redis server.
+        :return: True if successful, else False.
+        """
+
+        if self.__client is not None:
+            try: await self.__client.close()
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+            return True
+        return True
+
+    async def ensure_connection(self):
+
+        """
+        To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected.
+        :return: True if connected, else False.
+        """
+
+        # If we are not connected, we try to establish a connection:
+        if self.__client is None: return await self.connect()
+
+        # Else we check if we are connected. If not, we try to connect.
+        # But we check only once in a while. In the meantime, we assume that we are connected.
+        elif self.__requests_since_last_ping > self.__ping_counter:
+            try:
+                await self.__client.ping()
+                self.__requests_since_last_ping = 0
+                return True
+            except Exception as exception:
+                self.__printer(exception)
+                return await self.connect()
+        else:
+            self.__requests_since_last_ping += 1
+            return True
+
+    async def set(self, key, value, expiry: float = 120, raise_exception = False):
+
+        """
+        Saves some value to the cache.
+        :param key: The key with which the data will be stored and retrieved.
+        :param value: The value to store.
+        :param expiry: The time in seconds after which the data will expire. Must be a positive number.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: True if cached, else False.
+        """
+
+        await self.ensure_connection()
+
+        try:
+
+            # We handle conversion for unsupported Pythonic datatypes:
+            original_type = type(value).__name__
+            converter = self.__set_converters.get(original_type)
+            if converter is not None: value = converter(value)
+
+            # Here we actually try to store the data:
+            response = await self.__client.setex(
+                key,
+                int(expiry),
+                json.to_string({
+                    "value": value,
+                    "type": original_type
+                }, indent = 0, no_space = False)
+            )
+            return response
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return False
+
+    async def get(self, key, raise_exception = False, on_fail = None):
+
+        """
+        Retrieve the cached value.
+        :param key: The key with which the data was saved.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :param on_fail: What to return if the process fails due to an exception.
+        :return: The retrieved data or null if not found.
+        """
+
+        await self.ensure_connection()
+
+        try:
+            data = await self.__client.get(key)
+            data = data.decode("utf-8")
+            data = json.from_string(data)
+            converter = self.__get_converters.get(data["type"])
+            if converter is not None: return converter(data["value"])
+            return data["value"]
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return on_fail
+
+    async def delete(self, key, raise_exception = False):
+
+        """
+        Prematurely delete the value from the cache before it expires.
+        :param key: The key with which the data was saved.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: True if deleted, else False.
+        """
+
+        await self.ensure_connection()
+
+        try:
+            response = await self.__client.delete(key)
+            return True if response else False
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return False
+
+    async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False):
+
+        """
+        To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the
+        counter will be created and the value will be incremented from 0.
+        :param key: The name of the counter.
+        :param value: The amount to increment the value by. Send negative values to count backwards.
+        :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created.
+            This value has to be an integer. If a float is passed, the value will be rounded off.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed.
+        """
+
+        await self.ensure_connection()
+
+        try:
+
+            already_existed = await self.__client.exists(key)
+            new_value = await self.__client.incrby(key, value)
+            if expiry and not already_existed: await self.__client.expire(key, int(expiry))
+            return new_value
+
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+
+    def set_complex(x):
+        return {"r": x.real, "i": x.imag}
+
+    def get_complex(x):
+        return complex(x["r"], x["i"])
+
+    async def main():
+
+        my_cache = AsyncRedisCache(
+            connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0",
+            ping_counter = 100,
+            debug = False
+        )
+
+        my_cache.add_converter(
+            type_name = type(2j).__name__,
+            set_converter_func = set_complex,
+            get_converter_func = get_complex
+        )
+
+        value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624")
+        print("GET:", value)
+        print("TYP:", type(value), end = "\n\n")
+
+        # success = await my_cache.set(
+        #     key = "name",
+        #     value = {"first": "John", "last": "Doe"},
+        #     expiry = 10
+        # )
+        # print("SET:", success, end = "\n\n")
+        #
+        # value = await my_cache.get(key = "name")
+        # print("GET:", value)
+        # print("TYP:", type(value), end = "\n\n")
+        #
+        # success = await my_cache.delete(key = "name")
+        # print("DEL:", success, end = "\n\n")
+        #
+        # value = await my_cache.get(key = "cnt")
+        # print("GET:", value)
+        # print("TYP:", type(value), end = "\n\n")
+        #
+        # await my_cache.delete(key = "cnt")
+        # for _ in range(50):
+        #     await asyncio.sleep(1.0)
+        #     counter = await my_cache.count(key = "cnt", value = 1, expiry = 10)
+        #     print("COUNTER:", counter)
+
+    asyncio.run(main())

cache/async_redis_cache_v2.py

@@ -0,0 +1,568 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 26th Oct., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to cache data for fast access. This version has the change that it can handle custom
+        serializers by way of dependency injection.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To use redis:
+import redis.asyncio as redis
+
+# Other utils:
+from utils_v2.string import json
+from utils_v2.serialization.pickle_serializer import PickleSerializer
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+# To make a decorator:
+from functools import wraps
+
+# For hashing and shortening the hash:
+import hashlib
+import base64
+
+# To work with datatypes:
+from typing import List, Any
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   WRAPPERS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def cache_it(cache = None, expiry = 120):
+
+    """
+    This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
+    If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
+    cache instead of going through the whole function again.
+    :param cache: The instance of 'AsyncRedisCache'.
+    :param expiry: The time in seconds after which the cached data must be cleared.
+    :return: The decorator that automatically caches your data.
+    """
+
+    def decorator(func):
+
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+
+            # We first use the name of the function and the inputs given to it to generate a key for Redis with the
+            # simple hashing and shortening by way of base64 strings:
+            inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
+            sha256_hash = hashlib.sha256()
+            sha256_hash.update(inputs_given.encode("utf-8"))
+            hashed_key = sha256_hash.digest()
+            base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+            # Now we check if we have the value in cache:
+            try: response = await cache.get(base64_key, raise_exception = True)
+
+            # If the key doesn't exist, we pass through the function and store the results.
+            except:
+                response = await func(*args, **kwargs)
+                await cache.set(key = base64_key, value = response, expiry = expiry)
+
+            # Return the response from the wrapped function.
+            return response
+
+        return wrapper
+
+    return decorator
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def cache_class_methods(attr_name, expiry = 120):
+
+    """
+    This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
+    If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
+    cache instead of going through the whole function again.
+    :param attr_name: The name of the variable that has an instance of "AsyncRedisCache".
+    :param expiry: The time in seconds after which the cached data must be cleared.
+    :return: The decorator that automatically caches your data.
+    """
+
+    def decorator(func):
+
+        @wraps(func)
+        async def wrapper(self, *args, **kwargs):
+
+            # Get the cache object first:
+            cache_obj = getattr(self, attr_name)
+
+            # We first use the name of the function and the inputs given to it to generate a key for Redis with the
+            # simple hashing and shortening by way of base64 strings:
+            inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
+            sha256_hash = hashlib.sha256()
+            sha256_hash.update(inputs_given.encode("utf-8"))
+            hashed_key = sha256_hash.digest()
+            base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+            # Now we check if we have the value in cache:
+            try: response = await cache_obj.get(base64_key, raise_exception = True)
+
+            # If the key doesn't exist, we pass through the function and store the results.
+            except:
+                response = await func(self, *args, **kwargs)
+                await cache_obj.set(key = base64_key, value = response, expiry = expiry)
+
+            # Return the response from the wrapped function.
+            return response
+
+        return wrapper
+
+    return decorator
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncRedisCache:
+
+    def __init__(
+            self,
+            connection_string,
+            serializer = None,
+            ping_counter = 1_000,
+            debug = False,
+            debug_prefix = "R-Cache | "
+    ):
+
+        """
+        Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools,
+        strings, dicts, lists, sets, and tuples :)
+        :param connection_string: The connection URL for connecting to Redis.
+        :param serializer: The serializer to use.
+        :param ping_counter: The number of requests to Redis after which you want to ping to ensure connection.
+        :param debug: Whether, or not, you want to show debugging messages from the start.
+        :param debug_prefix: The prefix text to show with the debugging messages.
+        """
+
+        # Note down the configuration:
+        self.__client = None
+        self.__serializer = serializer or PickleSerializer()
+        self.__ping_counter = ping_counter
+        self.__requests_since_last_ping = 0
+        self.__connection_string = connection_string
+
+        # For debugging:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    @staticmethod
+    def make_key(*args, **kwargs) -> str:
+
+        """
+        Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the
+        inputs stay the same, the output stays the same.
+        :param args: Any number of args that you would like to use to generate the key.
+        :param kwargs: Any number of kwargs that you would like to use to generate the key.
+        :return: A string that can be used as a key to store values on Redis.
+        """
+
+        # Take everything into one plain text string:
+        plain_text = "".join(str(a) for a in args)
+        plain_text += json.to_string(kwargs, no_space = True)
+
+        # Hash the plain text value, and create a key from it:
+        sha256_hash = hashlib.sha256()
+        sha256_hash.update(plain_text.encode("utf-8"))
+        hashed_key = sha256_hash.digest()
+        base64_key = base64.b64encode(hashed_key).decode("utf-8")
+
+        # Done here:
+        return base64_key
+
+    async def connect(self) -> bool:
+
+        """
+        To make an asynchronous connection request to the Redis server to establish a connection.
+        :return: True if connected, else False.
+        """
+
+        try:
+            self.__client = redis.from_url(
+                self.__connection_string
+            )
+            return True
+
+        except Exception as exception:
+            self.__printer(exception)
+            return False
+
+    async def disconnect(self) -> bool:
+
+        """
+        Close the connection to the Redis server.
+        :return: True if successful, else False.
+        """
+
+        if self.__client is not None:
+            try: await self.__client.close()
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+            return True
+        return True
+
+    async def ensure_connection(self) -> bool:
+
+        """
+        To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected.
+        :return: True if connected, else False.
+        """
+
+        # If we are not connected, we try to establish a connection:
+        if self.__client is None: return await self.connect()
+
+        # Else we check if we are connected. If not, we try to connect.
+        # But we check only once in a while. In the meantime, we assume that we are connected.
+        elif self.__requests_since_last_ping > self.__ping_counter:
+            try:
+                await self.__client.ping()
+                self.__requests_since_last_ping = 0
+                return True
+            except Exception as exception:
+                self.__printer(exception)
+                return await self.connect()
+        else:
+            self.__requests_since_last_ping += 1
+            return True
+
+    async def list_keys(
+            self,
+            match: str = "*",
+            raise_exception: bool = False
+    ) -> List[str] | None:
+
+        """
+        To get a list of all the keys that match the pattern.
+        :param match: To match a glob-style pattern. THIS IS NOT FULL-FLEDGED REGEX.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: The list of keys if successful, else None.
+        """
+
+        # Standard connectivity check:
+        await self.ensure_connection()
+
+        # Start with an empty list:
+        keys_list = []
+
+        try:
+
+            # Start at the beginning,
+            # and break out of the loop if the pointer returns to zero:
+            pointer = 0
+            while True:
+                pointer, keys = await self.__client.scan(pointer, match = match)
+                keys_list = keys_list + [k.decode("utf-8") for k in keys]
+                if pointer == 0: break
+
+            # Done here:
+            return list(set(keys_list)) if keys_list else None
+
+        # If an exception occurs in th eprocess:
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return None
+
+    async def ttl(
+            self,
+            key: str | bytes,
+            raise_exception: bool = False
+    ) -> int | float | None:
+
+        """
+        To get the no. of seconds till the expiry of some key.
+        :param key: The key to check the expiry of.
+        :param raise_exception:
+        :return: -1 if the key is persistent (i.e., no expiry time set), -2 if the key does not exist, or the time left
+            in seconds if the key exists and is not persistent. If something goes wrong, you will get a null value.
+        """
+
+        # Standard connectivity check:
+        await self.ensure_connection()
+
+        # Start by assuming failure:
+        ttl = None
+
+        try:
+
+            # Get the TTL of the key in ms,
+            # convert it to seconds and return the value:
+            ttl_ms = await self.__client.pttl(key)
+            if ttl_ms >= 0: ttl = ttl_ms / 1_000
+            return ttl
+
+        # If an exception occurs in th eprocess:
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return None
+
+    async def set(
+            self,
+            key: str | bytes,
+            value: Any,
+            expiry: float = None,
+            raise_exception: bool = False
+    ) -> bool:
+
+        """
+        Saves some value to the cache. If an expiry is specified, the data will be deleted after that many seconds.
+        :param key: The key with which the data will be stored and retrieved.
+        :param value: The value to store.
+        :param expiry: The time in seconds after which the data will expire. Must be a positive number.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: True if cached, else False.
+        """
+
+        # Standard connectivity check:
+        await self.ensure_connection()
+
+        try:
+
+            # Here we actually try to set the data:
+            data = self.__serializer.serialize(value)
+            if expiry: response = await self.__client.setex(key, int(expiry), data)
+            else: response = await self.__client.set(key, data)
+            return response
+
+        # If an exception occurs in th eprocess:
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return False
+
+    async def get(
+            self,
+            key: str | bytes,
+            raise_exception: bool = False,
+            on_fail: Any = None
+    ) -> Any:
+
+        """
+        Retrieve the cached value.
+        :param key: The key with which the data was saved.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :param on_fail: What to return if the process fails due to an exception.
+        :return: The retrieved data or null if not found.
+        """
+
+        # Standard connectivity check:
+        await self.ensure_connection()
+
+        try:
+
+            # Here we try to fetch the data:
+            data = await self.__client.get(key)
+            data = self.__serializer.deserialize(data)
+            return data
+
+        # If an exception occurs in th eprocess:
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return on_fail
+
+    async def delete(
+            self,
+            key: str | bytes,
+            raise_exception: bool = False
+    ) -> bool:
+
+        """
+        Prematurely delete the value from the cache before it expires.
+        :param key: The key with which the data was saved.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: True if deleted, else False.
+        """
+
+        # Standard connectivity check:
+        await self.ensure_connection()
+
+        try:
+
+            # Try to manually delete the key before expiry:
+            response = await self.__client.delete(key)
+            return True if response else False
+
+        # If an exception occurs in th eprocess:
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return False
+
+    async def count(
+            self,
+            key: str | bytes,
+            value: int = 1,
+            expiry: float = None,
+            raise_exception: bool = False
+    ) -> int | None:
+
+        """
+        To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the
+        counter will be created and the value will be incremented from 0.
+        :param key: The name of the counter.
+        :param value: The amount to increment the value by. Send negative values to count backwards.
+        :param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created.
+            This value has to be an integer. If a float is passed, the value will be rounded off.
+        :param raise_exception: If you want to raise an exception if the process fails.
+        :return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed.
+        """
+
+        # Standard connectivity check:
+        await self.ensure_connection()
+
+        try:
+
+            # check if the key already exists,
+            # regardless of that, increment the counter:
+            already_existed = await self.__client.exists(key)
+            new_value = await self.__client.incrby(key, value)
+
+            # If the key didn't already exist, specify the expiry:
+            if expiry and not already_existed: await self.__client.expire(key, int(expiry))
+
+            # Done here:
+            return new_value
+
+        # If an exception occurs in th eprocess:
+        except Exception as exception:
+            self.__printer(exception)
+            if raise_exception: raise
+            else: return None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    from utils_v2.string import json
+
+    async def main():
+
+        my_cache = AsyncRedisCache(
+            connection_string = str(input("Paste your conn. str.: ")),
+            ping_counter = 100,
+            debug = True
+        )
+        print("Connecting.")
+        connected = await my_cache.connect()
+        print("Success:", connected)
+
+        # Proceed only if we connected to the database successfully:
+        if connected:
+
+            existing_keys = await my_cache.list_keys()
+            print(f"KEYS ({len(existing_keys)}):", json.to_string(existing_keys, default = str))
+
+            key = str(input("Paste a key to check its TTL: "))
+            key_ttl = await my_cache.ttl(key = key)
+            print(f"KEY '{key}' has {key_ttl} seconds of TTL.")
+
+            value = await my_cache.get(key = str(input("Paste a key to get: ")))
+            print("GET:", json.to_string(value, default = str) if isinstance(value, (dict, list)) else value)
+            print("TYP:", type(value), end = "\n\n")
+
+            success = await my_cache.set(
+                key = "name",
+                value = {"first": "John", "last": "Doe"},
+                expiry = 10
+            )
+            print("SET:", success, end = "\n\n")
+
+            value = await my_cache.get(key = "name")
+            print("GET:", value)
+            print("TYP:", type(value), end = "\n\n")
+
+            success = await my_cache.delete(key = "name")
+            print("DEL:", success, end = "\n\n")
+
+            print("Starting count test.")
+            await my_cache.delete(key = "cnt")
+            for _ in range(5):
+                print("Step:", _)
+                await asyncio.sleep(1.0)
+                counter = await my_cache.count(key = "cnt", value = 1, expiry = 10)
+                print("COUNTER:", counter)
+
+    asyncio.run(main())

database/async_firebase.py

@@ -0,0 +1,504 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+        Bhushan Thakkar
+
+    DATE:
+
+        Friday, 19th April, 2024
+
+    OBJECTIVE:
+
+        To provide an easy interface to work with Firebase.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+# ---
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For async operations:
+# ---
+import asyncio
+
+# For system-level activities:
+# ---
+import os
+
+# My async utils:
+# ---
+import async_json_utils
+
+# Firebase:
+# ---
+import firebase_admin
+import firebase_admin.firestore_async
+import firebase_admin.auth
+import firebase_admin.db
+
+# For debugging and logging:
+# ---
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncFirebase:
+
+    def __init__(
+            self,
+            credentials_json_path,
+            app_name,
+            max_connections = 5,
+            debug = True
+    ):
+
+        # Debugging print:
+        # ---
+        self.__printer = IceCreamDebugger(prefix = f"FBase ({app_name[:8]}) | ", includeContext = True)
+        if not debug: self.__printer.disable()
+        alert = f"Firebase session starting with max. {max_connections} connections."
+        self.__printer(alert)
+
+        # Initialize the instance:
+        # ---
+        self.__semaphore = asyncio.Semaphore(max_connections)
+        credentials = firebase_admin.credentials.Certificate(credentials_json_path)
+        self.__firebase_app = firebase_admin.initialize_app(credentials, name = app_name)
+        self.__firestore = firebase_admin.firestore_async.client(self.__firebase_app)
+
+    def __del__(self):
+        firebase_admin.delete_app(self.__firebase_app)
+        self.__firestore.close()
+        alert = "Firebase session ended."
+        self.__printer(alert)
+
+    def __user_to_json(self, firebase_user):
+
+        user_json = {
+            "uid": firebase_user.uid,
+            "email": firebase_user.email,
+            "emailVerified": firebase_user.email_verified,
+            "displayName": firebase_user.display_name,
+            "phoneNo": firebase_user.phone_number,
+            "photoUrl": firebase_user.photo_url,
+            "customClaims": firebase_user.custom_claims,
+            "disabled": firebase_user.disabled,
+            "providerId": firebase_user.provider_id,
+            "providerData": firebase_user.provider_data,
+            "tenantId": firebase_user.tenant_id
+        }
+
+        return user_json
+
+    async def create_custom_token(self, uid):
+
+        async with self.__semaphore:
+
+            try:
+                custom_token = firebase_admin.auth.create_custom_token(
+                    uid = uid,
+                    app = self.__firebase_app
+                ).decode("utf-8")
+                return custom_token
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def authenticate_token(self, token):
+
+        """
+        To authenticate the given session token.
+        :param token: The session token generated by Firebase on a successful sign-in.
+        :return: Either the retrieved user information or a blank dictionary.
+        """
+
+        try:
+            user_info = firebase_admin.auth.verify_id_token(token, app = self.__firebase_app)
+            return user_info
+        except Exception as exception:
+            self.__printer(exception)
+            return {}
+
+    async def create_user(
+            self,
+            uid,
+            display_name = None,
+            password = None,
+            email = None,
+            phone_number = None
+    ):
+
+        """
+        Create a new user.
+        :param uid: The id to identify the user by.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                firebase_admin.auth.create_user(
+                    uid = uid,
+                    app = self.__firebase_app
+                )
+                return True
+
+            except firebase_admin.auth.UidAlreadyExistsError as excp:
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def delete_user(self, uid):
+
+        async with self.__semaphore:
+
+            try:
+                firebase_admin.auth.delete_user(
+                    uid = uid,
+                    app = self.__firebase_app
+                )
+                return True
+
+            except firebase_admin.auth.UserNotFoundError as excp:
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def get_user(self, uid):
+
+        async with self.__semaphore:
+
+            try:
+                user = firebase_admin.auth.get_user(
+                    uid = uid,
+                    app = self.__firebase_app
+                )
+                return self.__user_to_json(user)
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def list_users(self, users_per_page, page_token = None):
+
+        """
+        To get a list of users. Firebase has a limit of 1000 per call of this API. So we use page-tokens to fetch next
+        pages of users.
+        :param users_per_page: How many users you want to list in this API call (Max. 1,000).
+        :param page_token: To be used in case of pagination.
+        :return: The list of users and the page token to be used for the next call. Will be None in case of failure.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                users = firebase_admin.auth.list_users(
+                    page_token = page_token,
+                    max_results = min(users_per_page, 1000),
+                    app = self.__firebase_app
+                )
+                next_page_token = users.next_page_token if users.has_next_page else None
+                users = users.users
+                users_json = [self.__user_to_json(user) for user in users]
+                return users_json, next_page_token
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None, None
+
+    async def create_document(self, collection_path, document_name, document_data = None):
+
+        """
+        To create a new document with the specified data in an existing collection.
+        :param collection_path: The path of the collection (can be a sub-collection).
+        :param document_name: The name of the document you want to create.
+        :param document_data: The data that you want to populate in the document.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.collection(
+                    collection_path
+                ).add(
+                    document_id = document_name,
+                    document_data = document_data or {}
+                )
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def delete_document(self, path):
+
+        """
+        To delete a document.
+        :param path: The path of the document.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).delete()
+                return True
+
+            except Exception as excp:
+                print("FIRESTORE DOCUMENT DELETION EXCEPTION:", excp)
+                return False
+
+    async def set_document(self, path, data):
+
+        """
+        To overwrite the data in a document.
+        :param path: The path of the document.
+        :param data: The data you want to update as a dictionary.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).set(data)
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def update_document(self, path, data):
+
+        """
+        To update the data in a document. Provide only the fields that you want to update.
+        :param path: The path of the document.
+        :param data: The data you want to update as a dictionary.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).update(data)
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def get_document(self, path):
+
+        """
+        To get the data in a document.
+        :param path: The path of the document.
+        :return: The dictionary of data as found in the path specified or None if the operation failed.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(path).get()
+                return snapshot.to_dict()
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def get_document_fields(self, path, fields):
+
+        """
+        To get only specific fields (keys) in a document. Like how projections are used in Mongo.
+        :param path: The path of the document.
+        :param fields: The list of fields (keys) of the document that you want.
+        :return: The dictionary of data as found in the path specified.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                if type(fields) is not list: fields = [fields]
+                snapshot = await self.__firestore.document(path).get(fields)
+                return snapshot.to_dict()
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+    async def create_collection(self, collection_name, document_name, document_data):
+
+        """
+        Creates a new collection in the root of the database. Note that Firebase doesn't allow creating new empty
+        collections, so we must add one first document in it.
+        :param collection_name: The name of the collection you want to create.
+        :param document_name: The name of the first document you want to put in the collection.
+        :param document_data: The data that you want to put in the first document of the new collection.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.collection(
+                    collection_name
+                ).add(
+                    document_id = document_name,
+                    document_data = document_data
+                )
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def create_sub_collection(
+            self,
+            document_path,
+            sub_collection_name,
+            sub_document_name = None,
+            sub_document_data = None
+    ):
+
+        """
+        Firebase allows you to create collections inside documents. This method is built for that. Note that Firebase
+        doesn't allow creating new empty collections, so we must add one first document in it.
+        :param document_path: The path of the document in which you want to create a new collection.
+        :param sub_collection_name: The name of the collection you want to create.
+        :param sub_document_name: The name of the first document you want to put in the collection.
+        :param sub_document_data: The data that you want to put in the first document of the new collection.
+        :return: True or False based on the success of the operation.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshot = await self.__firestore.document(
+                    document_path
+                ).collection(
+                    sub_collection_name
+                ).add(
+                    document_id = sub_document_name,
+                    document_data = sub_document_data or {}
+                )
+                return True
+
+            except Exception as exception:
+                self.__printer(exception)
+                return False
+
+    async def get_collection(self, path):
+
+        """
+        Get a whole collection's data.
+        :param path: The path of the collection.
+        :return: The collection.
+        """
+
+        async with self.__semaphore:
+
+            try:
+                snapshots = await self.__firestore.collection(path).get()
+                return {snapshot.id: snapshot.to_dict() for snapshot in snapshots}
+
+            except Exception as exception:
+                self.__printer(exception)
+                return None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import time
+    from shared import constants
+
+    user_data = {
+        "firebaseUid": None,
+        "roles": "ClientAdmin",
+        "sessionToken": "siddhesh_20240419"
+    }
+
+    async def test():
+        my_firebase = AsyncFirebase(
+            f"{constants.PROJECT_DIRECTORY}/utils/cred/firebase_certs.json",
+            "myFire",
+            max_connections = 5
+        )
+        print("\n\n---\n\n")
+        results = await my_firebase.get_document("testCollection/testDoc")
+        print(async_json_utils.to_json_string(results))
+        print("\n\n---\n\n")
+        results = await my_firebase.get_document_fields("testCollection/testDoc", ["sampleMap"])
+        print(async_json_utils.to_json_string(results))
+        # await my_firebase.create_sub_collection(
+        #     "myDeepCollection/deepDocId",
+        #     "subCollection3",
+        #     "subDoc",
+        #     {"sub_key": "sub_val"}
+        # )
+        # await my_firebase.create_collection(
+        #     "rootCollection",
+        #     "subDoc",
+        #     {"sub_key": "sub_val"}
+        # )
+        # await my_firebase.delete_document("activeSessions/9d402f1502dfd55a4326fa7fc8e6cb7d")
+        # print(async_json_utils.to_json_string(await my_firebase.get_collection("myDeepCollection")))
+
+    start_time = time.time()
+    asyncio.run(test())
+    print(f"FINISHED IN {time.time() - start_time} SECONDS.")

database/async_mongo.py

@@ -0,0 +1,843 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 28th May, 2024
+
+    OBJECTIVE:
+
+        To have one central place from where all async database connectivity happens.
+
+    REFERENCES:
+
+        01. https://motor.readthedocs.io/en/stable/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For async behaviour:
+import asyncio
+
+# For datetime handling:
+import pytz
+import datetime
+
+# MongoDB:
+from motor.motor_asyncio import AsyncIOMotorClient
+from bson.objectid import ObjectId
+from bson.json_util import dumps, loads
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                EXCEPTIONS                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class MongoFindException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo find operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoInsertException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo insert operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoUpdateException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo update operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoReplaceException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo replace operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoDeleteException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo delete operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class MongoException(Exception):
+
+    def __init__(self, hint = None, origin = None):
+        self.__hint = hint
+        self.__origin = origin
+
+    def __str__(self):
+        message = "mongo operation failed"
+        if self.__hint is not None: message = f"{message} ({self.__hint})"
+        if self.__origin is not None: message = f"{self.__origin} --> {message}"
+        return message
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMongo:
+
+    __db = None
+    __db_name = None
+    __client = None
+
+    def __init__(
+            self,
+            database_name = "myDb",
+            max_connections = 5,
+            debug = True,
+            debug_only_errors = True,
+            host = "localhost",
+            port = 27017,
+            connection_string = None
+    ):
+
+        # Basic config:
+        self.__max_connections = max_connections
+        self.__db_name = database_name
+        self.__host = host,
+        self.__port = port
+        self.__connection_string = connection_string
+
+        # For debugging:
+        self.__debug_only_errors = debug_only_errors
+        self.__printer = IceCreamDebugger(prefix = f"Mongo ({self.__db_name}) | ", includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # Rate/access control:
+        self.__exclusive_semaphore = asyncio.Semaphore(1)
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    @staticmethod
+    def generate_id():
+
+        """
+        Just generates an '_id' in MongoDB style.
+        :return: The '_id' in MongoDB style.
+        """
+
+        return str(ObjectId())
+
+    async def connect(self):
+        try:
+            if self.__connection_string is None:
+                self.__client = AsyncIOMotorClient(
+                    self.__host,
+                    self.__port,
+                    maxPoolSize = self.__max_connections,
+                    minPoolSize = self.__max_connections
+                )
+            else:
+                self.__client = AsyncIOMotorClient(
+                    self.__connection_string,
+                    maxPoolSize = self.__max_connections,
+                    minPoolSize = self.__max_connections
+                )
+            self.__db = self.__client.get_database(self.__db_name)
+            if not self.__debug_only_errors:
+                server_info = await self.__client.server_info()
+                self.__printer(server_info)
+        except Exception as exception: self.__printer(exception)
+
+    async def ensure_connection(self):
+        if self.__db is None:
+            async with self.__exclusive_semaphore:
+                await self.connect()
+
+    @property
+    async def client(self):
+        await self.ensure_connection()
+        return self.__client
+
+    @staticmethod
+    def dict_to_dot_notation(input_dict, parent_key = "", separator = "."):
+        items = []
+        for k, v in input_dict.items():
+            new_key = f"{parent_key}{separator}{k}" if parent_key else k
+            if isinstance(v, dict) and v:
+                items.extend(AsyncMongo.dict_to_dot_notation(v, new_key, separator = separator).items())
+            else:
+                items.append((new_key, v))
+        return dict(items)
+
+    @staticmethod
+    def normalize_date_time(document):
+
+        """
+        MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it.
+        This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was
+        timezone naive, UTC timezone will be applied to it without changing the time value.
+        :param document: The document that you want to normalize the date-time in.
+        :return: The document with normalized datetime.
+        """
+
+        if isinstance(document, datetime.datetime):
+            utc_tz = pytz.timezone("UTC")
+            if document.tzinfo is None: document = utc_tz.localize(document)
+            else: document = document.astimezone(utc_tz)
+
+        if type(document) is list:
+            document = [AsyncMongo.normalize_date_time(item) for item in document]
+
+        if type(document) is dict:
+            document = {
+                AsyncMongo.normalize_date_time(k): AsyncMongo.normalize_date_time(v)
+                for k, v in document.items()
+            }
+
+        return document
+
+    @staticmethod
+    def __from_json_string(json_data):
+        return loads(json_data)
+
+    @staticmethod
+    def __to_json_string(python_data, indent = 4, default = None):
+        return dumps(python_data, indent = indent, default = default)
+
+    async def list_indexes(
+            self,
+            collection,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Lists out the indexes of a collection.
+        :param collection: The collection whose indexes you want to list out,
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The list of indexes or None if the action fails.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        indexes = None
+
+        # Try to list the indexes:
+        try:
+            responses = await self.__db[collection].list_indexes(session = session).to_list(None)
+            indexes = [{key: value for key, value in response.items()} for response in responses]
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not indexes: raise MongoException(hint = "list indexes")
+        return indexes
+
+    async def create_index(
+            self,
+            collection,
+            keys,
+            options = None,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Creates an index on a collection.
+        :param collection: The collection to create the index on.
+        :param keys: The keys (and sorting) to implement the index on.
+        :param options: Additional config.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the execution.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        success = False
+
+        # Try to make the insertion:
+        try:
+            options = options or {}
+            keys = [(k, v) for k, v in keys.items()]
+            response = await self.__db[collection].create_index(keys, session = session, **options)
+            if response: success = True
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not success: raise MongoException(hint = "create index")
+        return success
+
+    async def insert_one(
+            self,
+            collection,
+            document,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Insert data into a collection.
+        :param collection: The collection you want to feed the data into.
+        :param document: The data to be stored.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The id of the inserted data, or null if the action fails.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        inserted_id = None
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].insert_one(document.copy(), session = session)
+            inserted_id = response.inserted_id
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not inserted_id: raise MongoInsertException(hint = f"{collection}")
+        return inserted_id
+
+    async def insert_many(
+            self,
+            collection,
+            documents,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Insert a lot of data into a collection.
+        :param collection: The collection you want to feed the data into.
+        :param documents: The data to be stored.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The id of the inserted data, or null if the action fails.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        inserted_ids = []
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].insert_many(documents, session = session)
+            inserted_ids = response.inserted_ids
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not inserted_ids: raise MongoInsertException(hint = f"{collection}")
+        return inserted_ids
+
+    async def update_one(
+            self,
+            collection,
+            filter_json = None,
+            update_json = None,
+            upsert = False,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Update one document.
+        :param collection: The collection you want to update.
+        :param filter_json: The selection criteria to locate the document to update.
+        :param update_json: The values you want to update.
+        :param upsert: If you want to insert if the document doesn't already exist.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        success = False
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].update_one(
+                filter_json,
+                update_json,
+                upsert = upsert,
+                session = session
+            )
+            success = False if response.modified_count == 0 else True
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not success: raise MongoUpdateException(hint = f"{collection}")
+        return success
+
+    async def update_many(
+            self,
+            collection,
+            filter_json = None,
+            update_json = None,
+            upsert = False,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        Update many documents.
+        :param collection: The collection you want to update.
+        :param filter_json: The selection criteria to locate the document to update.
+        :param update_json: The values you want to update.
+        :param upsert: If you want to insert if the document doesn't already exist.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        update_count = 0
+
+        # Try to make the insertion:
+        try:
+            response = await self.__db[collection].update_many(
+                filter_json,
+                update_json,
+                upsert = upsert,
+                session = session
+            )
+            update_count = response.modified_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not update_count: raise MongoUpdateException(hint = f"{collection}")
+        return update_count
+
+    async def count(
+            self,
+            collection,
+            filter_json = None
+    ):
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            if filter_json is None: filter_json = {}
+            count = await self.__db[collection].count_documents(filter_json)
+        except Exception as exception: self.__printer(exception)
+
+        # Done here:
+        return count
+
+    async def bulk_write(
+            self,
+            collection,
+            requests,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To perform various individual operations in one go. You will have to import individual actions like "UpdateOne"
+        and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method.
+        :param collection: The collection you want to run the requests on.
+        :param requests: The array of requests (operations) to be performed.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return:
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            response = await self.__db[collection].bulk_write(requests, session = session)
+            count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not count: raise MongoException(hint = "bulk write")
+        return count
+
+    async def find_many(
+            self,
+            collection,
+            filter_json,
+            projections = None,
+            skip = 0,
+            limit = None,
+            sort = None,
+            session = None,
+            as_json_string = False,
+            indent = 4,
+            raise_exception = False
+    ):
+
+        """
+        Finds one or more records that match the given conditions.
+        :param collection: The name of the collection to perform the search in.
+        :param filter_json: The filter criteria.
+        :param projections: What parts of the matching data you want to fetch.
+        :param skip: The no. of records to skip before picking next ones. Needed for pagination.
+        :param limit: The max. no. of records you want to fetch.
+        :param sort: The sorting rules to apply.
+        :param session: The session if you need to do this in a transaction.
+        :param as_json_string: Whether you want it as a JSON string or a Python dict/list.
+        :param indent: The indentation to use if you want it as a JSON string.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        results = None
+
+        # Assume defaults:
+        if sort is None: sort = {"_id": -1}
+        if limit is None: limit = 10
+
+        # try to query the data:
+        try:
+            results = await self.__db[collection].find(
+                filter_json,
+                projections,
+                session = session
+            ).sort(sort).skip(skip).limit(limit).to_list(None)
+            if as_json_string: results = self.__to_json_string(results, indent = indent, default = str)
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not results: raise MongoFindException(hint = f"{collection}")
+        return results
+
+    async def find_one(
+            self,
+            collection,
+            filter_json,
+            projections = None,
+            session = None,
+            as_json_string = False,
+            indent = 4,
+            raise_exception = False
+    ):
+
+        """
+        Finds one record that matches the given conditions.
+        :param collection: The name of the collection to perform the search in.
+        :param filter_json: The filter criteria.
+        :param projections: What parts of the matching data you want to fetch.
+        :param session: The session if you need to do this in a transaction.
+        :param as_json_string: Whether you want it as a JSON string or a Python dict/list.
+        :param indent: The indentation to use if you want it as a JSON string.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        results = None
+
+        # try to query the data:
+        try:
+            results = await self.__db[collection].find_one(filter_json, projections, session = session)
+            if as_json_string: results = self.__to_json_string(results, indent = indent, default = str)
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not results: raise MongoFindException(hint = f"{collection}")
+        return results
+
+    async def replace_one(
+            self,
+            collection,
+            filter_json,
+            replacement_json,
+            upsert = False,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To delete one document from a collection.
+        :param collection: The collection from which you want to delete many records.
+        :param filter_json: The filter criteria.
+        :param replacement_json: The data to put in place of the existing document.
+        :param upsert: If you want to insert if the document doesn't already exist.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        success = False
+
+        # try to query the data:
+        try:
+            result = await self.__db[collection].replace_one(
+                filter_json,
+                replacement_json,
+                upsert = upsert,
+                session = session
+            )
+            if result.modified_count or result.upserted_id: success = True
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not success: raise MongoReplaceException(hint = f"{collection}")
+        return success
+
+    async def delete_one(
+            self,
+            collection,
+            filter_json,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To delete one document from a collection.
+        :param collection: The collection from which you want to delete many records.
+        :param filter_json: The filter criteria.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            result = await self.__db[collection].delete_one(filter_json, session = session)
+            count = result.deleted_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}")
+        return count
+
+    async def delete_many(
+            self,
+            collection,
+            filter_json,
+            session = None,
+            raise_exception = False
+    ):
+
+        """
+        To delete many documents from a collection.
+        WARNING: sending {} in the filter would mean deleting ALL the documents.
+        :param collection: The collection from which you want to delete many records.
+        :param filter_json: The filter criteria.
+        :param session: The session if you need to do this in a transaction.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: True or False based on the success of the operation.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        count = 0
+
+        # try to query the data:
+        try:
+            result = await self.__db[collection].delete_many(filter_json, session = session)
+            count = result.deleted_count
+        except Exception as exception:
+            self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}")
+        return count
+
+    async def aggregate(
+            self,
+            collection,
+            pipeline,
+            limit = None,
+            raise_exception = False
+    ):
+
+        """
+        Perform an advance query on the data.
+        :param collection: The collection to perform the query on.
+        :param pipeline: The pipeline of actions to take. Must be a list.
+        :param limit: The max. no. of records to retrieve.
+        :param raise_exception: Whether, or not, you want to raise an exception when something fails.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        # Ensure you are connected:
+        await self.ensure_connection()
+
+        # Assume failure:
+        results = None
+
+        # try to perform the aggregation action:
+        try: results = await self.__db[collection].aggregate(pipeline).to_list(limit)
+        except Exception as exception: self.__printer(exception)
+
+        # Check results and return:
+        if raise_exception and not results: raise MongoException(hint = "aggregation")
+        return results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

database/async_mongo_storage.py

@@ -0,0 +1,358 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 28th May, 2024
+
+    OBJECTIVE:
+
+        To have one central place from where all async database connectivity happens.
+
+    REFERENCES:
+
+        01. https://motor.readthedocs.io/en/stable/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+# ---
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For system-level activity:
+import io
+
+# For async behaviour:
+import asyncio
+
+# MongoDB for File Storage:
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
+from bson.objectid import ObjectId
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMongoStorage:
+
+    __db = None
+    __db_name = None
+    __client = None
+    __store = None
+
+    def __init__(
+            self,
+            connection_string = None,
+            max_connections = 5,
+            host_name = "localhost",
+            port = 27017,
+            database_name = "fileStore",
+            debug = True,
+            debug_prefix = "GridFS (M) | ",
+            debug_only_errors = True
+    ):
+
+        # Database Initialization:
+        self.__host_name = host_name
+        self.__port = port
+        self.__db_name = database_name
+        self.__connection_string = connection_string
+        self.__max_connections = max_connections
+
+        # Debugging:
+        self.__debug_only_errors = debug_only_errors
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+    @staticmethod
+    def generate_id():
+
+        """
+        Just generates an '_id' in MongoDB style.
+        :return: The '_id' in MongoDB style.
+        """
+
+        return str(ObjectId())
+
+    async def connect(self):
+
+        """
+        Initialize the database connection.
+        :return: Nothing.
+        """
+
+        if self.__connection_string is None:
+            self.__client = AsyncIOMotorClient(
+                self.__host_name,
+                self.__port,
+                maxPoolSize = self.__max_connections,
+                minPoolSize = self.__max_connections
+            )
+        else:
+            self.__client = AsyncIOMotorClient(
+                self.__connection_string,
+                maxPoolSize = self.__max_connections,
+                minPoolSize = self.__max_connections
+            )
+        self.__db = self.__client.get_database(self.__db_name)
+        self.__store = AsyncIOMotorGridFSBucket(self.__db)
+
+    @property
+    def fs(self):
+
+        """
+        To access the features that have not been wrapped in this reportlab directly.
+        This could include things like streaming files chunk-by-chunk.
+        :return: The file-store instance.
+        """
+
+        return self.__store
+
+    async def write_from_memory(self, file_name, file_data, metadata_json = None):
+
+        """
+        Save a file (from RAM) to Mongo. Suitable for smaller files (a few MBs max.).
+        :param file_name: The name of the file.
+        :param file_data: The data of the file (held in RAM).
+        :param metadata_json: A JSON of metadata information that can later be used to search files (RECOMMENDED).
+        :return: The file's id as a string (if it gets saved) or None.
+        """
+
+        if self.__store is None: await self.connect()
+
+        file_data.seek(0)
+        file_size = file_data.__sizeof__()
+
+        file_id = None
+        try: file_id = await self.__store.upload_from_stream(file_name, file_data, metadata = metadata_json)
+        except Exception as exception: self.__printer(exception, file_name, file_size, file_id)
+        if not self.__debug_only_errors: self.__printer(file_name, file_size, file_id)
+        return str(file_id)
+
+    async def read_to_memory(self, file_identifier, by_id = True):
+
+        """
+        To retrieve a file (in RAM) based on the provided identifier.
+        Suitable for smaller files (a few MBs max.).
+        :param file_identifier: Either the name or the "_id" of the file.
+        :param by_id: Set to True if you are fetching by the "_id" of the file.
+        :return: Either the file (in RAM) or None.
+        """
+
+        if self.__store is None: await self.connect()
+
+        file_data = None
+
+        try:
+            if by_id: grid_out = await self.__store.open_download_stream(ObjectId(file_identifier))
+            else: grid_out = await self.__store.open_download_stream_by_name(file_identifier)
+            file_data = io.BytesIO(await grid_out.read())
+            file_data.seek(0)
+
+        except Exception as exception:
+            file_data = None
+            self.__printer(exception, file_identifier, by_id)
+
+        if not self.__debug_only_errors: self.__printer(file_identifier, by_id)
+        return file_data
+
+    async def delete_file_by_id(self, file_id):
+
+        """
+        Tries to delete one file by the id.
+        :param file_id: The id of the file in the database.
+        :return: True or False based on the success of the operation.
+        """
+
+        if self.__store is None: await self.connect()
+
+        deleted = False
+
+        try:
+            response = await self.__store.delete(file_id = ObjectId(file_id))
+            deleted = True
+        except Exception as exception:
+            self.__printer(exception, file_id, deleted)
+
+        return deleted
+
+    def __format_metadata_json(self, metadata_json):
+
+        """
+        NOTE: ONLY USE WHEN SEARCHING FILES BY METADATA.
+        MongoDB expects dot-notation while searching for files by the metadata. We are making a function to search
+        files assuming that the conditions are to be applied to the metadata itself. So this function add the
+        dot-notation to the right places to conduct a successful search.
+        :param metadata_json: The JSON to format.
+        :return: The formatted JSON that has the right dot-notation.
+        """
+
+        formatted_metadata_json = {}
+
+        for key, value in metadata_json.items():
+            if not key.startswith("$"): key = f"metadata.{key}"
+            else:
+                if type(value) is dict: value = self.__format_metadata_json(value)
+                if type(value) is list: value = [self.__format_metadata_json(item) for item in value]
+            formatted_metadata_json[key] = value
+
+        return formatted_metadata_json
+
+    async def find_file_by_metadata(self, metadata_json, limit = None, skip = None, sort = None):
+
+        """
+        This method only lists the files that match the criteria mentioned in the metadata JSON.
+        :param metadata_json: The JSON that describes what you want to find.
+        :param limit: Max. no. of records to retrieve.
+        :param skip: No. of starting results to skip. Useful for pagination.
+        :param sort: The sorting conditions to follow.
+        :return: A list of (JSONs of) files that match the conditions. The list can be empty.
+        """
+
+        if self.__store is None: await self.connect()
+
+        files_list = []
+
+        try:
+            limit = limit or 10
+            skip = skip or 0
+            sort = {"_id": -1} if not isinstance(sort, dict) else sort
+            formatted_metadata_json = self.__format_metadata_json(metadata_json)
+            return await self.__store.find(
+                formatted_metadata_json
+            ).sort(sort).skip(skip).limit(limit).to_list(None)
+
+        except Exception as exception: self.__printer(exception, metadata_json, len(files_list))
+
+        if not self.__debug_only_errors: self.__printer(metadata_json, len(files_list))
+        return files_list
+
+    async def find_file_by_id(self, file_id):
+
+        """
+        This method allows you to get the file's info from the id of the file.
+        :param file_id: The id that was assigned by Mongo during upload.
+        :return: The file's info or None if the file doesn't exist.
+        """
+
+        if self.__store is None: await self.connect()
+
+        file_info = None
+
+        try:
+            formatted_metadata_json = {"_id": ObjectId(file_id)}
+            file_info = (await self.__store.find(formatted_metadata_json).to_list(1))[0]
+
+        except Exception as exception: self.__printer(exception, file_id, file_info)
+
+        return file_info
+
+    async def get_file_name(self, file_id):
+
+        """
+        Returns the file name if the id of the file is known.
+        :param file_id: The id of the file as assigned by MongoDB when the file was stored.
+        :return: The file's name (if it exists), or None.
+        """
+
+        # Ensure that we are connected:
+        if self.__store is None: await self.connect()
+
+        # Ensure that the input given is of 'ObjectId' type:
+        if type(file_id) is not ObjectId: file_id = ObjectId(str(file_id))
+
+        # Fetch and return the file name:
+        files_list = await self.__store.find(
+            {"_id": file_id},
+            {"filename": True}
+        ).sort({"_id": -1}).limit(1).to_list(None)
+        try: file_name = files_list[0]["filename"]
+        except: file_name = None
+        return file_name
+
+    async def aggregate(
+            self,
+            collection,
+            pipeline,
+            limit = None
+    ):
+
+        """
+        Perform an advance query on the data.
+        :param collection: The collection to perform the query on.
+        :param pipeline: The pipeline of actions to take. Must be a list.
+        :param limit: The max. no. of records to retrieve.
+        :return: The array of matching records or null if there was an exception.
+        """
+
+        if self.__store is None: await self.connect()
+
+        results = None
+        try: results = await self.__db[collection].aggregate(pipeline).to_list(limit)
+        except Exception as exception: self.__printer(exception)
+        return results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

database/async_mysql_v2.py

@@ -0,0 +1,434 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 30th Aug., 2024
+
+    OBJECTIVE:
+
+        To be able to access SQL-based databases from python in a simple way.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# MySQL Database:
+import aiomysql
+import decimal
+
+# For data-crunching:
+import pandas as pd
+
+# For time-keeping:
+import time
+
+# OS-level operations:
+import os
+
+# My utils:
+from utils_v2.string import json
+
+# For async activities:
+import asyncio
+
+# For debugging:
+from icecream import IceCreamDebugger
+import traceback
+
+# To work with datatypes:
+from typing import List
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMySQL:
+
+    def __init__(
+            self,
+            pool_size,
+            *args,
+            **kwargs
+    ):
+
+        """
+        A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as
+        JSON-like structures (list or dict). The format for the results was very specific to our use case for serving
+        Bicree's requirement. This may not serve your requirement at all.
+        :param pool_size: The number of connections to maintain n a pool.
+        :param args: Any arguments to pass. Not used.
+        :param kwargs: Pass the connection configuration from here.
+        """
+
+        # Set up the variables:
+        self.__args = args
+        self.__kwargs = kwargs
+        self.__min_pool_size = 10
+        self.__max_pool_size = max(pool_size, self.__min_pool_size)
+        self.__pool = None
+
+        # Set up the debugging tools:
+        self.__printer = IceCreamDebugger(prefix = "MySQL | ", includeContext = True)
+
+    def __del__(self):
+        pass
+
+    async def connect(self):
+
+        """
+        Establish a connection and create a pool of connections to call from.
+        :return: None.
+        """
+
+        try:
+
+            self.__kwargs["db"] = self.__kwargs.pop("database")
+            self.__pool = await aiomysql.create_pool(
+                minsize = self.__min_pool_size,
+                maxsize = self.__max_pool_size,
+                loop = asyncio.get_event_loop(),
+                **self.__kwargs
+            )
+
+        except Exception as exception:
+            self.__printer(exception)
+            self.__pool = None
+
+    async def ensure_connection(self):
+
+        """
+        Tries to ensure that a connection is present.
+        Can be called before every function to make sure that our pool is established.
+        :return: None.
+        """
+
+        if self.__pool is None: await self.connect()
+
+    @staticmethod
+    def __parse_row(row):
+
+        """
+        Converts from the custom objects of 'aiomysql' to types that are supported by Python.
+        :param row: The row from the result.
+        :return: The parsed row which will have types that are closer to being native to Python..
+        """
+
+        parsed_row = []
+        for item in row:
+            if isinstance(item, decimal.Decimal): parsed_row.append(float(item))
+            else: parsed_row.append(item)
+        return parsed_row
+
+    async def fetch_all(self, cursor):
+
+        # Make a variable to hold all the result sets.
+        # Needed for when the procedure responds with many "tables":
+        all_result_sets = []
+
+        # Iterate over all result sets,
+        # and process them one-by-one:
+        while True:
+
+            # Process the current result set:
+            this_result_set = []
+            result = await cursor.fetchall()
+            if not cursor.description: break
+            columns = [desc[0] for desc in cursor.description]
+            for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row))))
+            all_result_sets.append(this_result_set)
+
+            # Move to the next set,
+            # or break out of the loop if all done:
+            if not await cursor.nextset(): break
+
+        # Done here:
+        return all_result_sets
+
+    async def call_procedure(
+            self,
+            procedure_name: str,
+            procedure_args: tuple,
+            commit: bool = True
+    ):
+
+        """
+        To call stored procedures and retrieve all the responses.
+        :param procedure_name: The name of the stored procedure that must be called.
+        :param procedure_args: The args to be sent to the stored procedure.
+        :param commit: Whether, or not, you would like to commit the execution.
+        :return: The raw result set as received from the database.
+        """
+
+        # Make sure we have a connection:
+        await self.ensure_connection()
+
+        # Make a variable to hold all the result sets.
+        # Needed for when the procedure responds with many "tables":
+        all_result_sets = []
+
+        # Call the procedure and get the results:
+        async with self.__pool.acquire() as connection:
+            async with connection.cursor() as cursor:
+                await cursor.callproc(procedure_name, procedure_args)
+                all_result_sets = await self.fetch_all(cursor)
+                if commit: await connection.commit()
+
+                # # Iterate over all result sets,
+                # # and process them one-by-one:
+                # while True:
+                #     this_result_set = []
+                #     result = await cursor.fetchall()
+                #     if not cursor.description: break
+                #     columns = [desc[0] for desc in cursor.description]
+                #     for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row))))
+                #     all_result_sets.append(this_result_set)
+                #     await cursor.nextset()
+
+        # Done here:
+        return all_result_sets
+
+    async def call_procedure_and_get_json(
+            self,
+            procedure_name,
+            procedure_args,
+            commit: bool = True,
+            retry_count = 1,
+            backoff_seconds = 0.5,
+            backoff_multiplier = 1.1,
+            return_exception = False
+    ):
+
+        """
+        The method to call when you need to call a stored procedure and retrieve the response as a JSON-like object.
+        This is custom formatting based on the structure created by Mr. bhushan Thakkar in late April (2024).
+        :param procedure_name: The name of the stored procedure that must be called.
+        :param procedure_args: The args to be sent to the stored procedure.
+        :param commit: Whether, or not, you would like to commit the execution.
+        :param retry_count: The max. number of times to try in case one or more attempts fail.
+        :param backoff_seconds: The time to wait before making the next attempt if the retry count is more than 1.
+        :param backoff_multiplier: The factor that dictates how much to modify the time delay by when waiting to retry.
+        :param return_exception: Whether, or not, you would like to return the exception object if something goes wrong.
+        :return: The formatted response and the exception (if asked for).
+        """
+
+        # Note down the start time:
+        start_ts = time.time()
+
+        # Try to get the data from the database:
+        results = []
+        exception = None
+        for _ in range(retry_count):
+            try: results = await self.call_procedure(
+                procedure_name = procedure_name,
+                procedure_args = procedure_args,
+                commit = commit
+            )
+            except Exception as exc: exception = exc
+            if exception is None: break
+            await asyncio.sleep(backoff_seconds)
+            backoff_seconds = backoff_seconds * backoff_multiplier
+
+        # If the results are blank:
+        if len(results) == 0:
+            formatted_results = {
+                "status": 0,
+                "message": "Please contact admin (NE)" if exception is None else "Please contact admin (E)",
+                "seconds": time.time() - start_ts,
+                "data": {}
+            }
+            if return_exception: return formatted_results, exception
+            else: return formatted_results
+
+        # Extract the very basic success or failure indicators:
+        formatted_results = {
+            "status": results[0][0]["status"],
+            "message": results[0][0].get("message", "ok"),
+            "seconds": 0.0,
+            "data": {}
+        }
+
+        # Handle the remaining keys of the zeroth result set:
+        for key, value in results[0][0].items():
+            if key not in formatted_results.keys():
+                formatted_results["data"][key] = value
+
+        # Format
+        for index in range(len(results)):
+            if index > 0: formatted_results["data"][f"rs{index-1}"] = results[index]
+
+        # Note down the time taken:
+        formatted_results["seconds"] = time.time() - start_ts
+
+        # Done here:
+        if return_exception: return formatted_results, exception
+        else: return formatted_results
+
+    async def execute_one(
+            self,
+            query: str,
+            commit: bool = True,
+            return_exception: bool = False
+    ):
+
+        """
+        Runs one command / query in SQL.
+        :param query: The query / command to run.
+        :param commit: Whether, or not, you would like to commit the execution.
+        :param return_exception: Whether, or not, you would like to return the exception from this function.
+        :return: Either just the result or the result and the exception.
+        """
+
+        # Make sure we have a connection:
+        await self.ensure_connection()
+
+        # Start by assuming failure:
+        rows_affected = None
+        results = None
+        excp = None
+
+        try:
+
+            # Get a connection and execute the command:
+            async with self.__pool.acquire() as connection:
+                async with connection.cursor() as cursor:
+                    rows_affected = await cursor.execute(query)
+                    results = await self.fetch_all(cursor)
+                    if commit: await connection.commit()
+
+        # SQL-specific errors:
+        except aiomysql.MySQLError as exception:
+            self.__printer("SQL Exception", exception)
+            excp = exception
+
+        # Other errors:
+        except Exception as exception:
+            self.__printer("Other Exception", exception)
+            excp = exception
+
+        # Done here:
+        if return_exception: return rows_affected, results, excp
+        else: return rows_affected, results
+
+    async def execute_many(
+            self,
+            query: str,
+            data: List[tuple],
+            commit: bool = True,
+            return_exception: bool = False
+    ):
+
+        """
+        Runs many commands / queries in SQL.
+        Consider the following example:
+        QUERY: "INSERT INTO pincodeMaster (pincode, city, state) VALUES (%s, %s, %s);"
+        DATA: [
+            ('110001', 'New Delhi', 'Delhi'),
+            ('500001', 'Hyderabad', 'Telangana'),
+            ('600001', 'Chennai', 'Tamil Nadu')
+        ]
+        :param query: The query / command to run.
+        :param data: The data to substitute into the query string.
+        :param commit: Whether, or not, you would like to commit the execution.
+        :param return_exception: Whether, or not, you would like to return the exception from this function.
+        :return: Either just the result or the result and the exception.
+        """
+
+        # Make sure we have a connection:
+        await self.ensure_connection()
+
+        # Start by assuming failure:
+        rows_affected = None
+        results = None
+        excp = None
+
+        try:
+
+            # Get a connection and execute the command:
+            async with self.__pool.acquire() as connection:
+                async with connection.cursor() as cursor:
+                    rows_affected = await cursor.executemany(query, data)
+                    results = await self.fetch_all(cursor)
+                    if commit: await connection.commit()
+
+        # SQL-specific errors:
+        except aiomysql.MySQLError as exception:
+            self.__printer("SQL Exception", exception)
+            excp = exception
+
+        # Other errors:
+        except Exception as exception:
+            self.__printer("Other Exception", exception)
+            excp = exception
+
+        # Done here:
+        if return_exception: return rows_affected, results, excp
+        else: return rows_affected, results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

dataframe/splitter.py

@@ -0,0 +1,118 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 22nd Oct., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to split dataframes into chunks and divide the workload nto more manageable batches.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+    USAGE EXAMPLE:
+
+        for sub_df in DataFrameSplitter(df, chunk_size = 50):
+
+            # Do your task here:
+            pass
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DataFrameSplitter:
+
+    def __init__(self, df, chunk_size):
+
+        """
+        Use this to process your dataframe in batches. Useful for when you need to send out alerts at intervals or need
+        to maintain checkpoints.
+        :param df: The dataframe to iterate over.
+        :param chunk_size: The max. no. of rows to pick at once.
+        """
+
+        self.df = df
+        self.row_count = df.shape[0]
+        self.chunk_size = chunk_size
+        self.offset = 0
+
+    def __iter__(self):
+        self.offset = 0
+        return self
+
+    def __next__(self):
+        if self.offset >= self.row_count:
+            raise StopIteration
+        end = self.offset + self.chunk_size
+        chunk = self.df.iloc[self.offset:end]
+        self.offset = end
+        return chunk
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

date_time/date_time.py

@@ -0,0 +1,338 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 21st jun, 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to work with time.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For date and time handling:
+import pytz
+from datetime import datetime, time, timedelta, tzinfo
+import dateparser
+
+# To handle date-time objects from a Numpy array and Pandas Dataframe:
+import numpy as np
+import pandas as pd
+
+# To work with datatypes:
+from typing import List
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Date-Time Formats:
+DATE_TIME_FORMATS = (
+    "%d/%m/%y",
+    "%d-%b-%y",
+    "%d-%m-%y",
+    "%d.%m.%y",
+    "%d/%m/%Y",
+    "%d-%b-%Y",
+    "%d-%m-%Y",
+    "%d.%m.%Y",
+    "%d/%b",
+    "%d%m%Y",
+    "%Y%m%d",
+    "%Y/%m/%d",
+    "%Y-%m-%d",
+    "%Y-%m-%d %H:%M:%S"
+)
+
+# Useful Timezones:
+TIMEZONE_UTC = pytz.timezone("UTC")
+TIMEZONE_IST = pytz.timezone("Asia/Kolkata")
+TIMEZONE_ET = pytz.timezone("America/New_York")
+TIMEZONE_CT = pytz.timezone("America/Chicago")
+TIMEZONE_MT = pytz.timezone("America/Denver")
+TIMEZONE_PT = pytz.timezone("America/Los_Angeles")
+TIMEZONE_JST = pytz.timezone("Asia/Tokyo")
+TIMEZONE_CET = pytz.timezone("Europe/Paris")
+TIMEZONE_GMT = pytz.timezone("GMT")
+TIMEZONE_AEST = pytz.timezone("Australia/Sydney")
+TIMEZONE_NZST = pytz.timezone("Pacific/Auckland")
+TIMEZONE_CST = pytz.timezone("Asia/Shanghai")
+TIMEZONE_KST = pytz.timezone("Asia/Seoul")
+TIMEZONE_MSK = pytz.timezone("Europe/Moscow")
+TIMEZONE_BRT = pytz.timezone("America/Sao_Paulo")
+TIMEZONE_GST = pytz.timezone("Asia/Dubai")
+TIMEZONE_SAST = pytz.timezone("Africa/Johannesburg")
+TIMEZONE_AST = pytz.timezone("Asia/Riyadh")
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def translate_date_time_string(
+        datetime_string: str,
+        source_format: str = None,
+        destination_format: str = "%Y-%m-%dT%H:%M:%S"
+) -> str | None:
+
+    """
+    To convert an input datetime string to a different format.
+    :param datetime_string: The datetime string to translate.
+    :param source_format: The current format of the string. If not provided, dateparser will be used.
+    :param destination_format: The format to convert to.
+    :return: The converted datetime string.
+    """
+
+    try:
+
+        if source_format is None: datetime_obj = dateparser.parse(datetime_string)
+        else: datetime_obj = datetime.strptime(datetime_string, source_format)
+        return datetime_obj.strftime(destination_format)
+
+    except Exception as exception:
+        return None
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def parse_date_time(
+        input_value: datetime | str | int | float,
+        timezone: str | tzinfo = None,
+        date_formats: List[str] = None
+) -> datetime:
+
+    """
+    To take any kind of input and interpret the datetime from it.
+    :param input_value: Either a string or an integer or some form of datetime representation.
+    :param timezone: The timezone to apply to the interpreted datetime. IF THE INPUT IS NAIVE, THIS TIMEZONE WILL BE
+        APPLIED AS IS, ELSE THE TIMEZONE WILL BE TRANSLATED.
+    :param date_formats: The string formats to consider when parsing a string input.
+    :return: The parsed datetime or null.
+    """
+
+    datetime_object = None
+    date_formats = date_formats or DATE_TIME_FORMATS
+
+    # It could either be in seconds or milliseconds from epoch time's base date (January 1, 1970),
+    # or it could be days since Microsoft Excel's base date (December 31, 1899).
+    if isinstance(input_value, (int, float, np.number)) and not np.isnan(input_value):
+        if input_value > 9999999999.0: datetime_object = datetime.fromtimestamp(input_value / 1000.0)
+        if input_value > 999999.0: datetime_object = datetime.fromtimestamp(input_value)
+        else: datetime_object = datetime.fromtimestamp(input_value * 24 * 60 * 60.0) - timedelta(days = 25569)
+
+    # The input can even be a pre-formatted date:
+    if isinstance(input_value, str):
+        datetime_object = dateparser.parse(
+            input_value,
+            date_formats = date_formats,
+            settings = {
+                "DATE_ORDER": "DMY",
+                "PREFER_DAY_OF_MONTH": "first",
+            }
+        )
+
+    # If the type is a datetime object, then return it as it is:
+    if isinstance(input_value, datetime):
+        datetime_object = input_value
+
+    # If the type is the native datetime format of pandas:
+    if isinstance(input_value, pd._libs.tslibs.timestamps.Timestamp):
+        datetime_object = input_value.to_pydatetime()
+
+    # Process the timezone:
+    if datetime_object is not None and timezone is not None:
+        datetime_object = to_timezone(datetime_object, timezone)
+
+    # Done here:
+    return datetime_object
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_current_date_time(
+        timezone: str | tzinfo = None,
+        as_string: bool = False
+) -> datetime:
+
+    """
+    Returns the current time as a datetime object.
+    :param timezone: The timezone to apply to the returned datetime.
+    :param as_string: Whether, or not, you want the output as a string.
+    :return: The datetime object/string representing the current time.
+    """
+
+    if timezone is not None and isinstance(timezone, str): timezone = pytz.timezone(timezone)
+    now = datetime.now(timezone)
+    return now.isoformat() if as_string else now
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_current_ist_date_time(
+        as_string = False
+) -> datetime:
+
+    """
+    Gives out the current time in IST timezone.
+    :param as_string: Whether, or not, you want the output as a string.
+    :return: The datetime object or string representing the current time.
+    """
+
+    return get_current_date_time(
+        timezone = TIMEZONE_IST,
+        as_string = as_string
+    )
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_current_utc_date_time(
+        as_string: bool = False
+) -> datetime:
+
+    """
+    Gives out the current time in UTC timezone.
+    :param as_string: Whether, or not, you want the output as a string.
+    :return: The datetime object or string representing the current time.
+    """
+
+    return get_current_date_time(
+        timezone = TIMEZONE_UTC,
+        as_string = as_string
+    )
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def as_if_timezone(
+        datetime_object: datetime,
+        timezone: str | tzinfo
+) -> datetime:
+
+    """
+    Ignores existing timezone info and applies the intended timezone.
+    The time stays the same, only the timezone marker changes.
+    e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-09 00:00:00+00:00
+    HINT: IT PRETENDS "AS IF" THE TIMEZONE WAS THE INPUT TIMEZONE.
+    :param datetime_object: The datetime object on which the timezone needs to be applied.
+    :param timezone: The timezone that needs to be applied.
+    :return: A timezone-aware datetime object.
+    """
+
+    tz_object = pytz.timezone(timezone) if isinstance(timezone, str) else timezone
+    return tz_object.localize(datetime_object.replace(tzinfo = None))
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def to_timezone(
+        datetime_object: datetime,
+        timezone: str | tzinfo
+) -> datetime:
+
+    """
+    Converts from one timezone to another. The time is adjusted by computing the difference between the two timezones.
+    NOTE: THIS FUNCTION APPLIES TH EINPUT TIMEZONE IF THE INPUT DATETIME WAS TIMEZONE-NAIVE.
+    e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-08 18:30:00+00:00
+    :param datetime_object: The datetime object on which the timezone needs to be applied.
+    :param timezone: The timezone that needs to be applied.
+    :return: A timezone-aware datetime object.
+    """
+
+    if isinstance(timezone, str): timezone = pytz.timezone(timezone)
+    if datetime_object.tzinfo is None: dto = datetime_object.astimezone(timezone)
+    else: dto = datetime_object.astimezone(timezone)
+    return dto
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def round_datetime(
+        datetime_object: datetime,
+        seconds: int
+) -> datetime:
+
+    """
+    'Snaps' the time to the closest 'n-second' window. For example, if you want to round the time off to the nearest
+    5-minute period (maybe for use cases like trading), you set the value of seconds to 300. This way any input value
+    of, say, 12:01:50 AM gets converted to 12:00:00 AM; and a value of 12:02:31 AM gets converted to 12:05:00 AM.
+    :param datetime_object: The input datetime that you want to round off.
+    :param seconds: The period of rounding in seconds. 60 for 1 minute, 300 for 5 minutes and so on.
+    :return: The rounded date-time (with the original timezone).
+    """
+
+    # For timezone-naive cases:
+    if datetime_object.tzinfo is None:
+        total_seconds = datetime_object.timestamp()
+        rounded_seconds = round(total_seconds / seconds) * seconds
+        rounded_datetime = datetime.fromtimestamp(rounded_seconds)
+        return rounded_datetime
+
+    # For timezone-aware cases:
+    else:
+        original_tz = datetime_object.tzinfo
+        _dt = to_timezone(datetime_object, timezone = original_tz)
+        total_seconds = _dt.timestamp()
+        rounded_seconds = round(total_seconds / seconds) * seconds
+        rounded_datetime = datetime.fromtimestamp(rounded_seconds, tz = original_tz)
+        return rounded_datetime
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

goog/controllers/base.py

@@ -0,0 +1,475 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 26th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a base class for common behaviour of Google's APIs.
+
+    REFERENCES:
+
+        1. Quickstart: https://developers.google.com/gmail/api/quickstart/python
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.date_time import date_time
+from utils_v2.goog.models.api_call import GoogleApiResponse
+from utils_v2.goog.models.auth_tokens import GoogleAuthTokens
+
+# Related to Google:
+from google_auth_oauthlib.flow import InstalledAppFlow
+
+# To make API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# For working with datatypes:
+from typing import Literal, List
+
+# For debugging:
+from icecream import IceCreamDebugger
+import inspect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncGoogleBase:
+
+    def __init__(
+            self,
+            service_name: str,
+            oauth_json: dict,
+            http_client: httpx.AsyncClient,
+            redirect_url: str = None,
+            debug = True,
+            debug_prefix = "GMail | ",
+            debug_only_errors = True
+    ):
+
+        """
+        To initialize any Google API from one base class. The client's id and secret are available in the file
+        downloaded form https://console.cloud.google.com/apis/credentials (do not forget to select your app).
+        :param service_name: A string to identify this service.
+        :param oauth_json: The OAuth credentials downloaded from https://console.cloud.google.com/apis/credentials
+        :param http_client: An asynchronous HTTP client to make API calls.
+        :param redirect_url: Where you would like to receive the confirmation of the user authorization.
+        :param debug: Whether, or not, you would like to show debugging messages on the terminal.
+        :param debug_prefix: The prefix string to identify the debugging messages.
+        :param debug_only_errors: Whether you would like to show all debugging messages or just error messages.
+        """
+
+        # Prepare the debugging utility:
+        self._debug_prefix = debug_prefix
+        self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self._printer.disable()
+        self._debug_only_errors = debug_only_errors
+
+        # Accept the input configuration:
+        self._service_name = service_name
+        self._http_client = http_client
+        self._oauth_json = oauth_json
+        self._client_id = self._oauth_json["web"]["client_id"]
+        self._client_secret = self._oauth_json["web"]["client_secret"]
+        self._redirect_url = redirect_url
+
+    def enable_debug(self):
+        self._printer.enable()
+
+    def disable_debug(self):
+        self._printer.disable()
+
+    def debug_only_errors(self):
+        self._debug_only_errors = True
+
+    def debug_everything(self):
+        self._debug_only_errors = False
+
+    # ┏┓           •
+    # ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
+    # ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
+    #       ┛
+
+    @property
+    def client_id(self):
+        return self._client_id
+
+    @property
+    def client_secret(self):
+        return self._client_secret
+
+    # ┏┓┏┓   ┓   ┏┓ ┏┓
+    # ┃┃┣┫┓┏╋┣┓  ┏┛ ┃┫
+    # ┗┛┛┗┗┻┗┛┗  ┗━•┗┛
+
+    async def get_authorization_url(
+            self,
+            scopes: List[str],
+            state: str = None,
+            access_type: Literal["online", "offline"] = "offline",
+            approval_prompt: Literal["auto", "force", "consent"] = "auto",
+            include_granted_scopes: Literal["true", "false"] = "true",
+            user_email: str = None
+    ) -> str:
+
+        """
+        TO get the OAuth2.0 authorization URL for one user.
+        DOCUMENTATION:
+            1. https://developers.google.com/identity/protocols/oauth2/web-server
+        :param scopes: The set of permission you want the user to give.
+        :param state: A unique identifier for your user. If not supplied, a random string will be generated.
+        :param access_type: Set the value to offline if your application needs to refresh access tokens when the user is
+            not present at the browser.
+        :param approval_prompt: "force" ensures that the consent screen is always shown to the user, regardless of
+            whether the user has previously granted consent for the requested scopes. It forces the user to re-approve
+            the app's access, which can be useful if the app is requesting new permissions or if the consent needs to be
+            explicitly confirmed. "consent" ensures the user's consent is required if they haven't approved the app's
+            requested permissions yet. "auto" allows Google to automatically determine whether the consent screen should
+            be shown.
+        :param include_granted_scopes: Enables applications to use incremental authorization to request access to
+            additional scopes in context. If you set this parameter's value to true and the authorization request is
+            granted, then the new access token will also cover any scopes to which the user previously granted the
+            application access.
+        :param user_email:
+        :return:
+        """
+
+        # Create a flow:
+        flow = InstalledAppFlow.from_client_config(
+            self._oauth_json,
+            scopes = scopes,
+            redirect_uri = self._redirect_url
+        )
+
+        # Get an authorization URL:
+        auth_url, state = flow.authorization_url(
+            access_type = access_type,
+            approval_prompt = approval_prompt,
+            include_granted_scopes = include_granted_scopes,
+            login_hint = user_email,
+            state = state
+        )
+
+        # Done here:
+        return auth_url
+
+    async def get_authorization_tokens(
+            self,
+            scopes: List[str],
+            redirect_url: str
+    ) -> GoogleAuthTokens:
+
+        """
+        When the user accepts or declines an authorization request, Google sends you an alert on your redirect URL. Pass
+        the URL as it is to this method to generate the authorization tokens that you can store in the database and
+        reuse for this user's activities.
+        :param scopes: The set of permissions the user granted.
+        :param redirect_url: The exact URL that was hit (with the query params) that Google hit when the user did
+            something on your authorization URL. Fortunately, this URL is readily available in Quart and Flask by
+            calling 'request.url'.
+        :return: The authorization tokens.
+        """
+
+        # Create a flow:
+        flow = InstalledAppFlow.from_client_config(
+            self._oauth_json,
+            scopes = scopes,
+            redirect_uri = self._redirect_url
+        )
+
+        # Get the credentials:
+        credentials = flow.fetch_token(authorization_response = redirect_url)
+        ttl = credentials["expires_in"] - 60
+        return GoogleAuthTokens(
+            accessToken = credentials["access_token"],
+            refreshToken = credentials["refresh_token"],
+            expiresAt = date_time.get_current_utc_date_time() + datetime.timedelta(seconds = ttl),
+            scopes = credentials["scope"]
+        )
+
+    # ┏┓          ┳┓      ┓•
+    # ┣ ┏┓┏┓┏┓┏┓  ┃┃┏┓┏┏┓┏┫┓┏┓┏┓
+    # ┗┛┛ ┛ ┗┛┛   ┻┛┗ ┗┗┛┗┻┗┛┗┗┫
+    #                          ┛
+
+    @staticmethod
+    async def __get_error_message(api_response: GoogleApiResponse) -> str:
+
+        """
+        To extract various kinds of error messages from Google's responses.
+        :param api_response: The formatted response from the API call.
+        :return: The message string.
+        """
+
+        try: return (await api_response.get_json())["error"]["message"]
+        except: return api_response.response.reason_phrase
+
+    # ┏┓┏┓┳  ┏┓  ┓┓•
+    # ┣┫┃┃┃  ┃ ┏┓┃┃┓┏┓┏┓
+    # ┛┗┣┛┻  ┗┛┗┻┗┗┗┛┗┗┫
+    #                  ┛
+
+    async def get(
+            self,
+            url: str,
+            headers: dict = None,
+            params: dict = None
+    ) -> GoogleApiResponse:
+
+        """
+        To call an API using the GET method.
+        :param url: The URL to call.
+        :param headers: The headers to pass.
+        :param params: The params to send in the query string itself.
+        :return: A structured response that includes the raw response, the exception (if any), and so on.
+        """
+
+        # Prepare the structure of the response:
+        api_response = GoogleApiResponse(
+            serviceName = self._service_name,
+            action = inspect.stack()[1].function,
+            url = url,
+            method = "GET"
+        )
+
+        try:
+
+            # Make the API call:
+            response = await self._http_client.get(
+                url = url,
+                headers = headers,
+                params = params
+            )
+
+            # Note down the results:
+            api_response.response = response
+            api_response.httpCode = response.status_code
+            api_response.message = await self.__get_error_message(api_response)
+
+        # If something goes wrong:
+        except Exception as exception:
+            api_response.exception = exception
+            api_response.message = str(exception)
+            self._printer(exception, api_response.url, api_response.method, headers, params)
+
+        # Done here:
+        return api_response
+
+    async def post(
+            self,
+            url: str,
+            headers: dict = None,
+            json: dict = None,
+            data: dict = None,
+            content: str | bytes = None
+    ) -> GoogleApiResponse:
+
+        """
+        To call an API using the POST method.
+        :param url: The URL to call.
+        :param headers: The headers to pass.
+        :param json: The params to send in the JSON body.
+        :param data: The params to send in the form-data in the body.
+        :param content: The raw content to be sent in the body (typically as an octet-stream).
+        :return: A structured response that includes the raw response, the exception (if any), and so on.
+        """
+
+        # Prepare the structure of the response:
+        api_response = GoogleApiResponse(
+            serviceName = self._service_name,
+            action = inspect.stack()[1].function,
+            url = url,
+            method = "POST"
+        )
+
+        try:
+
+            # Make the API call:
+            response = await self._http_client.post(
+                url = url,
+                headers = headers,
+                json = json,
+                data = data,
+                content = content
+            )
+
+            # Note down the results:
+            api_response.response = response
+            api_response.httpCode = response.status_code
+            api_response.message = await self.__get_error_message(api_response)
+
+        # If something goes wrong:
+        except Exception as exception:
+            api_response.exception = exception
+            api_response.message = str(exception)
+            self._printer(exception, api_response.url, api_response.method, headers, json, data)
+
+        # Done here:
+        return api_response
+
+    async def put(
+            self,
+            url: str,
+            headers: dict = None,
+            json: dict = None,
+            data: dict = None
+    ) -> GoogleApiResponse:
+
+        """
+        To call an API using the PUT method.
+        :param url: The URL to call.
+        :param headers: The headers to pass.
+        :param json: The params to send in the JSON body.
+        :param data: The params to send in the form-data in the body.
+        :return: A structured response that includes the raw response, the exception (if any), and so on.
+        """
+
+        # Prepare the structure of the response:
+        api_response = GoogleApiResponse(
+            serviceName = self._service_name,
+            action = inspect.stack()[1].function,
+            url = url,
+            method = "PUT"
+        )
+
+        try:
+
+            # Make the API call:
+            response = await self._http_client.put(
+                url = url,
+                headers = headers,
+                json = json,
+                data = data
+            )
+
+            # Note down the results:
+            api_response.response = response
+            api_response.httpCode = response.status_code
+            api_response.message = await self.__get_error_message(api_response)
+
+        # If something goes wrong:
+        except Exception as exception:
+            api_response.exception = exception
+            api_response.message = str(exception)
+            self._printer(exception, api_response.url, api_response.method, headers, json, data)
+
+        # Done here:
+        return api_response
+
+    async def delete(
+            self,
+            url: str,
+            headers: dict = None
+    ) -> GoogleApiResponse:
+
+        """
+        To call an API using the DELETE method.
+        :param url: The URL to call.
+        :param headers: The headers to pass.
+        :return: A structured response that includes the raw response, the exception (if any), and so on.
+        """
+
+        # Prepare the structure of the response:
+        api_response = GoogleApiResponse(
+            serviceName = self._service_name,
+            action = inspect.stack()[1].function,
+            url = url,
+            method = "DELETE"
+        )
+
+        try:
+
+            # Make the API call:
+            response = await self._http_client.delete(
+                url = url,
+                headers = headers
+            )
+
+            # Note down the results:
+            api_response.response = response
+            api_response.httpCode = response.status_code
+            api_response.message = await self.__get_error_message(api_response)
+
+        # If something goes wrong:
+        except Exception as exception:
+            api_response.exception = exception
+            api_response.message = str(exception)
+            self._printer(exception, api_response.url, api_response.method, headers)
+
+        # Done here:
+        return api_response
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

goog/gmail/gmail_client.py

@@ -0,0 +1,962 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 25th Nov., 2024
+
+    OBJECTIVE:
+
+        To manage e-mails in a GMail account.
+
+    REFERENCES:
+
+        1. GMail Quickstart: https://developers.google.com/gmail/api/quickstart/python
+        2. GMail Labels: https://developers.google.com/gmail/api/guides/labels
+        3. GMail Messages: https://developers.google.com/gmail/api/reference/rest/v1/users.messages
+        4. People Profile: https://developers.google.com/people/api/rest/v1/people/get
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+from utils_v2.mail import mail_parser
+
+# My Google utils:
+from utils_v2.goog.controllers.base import AsyncGoogleBase
+from utils_v2.goog.models.auth_tokens import GoogleAuthTokens
+from utils_v2.goog.models.api_call import GoogleApiResponse
+from utils_v2.goog.gmail.gmail_message import GMailMessage
+
+# Related to Google:
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from googleapiclient.discovery import build
+
+# To make API calls:
+import httpx
+
+# For asynchronous activities:
+import asyncio
+
+# To work with date and time:
+import datetime
+
+# For working with datatypes:
+from typing import Dict, Literal, List, Any
+
+# For debugging:
+from icecream import IceCreamDebugger
+import inspect
+
+# For computational help:
+import math
+
+# For base64 encoding:
+import base64
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Google Scopes:
+SCOPES_GMAIL_MAIL_MANAGEMENT = [
+    r"https://www.googleapis.com/auth/gmail.modify",
+    r"https://www.googleapis.com/auth/gmail.labels",
+    # r"profile",
+    r"https://www.googleapis.com/auth/userinfo.profile"
+]
+SCOPES_GMAIL_FULL = [
+    r"https://mail.google.com/",
+    # r"profile",
+    r"https://www.googleapis.com/auth/userinfo.profile"
+]
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncGMailClient(AsyncGoogleBase):
+
+    async def get_user_profile(
+            self,
+            tokens: GoogleAuthTokens,
+            user_id: str = "me",
+    ) -> GoogleApiResponse:
+
+        """
+        To get the list of labels of this user.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users/getProfile
+            2. https://developers.google.com/people/api/rest/v1/people/get
+            3. https://developers.google.com/people/api/rest/v1/people#Person
+        :param tokens: The object that holds the access token to the service.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the GMail API call:
+        if not self._debug_only_errors: self._printer("Getting User Profile.")
+        gmail_api_response = await self.get(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/profile",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"}
+        )
+
+        # If the call was successful:
+        if gmail_api_response.httpCode in [200]:
+            gmail_api_response.success = True
+            gmail_api_response.data = await gmail_api_response.get_json()
+            gmail_api_response.data["displayName"] = None
+            gmail_api_response.data["displayPictureUrl"] = None
+
+        # Make the People API call:
+        if not self._debug_only_errors: self._printer("Getting User Profile.")
+        people_api_response = await self.get(
+            url = f"https://people.googleapis.com/v1/people/me?personFields=names,photos,birthdays,phoneNumbers,genders,emailAddresses,addresses",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"}
+        )
+
+        # If the call was successful:
+        if people_api_response.httpCode in [200]:
+            people_api_response.success = True
+            people_api_response.data = await people_api_response.get_json()
+            for item in people_api_response.data.get("names", []):
+                if item["metadata"]["primary"]:
+                    gmail_api_response.data["displayName"] = item.get("displayName")
+            for item in people_api_response.data.get("photos", []):
+                if item["metadata"]["primary"]:
+                    gmail_api_response.data["displayPictureUrl"] = item.get("url")
+
+        # Done here:
+        return gmail_api_response
+
+    # ┓   ┓   ┓
+    # ┃ ┏┓┣┓┏┓┃┏
+    # ┗┛┗┻┗┛┗ ┗┛
+
+    async def list_labels(
+            self,
+            tokens: GoogleAuthTokens,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To get the list of labels of this user.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/list
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
+        :param tokens: The object that holds the access token to the service.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Listing All Labels.", user_id)
+        api_response = await self.get(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"}
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_json = await api_response.get_json()
+            api_response.data = {label["name"]: label for label in api_json.get("labels", [])}
+
+        # Done here:
+        return api_response
+
+    async def get_label(
+            self,
+            tokens: GoogleAuthTokens,
+            label_id: str,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To get one label of this user. the label will be identified by its id.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/get
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
+        :param tokens: The object that holds the access token to the service.
+        :param label_id: The id that Google assigned to the label.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Getting One Label.", user_id)
+        api_response = await self.get(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels/{label_id}",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"}
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_response.data = await api_response.get_json()
+
+        # Done here:
+        return api_response
+
+    async def create_label(
+            self,
+            tokens: GoogleAuthTokens,
+            label_name: str,
+            label_visibility: Literal["labelShow", "labelShowIfUnread", "labelHide"] = "labelShow",
+            message_visibility: Literal["show", "hide"] = "show",
+            label_text_color: str = "#434343",
+            label_background_color: str = "#000000",
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        Create one label for the user. Doesn't apply it to any mail, just creates it.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/create
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
+        :param tokens: The object that holds the access token to the service.
+        :param label_name: The display name of the label.
+        :param label_visibility: Whether, or not, you would like to show the label in the web UI.
+        :param message_visibility: Whether, or not, you would like to show messages with this label in the web UI.
+        :param label_text_color: The colour of the text of the label.
+        :param label_background_color: The colour of the background/tag of the label.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Creating One Label.", user_id)
+        api_response = await self.post(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+            json = {
+                "name": label_name,
+                "messageListVisibility": "show" if message_visibility else "hide",
+                "labelListVisibility": "labelShow" if label_visibility else "labelHide",
+                "color": {
+                    "textColor": label_text_color.lower(),
+                    "backgroundColor": label_background_color.lower()
+                }
+            }
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_response.data = await api_response.get_json()
+
+        # Done here:
+        return api_response
+
+    async def update_label(
+            self,
+            tokens: GoogleAuthTokens,
+            label_id: str,
+            label_name: str = None,
+            label_visibility: Literal["labelShow", "labelShowIfUnread", "labelHide"] = None,
+            message_visibility: Literal["show", "hide"] = None,
+            label_text_color: str = None,
+            label_background_color: str = None,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        Updates one label for the user.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/update
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
+        NOTE: Both or none of the colours must be updated. For this reason, a simple default will be chosen for the
+        other if only one is provided.
+        :param tokens: The object that holds the access token to the service.
+        :param label_id: The id that Google assigned to the label.
+        :param label_name: The display name of the label.
+        :param label_visibility: Whether, or not, you would like to show the label in the web UI.
+        :param message_visibility: Whether, or not, you would like to show messages with this label in the web UI.
+        :param label_text_color: The colour of the text of the label.
+        :param label_background_color: The colour of the background/tag of the label.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Format the JSON body:
+        json_body = {}
+        if label_name: json_body["name"] = label_name
+        if label_visibility: json_body["labelListVisibility"] = label_visibility
+        if message_visibility: json_body["messageListVisibility"] = message_visibility
+        if label_text_color or label_background_color:
+            json_body["color"] = {
+                "textColor": (label_text_color or "#434343").lower(),
+                "backgroundColor": (label_background_color or "#000000").lower()
+            }
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Updating One Label.", user_id)
+        api_response = await self.put(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels/{label_id}",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+            json = json_body
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_response.data = await api_response.get_json()
+
+        # Done here:
+        return api_response
+
+    async def delete_label(
+            self,
+            tokens: GoogleAuthTokens,
+            label_id: str,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To delete one label of this user. the label will be identified by its id.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/delete
+        :param tokens: The object that holds the access token to the service.
+        :param label_id: The id that Google assigned to the label.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Deleting One Label.", user_id)
+        api_response = await self.delete(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels/{label_id}",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"}
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200, 204]:
+            api_response.success = True
+
+        # Done here:
+        return api_response
+
+    # ┳┳┓
+    # ┃┃┃┏┓┏┏┏┓┏┓┏┓┏
+    # ┛ ┗┗ ┛┛┗┻┗┫┗ ┛
+    #           ┛
+
+    async def __list_messages_on_page(
+            self,
+            tokens: GoogleAuthTokens,
+            max_count: int = 100,
+            query: str = None,
+            label_ids: List[str] | str = None,
+            include_spam_and_trash: bool = False,
+            next_page_token: str = None,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To enlist messages on one page. Google allows at most 500 results on one page. This method respects that
+        pagination limit and returns only what Google gives. This method should be used internally by the class and the
+        class should expose another method that calls this one in loop to get any arbitrary no. of messages as the user
+        desires.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
+        :param tokens: The object that holds the access token to the service.
+        :param max_count: The no. of messages to fetch.
+        :param query: Any query filter that is supported by GMail.
+        :param label_ids: The list of labels' ids that the mails must have on them.
+        :param include_spam_and_trash: Whether, or not, you would like to include mails categorized as spam and trash.
+        :param next_page_token: The token to fetch the next set of results.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Build the needed params:
+        params_json = {
+            "maxResults": max_count,
+            "includeSpamTrash": include_spam_and_trash
+        }
+        if query: params_json["q"] = query
+        if next_page_token: params_json["pageToken"] = next_page_token
+        if label_ids: params_json["labelIds"] = label_ids if isinstance(label_ids, list) else [label_ids]
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Listing Messages for Page.", user_id, max_count, next_page_token)
+        api_response = await self.get(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+            params = params_json
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_json = await api_response.get_json()
+            api_response.data = {
+                "messages": {m["id"]: m for m in api_json.get("messages", [])},
+                "nextPageToken": api_json.get("nextPageToken"),
+                "resultSizeEstimate": api_json["resultSizeEstimate"],
+            }
+
+        # Done here:
+        return api_response
+
+    async def list_messages(
+            self,
+            tokens: GoogleAuthTokens,
+            max_count: int = 100,
+            query: str = None,
+            label_ids: List[str] | str = None,
+            include_spam_and_trash: bool = False,
+            next_page_token: str = None,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To enlist mail messages from a user's account.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
+        :param tokens: The object that holds the access token to the service.
+        :param max_count: The no. of messages to fetch.
+        :param query: Any query filter that is supported by GMail.
+        :param label_ids: The list of labels' ids that the mails must have on them.
+        :param include_spam_and_trash: Whether, or not, you would like to include mails categorized as spam and trash.
+        :param next_page_token: The token to fetch the next set of results.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # We create a variable that will hold the results.
+        # We must supply the URL, Method and a few other params here due to the custom looping functionality:
+        all_messages = GoogleApiResponse(
+            serviceName = self._service_name,
+            action = inspect.stack()[0].function,
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages",
+            method = "GET",
+            data = {
+                "messages": {},
+                "nextPageToken": None,
+                "resultSizeEstimate": 0
+            }
+        )
+
+        # Let's figure out how many times we'll have to loop through the process to retrieve the target no. of
+        # messages. Google allows you to fetch info about at most 500 messages in one go.
+        max_per_call = 500  # ... because Google allows at most 500 entries in one call.
+        iterations_needed = int(math.ceil(max_count / max_per_call))
+        last_iteration_count = max_count - int((max_per_call * (iterations_needed - 1)))
+
+        # Run the loop those many times:
+        results_size_estimate = 0
+        for iteration_no in range(iterations_needed):
+
+            # Figure out the count for this page:
+            if iterations_needed > 1:
+                if iteration_no < (iterations_needed - 1): iteration_count = max_per_call
+                else: iteration_count = last_iteration_count
+            else: iteration_count = max_count
+
+            # Retrieve the messages for this page:
+            iteration_response = await self.__list_messages_on_page(
+                tokens = tokens,
+                max_count = iteration_count,
+                query = query,
+                label_ids = label_ids,
+                include_spam_and_trash = include_spam_and_trash,
+                next_page_token = next_page_token
+            )
+
+            # Check if no data was received:
+            if (
+                    iteration_response.data is None or
+                    not iteration_response.data.get("messages")
+            ): break
+
+            # Now that we know that messages were received:
+            for k, v in iteration_response.data["messages"].items(): all_messages.data["messages"][k] = v
+            results_size_estimate += iteration_response.data["resultSizeEstimate"]
+
+            # Also copy the API call params:
+            all_messages.httpCode = iteration_response.httpCode
+            all_messages.message = iteration_response.message
+            all_messages.success = iteration_response.success
+
+            # If there is no next page after this, we break out of the loop:
+            next_page_token = iteration_response.data["nextPageToken"]
+            if next_page_token is None: break
+
+        # Format the final response:
+        all_messages.data["nextPageToken"] = next_page_token
+        all_messages.data["resultSizeEstimate"] = results_size_estimate
+
+        # Done here:
+        return all_messages
+
+    async def get_message(
+            self,
+            tokens: GoogleAuthTokens,
+            message_id: str,
+            return_raw: bool = False,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To get one message of this user. The message will be identified by its id. Note that, if you choose to return
+        the raw message, the message body will be compliant with RFC 5322 and RFC 2045 (among others).
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/get
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
+            2. https://developers.google.com/gmail/api/reference/rest/v1/Format
+        :param tokens: The object that holds the access token to the service.
+        :param message_id: The id that Google assigned to the message.
+        :param return_raw: Whether you want the raw message or the formatted message.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Getting One Message.", user_id)
+        api_response = await self.get(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/{message_id}",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+            params = {"format": "raw"}
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_json = await api_response.get_json()
+            raw_message = base64.urlsafe_b64decode(api_json["raw"]).decode()
+            if return_raw: api_response.data = raw_message
+            else:
+                parsed_message = mail_parser.parse(raw_message)
+                parsed_message["labels"] = api_json["labelIds"]
+                parsed_message["messageId"] = api_json["id"]
+                parsed_message["threadId"] = api_json["threadId"]
+                parsed_message["historyId"] = api_json["historyId"]
+                parsed_message["snippet"] = api_json["snippet"]
+                parsed_message["sizeEstimate"] = api_json["sizeEstimate"]
+                api_response.data = parsed_message
+
+        # Done here:
+        return api_response
+
+    async def modify_messages(
+            self,
+            tokens: GoogleAuthTokens,
+            message_ids: List[str] | str,
+            add_label_ids: List[str] | str = None,
+            remove_label_ids: List[str] | str = None,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To add or remove labels from one or more messages.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/batchModify
+        :param tokens: The object that holds the access token to the service.
+        :param message_ids: One or more message ids (assigned by Google).
+        :param add_label_ids: One or more label ids (not the display name of the label).
+        :param remove_label_ids: One or more label ids (not the display name of the label).
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Build the needed JSON:
+        body_json = {"ids": message_ids if isinstance(message_ids, list) else [message_ids]}
+        if add_label_ids: body_json["addLabelIds"] = add_label_ids if isinstance(add_label_ids, list) else [add_label_ids]
+        if remove_label_ids: body_json["removeLabelIds"] = remove_label_ids if isinstance(remove_label_ids, list) else [remove_label_ids]
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Modifying Message(s).", user_id)
+        api_response = await self.post(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/batchModify",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+            json = body_json
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200, 204]:
+            api_response.success = True
+
+        # Done here:
+        return api_response
+
+    async def delete_messages(
+            self,
+            tokens: GoogleAuthTokens,
+            message_ids: List[str] | str,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To PERMANENTLY delete one or more messages.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/batchDelete
+        :param tokens: The object that holds the access token to the service.
+        :param message_ids: One or more message ids (assigned by Google).
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Deleting Message(s).", user_id)
+        api_response = await self.post(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/batchDelete",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+            json = {"ids": message_ids if isinstance(message_ids, list) else [message_ids]}
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200, 204]:
+            api_response.success = True
+
+        # Done here:
+        return api_response
+
+    async def trash_message(
+            self,
+            tokens: GoogleAuthTokens,
+            message_id: str,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To move one message to trash.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/trash
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
+        :param tokens: The object that holds the access token to the service.
+        :param message_id: The id of the message (assigned by Google).
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Trashing One Message.", user_id)
+        api_response = await self.post(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/{message_id}/trash",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200, 204]:
+            api_response.success = True
+            api_response.data = await api_response.get_json()
+
+        # Done here:
+        return api_response
+
+    async def untrash_message(
+            self,
+            tokens: GoogleAuthTokens,
+            message_id: str,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To move one message to trash.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/trash
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
+        :param tokens: The object that holds the access token to the service.
+        :param message_id: The id of the message (assigned by Google).
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Un-Trashing One Message.", user_id)
+        api_response = await self.post(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/{message_id}/untrash",
+            headers = {"Authorization": f"Bearer {tokens.accessToken}"},
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_response.data = await api_response.get_json()
+
+        # Done here:
+        return api_response
+
+    async def send_message(
+            self,
+            tokens: GoogleAuthTokens,
+            message: GMailMessage,
+            thread_id: str = None,
+            user_id: str = "me"
+    ) -> GoogleApiResponse:
+
+        """
+        To send one message.
+        NOTE: If you want to apply a custom label to your outgoing mails, this API endpoint doesn't allow you to do that
+        in one go. Instead, you should note down the 'id' field from a successful response and use the 'modify_messages'
+        method of this class to immediately apply that label to the mail in a separate call.
+        DOCUMENTATION:
+            1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/send
+            2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
+        :param tokens: The object that holds the access token to the service.
+        :param message: The object that has the content of the message to be sent.
+        :param thread_id: Replies to an existing mail if the correct thread-id is specified. If not specified, a new
+            mail with a new thread-id is created.
+        :param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
+        :return: A structured response where the list of labels will be in the 'data' variable.
+        """
+
+        # Ensure that the tokens are valid:
+        await tokens.arefresh(
+            http_client = self._http_client,
+            client_id = self._client_id,
+            client_secret = self._client_secret,
+            force_refresh = False
+        )
+
+        # Construct the JSON body:
+        json_body = {"raw": message.get_raw_message(as_base64 = True)}
+        if thread_id: json_body["threadId"] = thread_id
+
+        # Make the API call:
+        if not self._debug_only_errors: self._printer("Sending One Message.", user_id)
+        api_response = await self.post(
+            url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/send",
+            headers = {
+                "Authorization": f"Bearer {tokens.accessToken}",
+                "Content-Type": "message/rfc822"
+            },
+            json = json_body
+        )
+
+        # If the call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            api_response.data = await api_response.get_json()
+
+        # Done here:
+        return api_response
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    # Create an HTTP client:
+    test_client = httpx.AsyncClient(
+        limits = httpx.Limits(
+            max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+            max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+        ),
+        timeout = httpx.Timeout(
+            connect = 2.5,  # ... Shorter connection timeout.
+            read = 2.5,  # ...... Like what EasyEcom gives.
+            write = 10.0,  # .... Time to wait for sending data.
+            pool = 120.0  # ..... Time to wait for a free connection from the pool.
+        )
+    )
+
+    # Read the secrets that give you access to the app:
+    secrets_file = r"../../../creds/goog/app/google_tcaoff_test_oauth_20241125.json"
+    secrets_dict = json.from_file(secrets_file)
+
+    async def main():
+
+        # Create an instance of the client:
+        my_gmail = AsyncGMailClient(
+            service_name = "gmail",
+            oauth_json = secrets_dict,
+            http_client = test_client,
+            redirect_url = r"https://api.thecaoffice.com/converse/mail/callback/gmail",
+            debug = True,
+            debug_prefix = "GMail (M) | ",
+            debug_only_errors = False
+        )
+
+        # Request Auth:
+        print("AUTH URL:", await my_gmail.get_authorization_url(
+            scopes = SCOPES_GMAIL_MAIL_MANAGEMENT,
+            state = "Bhopli",
+            approval_prompt = "force"
+        ))
+
+        # Get tokens from callback:
+        test_tokens = await my_gmail.get_authorization_tokens(
+            scopes = SCOPES_GMAIL_MAIL_MANAGEMENT,
+            redirect_url = input("Paste the redirect URL here: ")
+        )
+        print("TOKENS:", test_tokens)
+
+        # Test some feature:
+        response = await my_gmail.get_user_profile(tokens = test_tokens)
+        print("SUCCESS:", response.success)
+        print("SUMMARY:", response.to_markdown())
+        print("\n\n---\n\n")
+        print("DATA:", json.to_string(response.data, default = str))
+        if not response.success:
+            print("\n\n---\n\n")
+            print("FULL RESPONSE JSON:", json.to_string(await response.get_json()))
+
+
+    asyncio.run(main())

goog/gmail/gmail_message.py

@@ -0,0 +1,336 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 16th Jul, 2024
+
+    OBJECTIVE:
+
+        To be able to send out mails from code.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For working with mails:
+from email.mime.multipart import MIMEMultipart
+from email.mime.image import MIMEImage
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email import encoders
+
+# For random strings:
+import string
+import random
+
+# For system-level activities:
+import os
+
+# For working with files in RAM:
+import io
+
+# To work with Base64 encoding:
+import base64
+
+# To work with datatypes:
+from typing import List
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class GMailMessage:
+
+    def __init__(
+            self,
+            from_email: str,
+            to_email: str,
+            subject: str,
+            cc_emails: List[str] = None,
+            bcc_emails: List[str] = None
+    ):
+
+        """
+        Create an instance of the message that you would like to send.
+        :param from_email: The EMail ID of the sender.
+        :param to_email: The EMail ID of the recipient.
+        :param subject: The subject of the mail.
+        :param cc_emails: A list of recipients to add to the CC section.
+        :param bcc_emails: A list of recipients to add to the BCC section.
+        """
+
+        # Create the instance of the message:
+        self.message = MIMEMultipart()
+        self.message["From"] = from_email
+        self.message["To"] = to_email
+        self.message["Subject"] = subject
+        if cc_emails: self.message["CC"] = ",".join(cc_emails)
+        if bcc_emails: self.message["BCC"] = ",".join(bcc_emails)
+
+        # Note down the values for accessing later:
+        self.__from = from_email
+        self.__to = to_email
+        self.__cc = cc_emails,
+        self.__bcc = bcc_emails
+        self.__subject = subject
+
+    # ┏┓           •
+    # ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
+    # ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
+    #       ┛
+
+    @property
+    def from_mail(self):
+        return self.__from
+
+    @property
+    def to_mail(self):
+        return self.__to
+
+    @property
+    def cc_mails(self):
+        return self.__cc
+
+    @property
+    def bcc_mails(self):
+        return self.__bcc
+
+    @property
+    def subject(self):
+        return self.__subject
+
+    # ┏┓ ┓ ┓  ┏┓
+    # ┣┫┏┫┏┫  ┃ ┏┓┏┓╋┏┓┏┓╋
+    # ┛┗┗┻┗┻  ┗┛┗┛┛┗┗┗ ┛┗┗
+
+    def add_text(self, text):
+
+        """
+        Add plain-text to the mail body.
+        :param text: The text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(text, "plain"))
+
+    def add_html(self, html_text):
+
+        """
+        Add HTML text to the mail body.
+        :param html_text: The HTML text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(html_text, "html"))
+
+    def add_inline_image(
+            self,
+            image_file: str | io.BytesIO,
+            file_name: str = None,
+            content_id: str = None
+    ):
+
+        """
+        Add an inline image to the body of the mail.
+        NOTE: This is NOT the same as sending an image as an attachment.
+        :param image_file: The image data to attach to the mail body.
+        :param file_name: The name of the file. This is the same name by which it will be downloaded. You need not
+            specify this if the input file is specified as a path. Needed when you give the input file as a buffer.
+        :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not
+            specified, I will generate a random string. You may write a custom value here if you know what you are
+            doing. For most use cases, please ignore this field.
+        :return: None.
+        """
+
+        # Read the image as bytes:
+        image_bytes = None
+        if type(image_file) is str:
+            file_name = file_name or os.path.split(image_file)[-1]
+            with open(image_file, "rb") as opened_image_file:
+                image_bytes = opened_image_file.read()
+        if type(image_file) is io.BytesIO:
+            image_file.seek(0)
+            image_bytes = image_file.getvalue()
+
+        # Declare the part to be attached to the multipart message:
+        if image_bytes is not None:
+
+            # Create the HTML block if the image pointer is blank:
+            if content_id is None:
+                content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+                self.add_html(f"""
+                <html>
+                    <body>
+                        <p><img src="cid:{content_id}"></p>
+                    </body>
+                </html>
+                """)
+
+            # Then add the image:
+            image_part = MIMEImage(image_bytes)
+            image_part.add_header(
+                "Content-ID", f"<{content_id}>"
+            )
+            image_part.add_header(
+                "Content-Disposition",
+                f"inline; filename=\"{file_name}\"",
+            )
+            self.message.attach(image_part)
+
+    def add_attachment(
+            self,
+            attachment_file: str | io.BytesIO,
+            file_name: str = None
+    ):
+
+        """
+        Add a file as an attachment to the mail. This file, even if possible, will not be rendered on the screen in-line
+        with the body. It will be made available as a download.
+        :param attachment_file: The file that you would like to attach.
+        :param file_name: The name of the file. This is the same name by which it will be downloaded. You need not
+            specify this if the input file is specified as a path. Needed when you give the input file as a buffer.
+        :return:
+        """
+
+        # Declare the part to be attached to the multipart message:
+        part = MIMEBase("application", "octet-stream")
+
+        # If the attachment is a file stored in the local disk:
+        if isinstance(attachment_file, str):
+            file_name = file_name or os.path.split(attachment_file)[-1]
+            with open(attachment_file, "rb") as attachment:
+                part.set_payload(attachment.read())
+
+        # If the file is held in RAM:
+        elif isinstance(attachment_file, io.BytesIO):
+            attachment_file.seek(0)
+            part.set_payload(attachment_file.read())
+
+        # Encode and attach the file:
+        encoders.encode_base64(part)
+        part.add_header(
+            "Content-Disposition",
+            f"attachment; filename=\"{file_name}\"",
+        )
+        self.message.attach(part)
+
+    # ┏┓
+    # ┣ ┏┓┏┓┏┳┓┏┓╋
+    # ┻ ┗┛┛ ┛┗┗┗┻┗
+
+    def get_raw_message(
+            self,
+            as_base64: bool = True
+    ):
+
+        """
+        Get the raw string dump from the current contents of the message. This text will be compliant with RFC 5322 and
+        RFC 2045 (among others).
+        :param as_base64: If set to True, the response will be a URL-safe B64 output, else it'll be a raw string.
+        :return: The standardized raw text dump. either as a raw string or as a Base64 (url-safe) string.
+        """
+
+        if not as_base64: return self.message
+        else: return base64.urlsafe_b64encode(self.message.as_bytes()).decode()
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    my_mail = GMailMessage(
+        from_email = "sender@gmail.com",
+        to_email = "recipient@gmail.com",
+        subject = "Bhopli is the best!",
+        cc_emails = None,
+        bcc_emails = None
+    )
+    my_mail.add_html(
+        """
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <meta charset="UTF-8">
+            <title>Sample HTML String</title>
+            <style>
+                .heading {
+                    color: #ff9025;
+                }
+                .sub-heading {
+                    color: #000000;
+                }
+            </style>
+        </head>
+        <body>
+            <h1 class="heading">Hello, Bhopli!</h1>
+            <h2 class="sub-heading">Bhopli is the best, most well-behaved cat in the known universe.</h2>
+        </body>
+        </html>
+        """
+    )
+    my_mail.add_text("This is how you should pet her 👇")
+    my_mail.add_inline_image(r"../../../data/images/cat_petting.png")
+    my_mail.add_attachment(r"../../../data/pdf/sample_label.pdf")
+    print(my_mail.get_raw_message(as_base64 = True))

goog/gmail/json/messages/chatgpt_suggested_message_format.json

@@ -0,0 +1,69 @@
+{
+  "_id": ObjectId("..."),         // MongoDB auto-generated ID
+  "message_id": "unique-message-id",  // Unique identifier for the email
+  "thread_id": "thread-id",           // Optional: Group of related messages
+  "subject": "Subject of the email",
+  "from": {
+    "name": "Sender Name",              // Name of the sender
+    "email": "sender@example.com"        // Sender's email address
+  },
+  "to": [                               // List of recipients
+    {
+      "name": "Recipient Name",
+      "email": "recipient@example.com"
+    }
+  ],
+  "cc": [                               // List of CC recipients (optional)
+    {
+      "name": "CC Name",
+      "email": "cc@example.com"
+    }
+  ],
+  "bcc": [                              // List of BCC recipients (optional)
+    {
+      "name": "BCC Name",
+      "email": "bcc@example.com"
+    }
+  ],
+  "date": ISODate("2024-11-26T12:00:00Z"),  // Date the email was sent
+  "headers": {                          // Raw headers from the email
+    "X-Priority": "3",
+    "Content-Type": "multipart/alternative; boundary=\"boundary\"",
+    "X-Mailer": "Mailer XYZ"
+  },
+  "body": {                             // The body of the email, with parts if multipart
+    "text": "Plain text body content",  // Plain text part (if any)
+    "html": "<p>HTML body content</p>", // HTML part (if any)
+    "parts": [                          // List of parts for multipart emails
+      {
+        "content_type": "text/plain",
+        "content_transfer_encoding": "base64",
+        "content": "base64-encoded-content-here"
+      },
+      {
+        "content_type": "text/html",
+        "content_transfer_encoding": "base64",
+        "content": "base64-encoded-html-content-here"
+      }
+    ]
+  },
+  "attachments": [                      // Attachments in the email
+    {
+      "filename": "file1.pdf",
+      "content_type": "application/pdf",
+      "content_transfer_encoding": "base64",
+      "content": "base64-encoded-file-content"
+    },
+    {
+      "filename": "image1.jpg",
+      "content_type": "image/jpeg",
+      "content_transfer_encoding": "base64",
+      "content": "base64-encoded-image-content"
+    }
+  ],
+  "flags": {                            // Optional flags for internal tracking
+    "read": false,
+    "spam": false
+  },
+  "received_timestamp": ISODate("2024-11-26T12:01:00Z") // Time the email was received (optional)
+}

goog/gmail/json/messages/formatted_message_list_20241126.json

@@ -0,0 +1,36 @@
+{
+    "messages": {
+        "193669615aa33694": {
+            "threadId": "193669615aa33694"
+        },
+        "19365d0239aa1aaf": {
+            "threadId": "19365d0239aa1aaf"
+        },
+        "193641e1379da64d": {
+            "threadId": "193641e1379da64d"
+        },
+        "193640e74d93dc9b": {
+            "threadId": "193640e74d93dc9b"
+        },
+        "19363d6a7f50225a": {
+            "threadId": "19363d6a7f50225a"
+        },
+        "19363ba8e2582133": {
+            "threadId": "19363ba8e2582133"
+        },
+        "19362f9983e58ea6": {
+            "threadId": "19362f9983e58ea6"
+        },
+        "19360aabcb976cfc": {
+            "threadId": "19360aabcb976cfc"
+        },
+        "193604ad97c9fe62": {
+            "threadId": "193604ad97c9fe62"
+        },
+        "1935e92672e67f22": {
+            "threadId": "1935e92672e67f22"
+        }
+    },
+    "nextPageToken": "11954226706764006151",
+    "resultSizeEstimate": 402
+}

goog/gmail/json/messages/raw_message_02_19367930033154ca_with_attachment_20241126.json

@@ -0,0 +1,258 @@
+{
+    "id": "19367930033154ca",
+    "threadId": "19367930033154ca",
+    "labelIds": [
+        "UNREAD",
+        "IMPORTANT",
+        "CATEGORY_PERSONAL",
+        "INBOX"
+    ],
+    "snippet": "Sample content. Inline graphics done. Sample monospaced How about some Headers? Hello, World! These are quotes. How about some emojis? \ud83d\ude1a\ud83d\ude05\ud83d\ude05\ud83e\udd70\ud83d\ude18\ud83d\ude01\ud83d\ude01 There's also an attachment!",
+    "payload": {
+        "partId": "",
+        "mimeType": "multipart/mixed",
+        "filename": "",
+        "headers": [
+            {
+                "name": "Delivered-To",
+                "value": "pskhushal@gmail.com"
+            },
+            {
+                "name": "Received",
+                "value": "by 2002:a5d:4205:0:b0:382:44a8:2102 with SMTP id n5csp1327351wrq;        Tue, 26 Nov 2024 00:25:08 -0800 (PST)"
+            },
+            {
+                "name": "X-Received",
+                "value": "by 2002:a05:690c:6111:b0:6e3:fd6:6ccb with SMTP id 00721157ae682-6eee08c3459mr156525787b3.13.1732609507838;        Tue, 26 Nov 2024 00:25:07 -0800 (PST)"
+            },
+            {
+                "name": "ARC-Seal",
+                "value": "i=1; a=rsa-sha256; t=1732609507; cv=none;        d=google.com; s=arc-20240605;        b=VDostCh8Kr1QGeQoXgLjYyrkymtwKEVHTyPaNeSjy30Q0B4GTTIBIPbKgkL6UazG4T         s92UnE+Br4HJUSsqbtOdKTV1Pwt9lI3EzASDMX3OVGccdViTyUgGjgNuagV2fUVQugYi         BJWxLrWZXVVtKkLNqRDOEQgiPi6t869FTB4877V8dKxQs99Q8IBncekOq3f6+SmsFlIl         U3l6EUgaV/7MFzNRofsmXFGEG0TLto+Xgx5bMQ8B0OpjSFXR2Va/1xF5BmrzJTLgL6N5         zct3VIRLMxkf+8v3oniTG7yaaCQMPLxOq1pKXhP/MfKzgvdo5r14/kAzsj05rQ/L2/rT         2OrQ=="
+            },
+            {
+                "name": "ARC-Message-Signature",
+                "value": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605;        h=cc:to:subject:message-id:date:from:mime-version:dkim-signature;        bh=9NSLykdU3XzuxItzVTTQJjRRBR6yhJqdgYRarHa+VdY=;        fh=KtxgdfxhT3MZUXDfdB/Mxp08o9TbOvoJMvz1V2wYTKk=;        b=Xqtl2jfteUY2I+wdSwdllaqPKg1QP8hUyJpP/644zwGrOBdLbjCO8Po5C9KJfxUh7i         H7UMWfEWkulB52WFTDU/54UY8lnX7iGZEMtT/4twKUGToWPPWeG/wmh/I4W9ZEmA/qXG         OiRrBlJI3NJowsrnfoqXrpV/W5mz1SNuJBkj5k71+3vrPVlEmGwGmz8/MCG+RF2yV4nz         QEVg613QkuThPEEsgYdHbfWDlF8jnGbAS+Lyt8ZdixVvO4/Gq5swuGUHihfeV+Fxt5Kv         L2Wbqiq7UFJF9YJEtezJOdkdrTxs8dz6bTUZTrpWtRYxoBuH76HB+f9yrhOIRFTxor+9         18dA==;        dara=google.com"
+            },
+            {
+                "name": "ARC-Authentication-Results",
+                "value": "i=1; mx.google.com;       dkim=pass header.i=@gmail.com header.s=20230601 header.b=\"Adx/Ch2H\";       spf=pass (google.com: domain of orangebhopli@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=orangebhopli@gmail.com;       dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com;       dara=pass header.i=@gmail.com"
+            },
+            {
+                "name": "Return-Path",
+                "value": "<orangebhopli@gmail.com>"
+            },
+            {
+                "name": "Received",
+                "value": "from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])        by mx.google.com with SMTPS id 00721157ae682-6eee01062easor78223267b3.12.2024.11.26.00.25.07        (Google Transport Security);        Tue, 26 Nov 2024 00:25:07 -0800 (PST)"
+            },
+            {
+                "name": "Received-SPF",
+                "value": "pass (google.com: domain of orangebhopli@gmail.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;"
+            },
+            {
+                "name": "Authentication-Results",
+                "value": "mx.google.com;       dkim=pass header.i=@gmail.com header.s=20230601 header.b=\"Adx/Ch2H\";       spf=pass (google.com: domain of orangebhopli@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=orangebhopli@gmail.com;       dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com;       dara=pass header.i=@gmail.com"
+            },
+            {
+                "name": "DKIM-Signature",
+                "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=gmail.com; s=20230601; t=1732609507; x=1733214307; dara=google.com;        h=cc:to:subject:message-id:date:from:mime-version:from:to:cc:subject         :date:message-id:reply-to;        bh=9NSLykdU3XzuxItzVTTQJjRRBR6yhJqdgYRarHa+VdY=;        b=Adx/Ch2H1Vw5fzCEcIPf/EuHGviFMr1/Gw93WwruoabampXxraQ5hkcJNx0yi+zM2/         9wlmVqcDwiWM1dFrkc1hPNMyyH4h7hCXjGsDbK8hkUE/Fk1AR51fhjliB9AZFkps+PUR         21CYTDMzWWlHjKlBpnM1axU9suvv04NqDZ8Hr7LAXysd9eF1ypoTWjwWMJ5QGKpoQo6W         qf3m5yuqSXLAtbTKZN6K3qUO+S9ZRjUahw0eke7ieQ2uKR8dFwaoS899KhVBY5bZpLzr         gEjYkNbzZGN4bp/LIJcHmgueZ5J0L6V+zaYMIqNRp3wOwAe41nZSAf6PvLxixLZoTvyH         TmmA=="
+            },
+            {
+                "name": "X-Google-DKIM-Signature",
+                "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=1e100.net; s=20230601; t=1732609507; x=1733214307;        h=cc:to:subject:message-id:date:from:mime-version:x-gm-message-state         :from:to:cc:subject:date:message-id:reply-to;        bh=9NSLykdU3XzuxItzVTTQJjRRBR6yhJqdgYRarHa+VdY=;        b=F5yE7IslZpO2UHCSg9jCYE645+q7dW9WO9S0DIb4oRcTlz/qgI1fwnnB0vlWvgXQxJ         Af8/WUW6AJm/8mJ//SNXSDwE28OZiZqR+i/SYc0yontGepFawQAZg9xFRLspp7O/2UAL         EHrkssz9noe2wPPdcPGYanXfoiSLtX/3XnrGCg1P5Y4jdy7p6624f87Xx+Mno1t8q2xh         Odh2QrURKad50BqY+UB20+um9ommh4J0BOsD9WiVCfu5isvTCfwZYAtjYUJ48UVWLh3b         kAPAaSic4g1zQzlOychUnqFB95VrrODUNUACfKBuIR0YbUdsgjMzn9p4RQ1LAgr9NfvY         62kw=="
+            },
+            {
+                "name": "X-Forwarded-Encrypted",
+                "value": "i=1; AJvYcCVe+1Kjt1hhMEjYG2UyrzV/DvH4xn376Ya7lsLKkTIVIAJGmG8xKSkvvJAmXIAXRyYuWSNpPVXGyjSKXTgsLlU=@gmail.com"
+            },
+            {
+                "name": "X-Gm-Message-State",
+                "value": "AOJu0Ywb3oN+4RLkTx99fCaTMOlGwSTJR7AWBy7IvxJyBYNwLV6Of7c2 vtYqtaRl1CcUgUXmGnXqjfQsSX6DT4LLNPGCsrrzk4VXfV26FJA1iaDFm/ey7pRaAucm5dGPHRO 9oG7H+96wM0pzmY3Tlb9LUEY3NIdSR9H2"
+            },
+            {
+                "name": "X-Gm-Gg",
+                "value": "ASbGncuwgQwpQZrb7exEtts26PEAwkklAu9DD/4RvvBSiHYGDUD7+909oeyrMtfk66q IuA94Z8WcFFe3TabyhyXIjGTCEF5+Qg=="
+            },
+            {
+                "name": "X-Google-Smtp-Source",
+                "value": "AGHT+IFy7wXSaJCDrrXclKRzxMFtyv1HrgyWSfgyomoDnvBS4NPu9KBdzHjvcekKjw9uKzJAj5R2iOl+mZuCt5sjIws="
+            },
+            {
+                "name": "X-Received",
+                "value": "by 2002:a05:690c:6701:b0:6ee:b5a6:a67a with SMTP id 00721157ae682-6eee0a402acmr176495297b3.28.1732609506221; Tue, 26 Nov 2024 00:25:06 -0800 (PST)"
+            },
+            {
+                "name": "MIME-Version",
+                "value": "1.0"
+            },
+            {
+                "name": "From",
+                "value": "TheBhopli <orangebhopli@gmail.com>"
+            },
+            {
+                "name": "Date",
+                "value": "Tue, 26 Nov 2024 13:54:54 +0530"
+            },
+            {
+                "name": "Message-ID",
+                "value": "<CAA-t6761d=GwN3to+252S34rA=gjv+Ao+u5WVbz+Gj+5oiwb1g@mail.gmail.com>"
+            },
+            {
+                "name": "Subject",
+                "value": "Test Mail for GMail API."
+            },
+            {
+                "name": "To",
+                "value": "\"pskhushal@gmail.com\" <pskhushal@gmail.com>"
+            },
+            {
+                "name": "Cc",
+                "value": "\"khushal@easyfi.net.in\" <khushal@easyfi.net.in>, \"bhushan.thakkar@gmail.com\" <bhushan.thakkar@gmail.com>"
+            },
+            {
+                "name": "Content-Type",
+                "value": "multipart/mixed; boundary=\"00000000000027d46c0627cc96ea\""
+            }
+        ],
+        "body": {
+            "size": 0
+        },
+        "parts": [
+            {
+                "partId": "0",
+                "mimeType": "multipart/related",
+                "filename": "",
+                "headers": [
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/related; boundary=\"00000000000027d46c0627cc96e9\""
+                    }
+                ],
+                "body": {
+                    "size": 0
+                },
+                "parts": [
+                    {
+                        "partId": "0.0",
+                        "mimeType": "multipart/alternative",
+                        "filename": "",
+                        "headers": [
+                            {
+                                "name": "Content-Type",
+                                "value": "multipart/alternative; boundary=\"00000000000027d46c0627cc96e8\""
+                            }
+                        ],
+                        "body": {
+                            "size": 0
+                        },
+                        "parts": [
+                            {
+                                "partId": "0.0.0",
+                                "mimeType": "text/plain",
+                                "filename": "",
+                                "headers": [
+                                    {
+                                        "name": "Content-Type",
+                                        "value": "text/plain; charset=\"UTF-8\""
+                                    },
+                                    {
+                                        "name": "Content-Transfer-Encoding",
+                                        "value": "quoted-printable"
+                                    }
+                                ],
+                                "body": {
+                                    "size": 236,
+                                    "data": "U2FtcGxlIGNvbnRlbnQuDQoNCltpbWFnZTogY2hhdC5wbmddDQoNCklubGluZSBncmFwaGljcyBkb25lLg0KDQpTYW1wbGUgbW9ub3NwYWNlZA0KDQoqSG93IGFib3V0IHNvbWUgSGVhZGVycz8qDQoNCkhlbGxvLCBXb3JsZCEgVGhlc2UgYXJlIHF1b3Rlcy4NCg0KDQpIb3cgYWJvdXQgc29tZSBlbW9qaXM_IPCfmJrwn5iF8J-YhfCfpbDwn5iY8J-YgfCfmIENClRoZXJlJ3MgYWxzbyBhbiAqYXR0YWNobWVudCohDQo="
+                                }
+                            },
+                            {
+                                "partId": "0.0.1",
+                                "mimeType": "text/html",
+                                "filename": "",
+                                "headers": [
+                                    {
+                                        "name": "Content-Type",
+                                        "value": "text/html; charset=\"UTF-8\""
+                                    },
+                                    {
+                                        "name": "Content-Transfer-Encoding",
+                                        "value": "quoted-printable"
+                                    }
+                                ],
+                                "body": {
+                                    "size": 675,
+                                    "data": "PGRpdiBkaXI9Imx0ciI-U2FtcGxlIGNvbnRlbnQuPGRpdj48YnI-PGRpdj48aW1nIHNyYz0iY2lkOmlpX20zeTZ0dTg1MCIgYWx0PSJjaGF0LnBuZyIgd2lkdGg9IjIyMiIgaGVpZ2h0PSIyMjIiIHN0eWxlPSJtYXJnaW4tcmlnaHQ6IDBweDsiPjwvZGl2PjxkaXY-PGJyPjwvZGl2PjxkaXY-SW5saW5lIGdyYXBoaWNzIGRvbmUuPC9kaXY-PGRpdj48YnI-PC9kaXY-PGRpdj48Zm9udCBmYWNlPSJtb25vc3BhY2UiPlNhbXBsZSBtb25vc3BhY2VkPC9mb250PjwvZGl2PjxkaXY-PGJyPjwvZGl2PjxkaXY-PGI-PGZvbnQgc2l6ZT0iNiI-SG93IGFib3V0IHNvbWUgSGVhZGVycz88L2ZvbnQ-PC9iPjwvZGl2PjxkaXY-PGJyPjwvZGl2PjxibG9ja3F1b3RlIGNsYXNzPSJnbWFpbF9xdW90ZSIgc3R5bGU9Im1hcmdpbjowcHggMHB4IDBweCAwLjhleDtib3JkZXItbGVmdDoxcHggc29saWQgcmdiKDIwNCwyMDQsMjA0KTtwYWRkaW5nLWxlZnQ6MWV4Ij5IZWxsbywgV29ybGQhIFRoZXNlIGFyZSBxdW90ZXMuPC9ibG9ja3F1b3RlPjxkaXY-PGJyPjwvZGl2PjxkaXY-SG93IGFib3V0IHNvbWUgZW1vamlzP8Kg8J-YmvCfmIXwn5iF8J-lsPCfmJjwn5iB8J-YgTxicj48L2Rpdj48L2Rpdj48ZGl2PlRoZXJlJiMzOTtzIGFsc28gYW4gPGk-PHU-YXR0YWNobWVudDwvdT48L2k-ITwvZGl2PjwvZGl2Pg0K"
+                                }
+                            }
+                        ]
+                    },
+                    {
+                        "partId": "0.1",
+                        "mimeType": "image/png",
+                        "filename": "chat.png",
+                        "headers": [
+                            {
+                                "name": "Content-Type",
+                                "value": "image/png; name=\"chat.png\""
+                            },
+                            {
+                                "name": "Content-Disposition",
+                                "value": "inline; filename=\"chat.png\""
+                            },
+                            {
+                                "name": "Content-Transfer-Encoding",
+                                "value": "base64"
+                            },
+                            {
+                                "name": "Content-ID",
+                                "value": "<ii_m3y6tu850>"
+                            },
+                            {
+                                "name": "X-Attachment-Id",
+                                "value": "ii_m3y6tu850"
+                            }
+                        ],
+                        "body": {
+                            "attachmentId": "ANGjdJ88SNHhnjq5-Alk71WYYaL72CUhJn6_geE_GP7JZKZdcSIR56LPGVklvFO_IuMFqeDkDT2U4r8SXTSMb-uBnNrio-NeXiW2p8vPWnV2yq-YaZxG0xBD6WILt5Qwg_43lv9_TGnjSvWn5QrGzusOttPluVRIWtv0-dksvYXrHDbS5MC0IYX4qpv8dbvGto5rgH_iIsIMav2aXbm5_o_mmxlxO9GavC8G9ZWD9BbuLlKKMJQAXgGXlfOlWA1Ukr5Cr0UO5LZbeTtVX3dZMYs3QfMwlGkVLT7Gam6aZfW8JduWhObFP9ghg2haF4tooXcUFNOPX7e2Axto-OmYAXLaPZRh-xpSzEObOeQv8gAHihoXlkA5Y_A2m45-emjdSiwff2Ui0VRZs96_DMGO",
+                            "size": 17466
+                        }
+                    }
+                ]
+            },
+            {
+                "partId": "1",
+                "mimeType": "image/png",
+                "filename": "chat.png",
+                "headers": [
+                    {
+                        "name": "Content-Type",
+                        "value": "image/png; name=\"chat.png\""
+                    },
+                    {
+                        "name": "Content-Disposition",
+                        "value": "attachment; filename=\"chat.png\""
+                    },
+                    {
+                        "name": "Content-Transfer-Encoding",
+                        "value": "base64"
+                    },
+                    {
+                        "name": "Content-ID",
+                        "value": "<f_m3y6xx8e1>"
+                    },
+                    {
+                        "name": "X-Attachment-Id",
+                        "value": "f_m3y6xx8e1"
+                    }
+                ],
+                "body": {
+                    "attachmentId": "ANGjdJ-XMpv18VYpIKT1zc45Su99KXSI3r3YaVquSLfKE3DqgQ7XjO7p4iHRHvc47lIr70rLBk2H8lLlMifTCFxFLMy0q_uFgsCHASCDeq-opjPkeDxC3BaxACeyeNWaZYANXgUCFYqCygTLkIoeggJDBxmzPn99baxvoJqcRLXROO1mSWGB4K0PLJcP35r8Y2D9lVRjSAJupzf7HLYLzimY1Q-X6Ge2qNCNnree-rS47FNXQgvFfVdV_x7jcvLE7JJl5cBBIyus-TtTulMk0RrFbcn5zKH7AniW6IF7v-mLOUzpDVye9tIiVqc6W4EdLeP3wmIaMVy4shsAdHkKtbjwLxicycke8Vi5hAV50s0pCllN26rSHakAA9zswG9q5z3cbS5LNMdEUpgWt6ON",
+                    "size": 17466
+                }
+            }
+        ]
+    },
+    "sizeEstimate": 55270,
+    "historyId": "528587",
+    "internalDate": "1732609494000"
+}

goog/gmail/json/messages/raw_message_07_19362f9983e58ea6_20241126.json

@@ -0,0 +1,210 @@
+{
+    "id": "19362f9983e58ea6",
+    "threadId": "19362f9983e58ea6",
+    "labelIds": [
+        "CATEGORY_PROMOTIONS",
+        "UNREAD",
+        "INBOX"
+    ],
+    "snippet": "INDIAN FILM & TELEVISION DIRECTORS' ASSOCIATION G- 8/9/10, Crescent Towers, Near Morya House, Andheri (W), Mumbai - 400 053 Tel: 022 46088096 Mob: 9892885346/ 7021494476 www.directorsiftda.com",
+    "payload": {
+        "partId": "",
+        "mimeType": "multipart/related",
+        "filename": "",
+        "headers": [
+            {
+                "name": "Delivered-To",
+                "value": "pskhushal@gmail.com"
+            },
+            {
+                "name": "Received",
+                "value": "by 2002:a5d:4205:0:b0:382:44a8:2102 with SMTP id n5csp832066wrq;        Mon, 25 Nov 2024 02:59:05 -0800 (PST)"
+            },
+            {
+                "name": "X-Forwarded-Encrypted",
+                "value": "i=2; AJvYcCWTJ16JRatSfV54WvhqvY7z3mW9LUmKJsBYYVJp5gas68X0OXWAugWTJyyEQE0ESerQAquXl7ZIugA=@gmail.com"
+            },
+            {
+                "name": "X-Received",
+                "value": "by 2002:a17:906:3188:b0:a99:7bc0:bca9 with SMTP id a640c23a62f3a-aa50990b300mr1091084466b.3.1732532345051;        Mon, 25 Nov 2024 02:59:05 -0800 (PST)"
+            },
+            {
+                "name": "ARC-Seal",
+                "value": "i=1; a=rsa-sha256; t=1732532345; cv=none;        d=google.com; s=arc-20240605;        b=cGPTlt9iJZomIZrEX1YxFZt7CeiQG3g/9XJ/DHe6g6ghGuBopYyGtWRIq/b0u5ezsK         3JyasTQwmCnqAmvhDEgUZQM/f2Bk8PshykVmLfSUzhP0vUS2wfg90N1jmaMTefDGFvlF         lSjmHBlgpl4Vhn4BKGBnLuN3ttrMXvyZ6Vi4xGil9yDgW+KjwrL3fN2jdlFsOvhxODZR         EE7q5SC0waIenSAllTd+0CcG4TrH7Q0ZrQYR0yVH98JZ4UJN6v5xqhwBmTMtlPx3mWw2         d8H5HXwevTXeYzYf/F7iKAlaaWu2IPM7k/e3HAatqQk/pKHfh6cmYqN1G/MzENERvOsk         HmUQ=="
+            },
+            {
+                "name": "ARC-Message-Signature",
+                "value": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605;        h=to:subject:message-id:date:from:mime-version:dkim-signature;        bh=52sadrERSC8TqD31OwCOGWYBr43KUD3rPyq2rw8jz2Y=;        fh=3tsGDR7RWsLBUvHMNbctop2/zoZmFmKl/B1BkNJ2jsE=;        b=N9JHVDUsSOQO9HPax1y++VDKEbumii3FfXCPi2ec+NDD0lV4w+48dqXsrPO9k8XO03         FgGp+qnSgZps8TogBykk8a/bFPO4VJ0AZgt5b7v4i08Y+auiO+ywcP37yeZjB52iGVSU         U0TcBLTtqQSBtlirHN4lG5t4CvNTgDhKDwr9UjvOqtLgValnrsySlSpoQQ+CRhaxC3Pq         bEVn4CKh8dGlJWVvOA037UDZFBQtR6ygY7IbiXBEIFZsC49aH4lZbpyzA+Xh3H3X0Xtg         LrM/0hxRNj33PLOvbm5gdS7F32djBbaoIXRl057QZV0JQbPa1fvEeDOwJCv8H6rECfHK         m0UQ==;        dara=google.com"
+            },
+            {
+                "name": "ARC-Authentication-Results",
+                "value": "i=1; mx.google.com;       dkim=pass header.i=@directorsiftda-com.20230601.gappssmtp.com header.s=20230601 header.b=h3FDhhUH;       spf=fail (google.com: domain of contact@directorsiftda.com does not designate 209.85.220.41 as permitted sender) smtp.mailfrom=contact@directorsiftda.com;       dara=pass header.i=@gmail.com"
+            },
+            {
+                "name": "Return-Path",
+                "value": "<contact@directorsiftda.com>"
+            },
+            {
+                "name": "Received",
+                "value": "from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])        by mx.google.com with SMTPS id a640c23a62f3a-aa560bdd42csor237566b.0.2024.11.25.02.59.04        for <pskhushal@gmail.com>        (Google Transport Security);        Mon, 25 Nov 2024 02:59:04 -0800 (PST)"
+            },
+            {
+                "name": "Received-SPF",
+                "value": "fail (google.com: domain of contact@directorsiftda.com does not designate 209.85.220.41 as permitted sender) client-ip=209.85.220.41;"
+            },
+            {
+                "name": "Authentication-Results",
+                "value": "mx.google.com;       dkim=pass header.i=@directorsiftda-com.20230601.gappssmtp.com header.s=20230601 header.b=h3FDhhUH;       spf=fail (google.com: domain of contact@directorsiftda.com does not designate 209.85.220.41 as permitted sender) smtp.mailfrom=contact@directorsiftda.com;       dara=pass header.i=@gmail.com"
+            },
+            {
+                "name": "DKIM-Signature",
+                "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=directorsiftda-com.20230601.gappssmtp.com; s=20230601; t=1732532344; x=1733137144; dara=google.com;        h=to:subject:message-id:date:from:mime-version:from:to:cc:subject         :date:message-id:reply-to;        bh=52sadrERSC8TqD31OwCOGWYBr43KUD3rPyq2rw8jz2Y=;        b=h3FDhhUHelnzSe5+as+rP+HnwiQlFmbqO/XDImaXOHroIQfkSRt1MAwH2o8MiNS07l         fzAKjxqRmO+p+fy9FyRuQ7d0/TDALivdP4HW1CpU7nHUqqp8BamVDC06zS1RI2PCVo7s         qp4cxTIJsFTVaMauaTWQCIj9xstz9lGMa7DuhyJ8deSJU6zIAn0rFOhLISFZfZkVtZEy         Q4J4MBtf2DZVvaxqUNr/DqkaownkPsTWaxfXTE8Ztq41a9kb+2IfX+6kk6M80DF4pm8I         5Wju6mVd6xpms0UXJPtRPfRBFKEjEApYq6dypd37ETr+tkUynPrv10xfuSUvrZdzU73L         1uEQ=="
+            },
+            {
+                "name": "X-Google-DKIM-Signature",
+                "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=1e100.net; s=20230601; t=1732532344; x=1733137144;        h=to:subject:message-id:date:from:mime-version:x-gm-message-state         :from:to:cc:subject:date:message-id:reply-to;        bh=52sadrERSC8TqD31OwCOGWYBr43KUD3rPyq2rw8jz2Y=;        b=tQU+3IcgiGI90Bknb75WOMN2IpL9BIunrIP5KwfkJMvrRTWq9GzjUl0t6gUocRNiPn         1fJ2VgBd/nAvzUpg/2lcypTBbfp7019CxuW5aqungvCD5iWFzMUcOoudN0O4kMx0KrMX         DFThj9pD951evE/jteODFBQt0886+8Ofo8KIWaCsSiJ7uerf968pWkkJm6lkHsMCFf97         wgiA4kN8X8i9d+ZFs2yo5MxZykY1Ll2YYmMozgIS6jsXLb0pveLtA1L0SFxbqRU+/Njf         n6ibGo42xOWOM92qDEbJnDXNtlnOEVOAKA3sgazkG8pDpC9yf2RqMPWex9xCNAbbVNQE         KTog=="
+            },
+            {
+                "name": "X-Forwarded-Encrypted",
+                "value": "i=1; AJvYcCWtfq6Mi9XsVN+r9oOb64yoh/F6ROqg/9C0UMWDuXU/fz1A74BwcwwZQy0hyIykg2dwtgxUFBLIz0w=@gmail.com"
+            },
+            {
+                "name": "X-Gm-Message-State",
+                "value": "AOJu0YxUy7pf/My+et+YkWa4bqQ26C2PeklIXRKdME1VkRVdK9A8dtmN p34rPukc6VEwoEvOLsIh42JHO+nLEOYgcdIMTfklpdIp+2SaamAG+j/zSQFpfdFhWov4PO2vbqx aOztICe4xCfaWEiv4jKQ3arZkHmUWwrT6xb1sOQ=="
+            },
+            {
+                "name": "X-Gm-Gg",
+                "value": "ASbGncsXwRmXGJJAyTvAKY+pMS2y7/Mf8mmUI9AmHaycDBvZnbQkc20j125cXkjxmOG o3M/KYTcTZDKyHLOi5xRqMfOv1Qs8lwFelQ=="
+            },
+            {
+                "name": "X-Google-Smtp-Source",
+                "value": "AGHT+IFopa4irNtf4tkqOS0N6Tc13BHGQQ6fNDsmqoW9klWPS/JD7cYNG9eMDQkP9k7DPZCzWaTq4bPYSLxo6IJY2OU="
+            },
+            {
+                "name": "X-Received",
+                "value": "by 2002:a17:906:1daa:b0:aa5:3853:5532 with SMTP id a640c23a62f3a-aa53853589dmr630025266b.43.1732532343546; Mon, 25 Nov 2024 02:59:03 -0800 (PST)"
+            },
+            {
+                "name": "MIME-Version",
+                "value": "1.0"
+            },
+            {
+                "name": "From",
+                "value": "Iftda India <contact@directorsiftda.com>"
+            },
+            {
+                "name": "Date",
+                "value": "Mon, 25 Nov 2024 16:28:50 +0530"
+            },
+            {
+                "name": "Message-ID",
+                "value": "<CAAPQQGQPeA-z38EPhBSyMPmhyuWzc2uRyibVMg6uLd5V8hr5VQ@mail.gmail.com>"
+            },
+            {
+                "name": "Subject",
+                "value": "Sad Demise of Mr. Jalaj Dhir"
+            },
+            {
+                "name": "To",
+                "value": "undisclosed-recipients:;"
+            },
+            {
+                "name": "Content-Type",
+                "value": "multipart/related; boundary=\"000000000000e67bea0627ba9e63\""
+            },
+            {
+                "name": "Bcc",
+                "value": "pskhushal@gmail.com"
+            }
+        ],
+        "body": {
+            "size": 0
+        },
+        "parts": [
+            {
+                "partId": "0",
+                "mimeType": "multipart/alternative",
+                "filename": "",
+                "headers": [
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/alternative; boundary=\"000000000000e67be90627ba9e62\""
+                    }
+                ],
+                "body": {
+                    "size": 0
+                },
+                "parts": [
+                    {
+                        "partId": "0.0",
+                        "mimeType": "text/plain",
+                        "filename": "",
+                        "headers": [
+                            {
+                                "name": "Content-Type",
+                                "value": "text/plain; charset=\"UTF-8\""
+                            }
+                        ],
+                        "body": {
+                            "size": 672,
+                            "data": "W2ltYWdlOiBjb25kb2xlbmNlcyBNci4gSkFMQUogREhJUi5qcGddDQoNCg0KKklORElBTioqIEZJTE0gJiBURUxFVklTSU9OIERJUkVDVE9SUycgQVNTT0NJQVRJT04qDQpHLSA4LzkvMTAsIENyZXNjZW50IFRvd2VycywNCk5lYXIgTW9yeWEgSG91c2UsDQpBbmRoZXJpIChXKSwgTXVtYmFpIC0gNDAwIDA1Mw0KVGVsOiAwMjIgNDYwODgwOTYNCk1vYjogOTg5Mjg4NTM0Ni8gNzAyMTQ5NDQ3Ng0Kd3d3LmRpcmVjdG9yc2lmdGRhLmNvbSB8IEZhY2Vib29rL2RpcmVjdG9yc2lmdGRhDQoNCipESVNDTEFJTUVSOiogVGhpcyBlLW1haWwgbWF5IGJlIHByaXZpbGVnZWQgYW5kL29yIGNvbmZpZGVudGlhbCwgYW5kIHRoZQ0Kc2VuZGVyIGRvZXMgbm90IHdhaXZlIGFueSByZWxhdGVkIHJpZ2h0cyBhbmQgb2JsaWdhdGlvbnMuIEFueSBkaXN0cmlidXRpb24sDQp1c2Ugb3IgY29weWluZyBvZiB0aGlzIGUtbWFpbCBvciB0aGUgaW5mb3JtYXRpb24gaXQgY29udGFpbnMgYnkgb3RoZXIgdGhhbg0KYW4gaW50ZW5kZWQgcmVjaXBpZW50KHMpIGlzIHVuYXV0aG9yaXplZC4gSWYgeW91IHJlY2VpdmVkIHRoaXMgZS1tYWlsIGluDQplcnJvciwgcGxlYXNlIGFkdmlzZSB1cyAoYnkgcmV0dXJuIGUtbWFpbCBvciBvdGhlcndpc2UpIGltbWVkaWF0ZWx5IGFuZA0KZGVsZXRlIHRoaXMgZS1tYWlsLg0K"
+                        }
+                    },
+                    {
+                        "partId": "0.1",
+                        "mimeType": "text/html",
+                        "filename": "",
+                        "headers": [
+                            {
+                                "name": "Content-Type",
+                                "value": "text/html; charset=\"UTF-8\""
+                            },
+                            {
+                                "name": "Content-Transfer-Encoding",
+                                "value": "quoted-printable"
+                            }
+                        ],
+                        "body": {
+                            "size": 2187,
+                            "data": "PGRpdiBkaXI9Imx0ciI-PGRpdj48ZGl2IGNsYXNzPSJnbWFpbF9kZWZhdWx0IiBzdHlsZT0iZm9udC1mYW1pbHk6JnF1b3Q7dHJlYnVjaGV0IG1zJnF1b3Q7LHNhbnMtc2VyaWY7Y29sb3I6cmdiKDY4LDY4LDY4KSI-PC9kaXY-PGltZyBzcmM9ImNpZDppaV9tM3d4MGo5YzAiIGFsdD0iY29uZG9sZW5jZXMgTXIuIEpBTEFKIERISVIuanBnIiB3aWR0aD0iNDcyIiBoZWlnaHQ9IjMzNCI-PGJyPjxiciBjbGVhcj0iYWxsIj48L2Rpdj48ZGl2PjxkaXYgZGlyPSJsdHIiIGNsYXNzPSJnbWFpbF9zaWduYXR1cmUiIGRhdGEtc21hcnRtYWlsPSJnbWFpbF9zaWduYXR1cmUiPjxkaXYgZGlyPSJsdHIiPjxkaXY-PGRpdiBkaXI9Imx0ciI-PGRpdj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2PjxzcGFuPjxzcGFuIHN0eWxlPSJjb2xvcjpyZ2IoNjgsNjgsNjgpIj48c3Bhbj48Yj48Zm9udCBzaXplPSI0Ij48YnI-PC9mb250PjwvYj48L3NwYW4-PC9zcGFuPjwvc3Bhbj48ZGl2Pjxmb250IGZhY2U9InRhaG9tYSwgc2Fucy1zZXJpZiI-PGZvbnQ-PGI-SU5ESUFOPC9iPjwvZm9udD48Yj7CoEZJTE0gJmFtcDsgVEVMRVZJU0lPTiBESVJFQ1RPUlMmIzM5OyBBU1NPQ0lBVElPTjwvYj48L2ZvbnQ-PC9kaXY-PGRpdj48Zm9udCBmYWNlPSJ0YWhvbWEsIHNhbnMtc2VyaWYiPkctIDgvOS8xMCwgQ3Jlc2NlbnQgVG93ZXJzLMKgPGJyPjwvZm9udD48L2Rpdj48ZGl2IGRpcj0ibHRyIj48ZGl2Pjxmb250IGZhY2U9InRhaG9tYSwgc2Fucy1zZXJpZiI-TmVhciBNb3J5YSBIb3VzZSw8L2ZvbnQ-PC9kaXY-PGRpdj48Zm9udCBmYWNlPSJ0YWhvbWEsIHNhbnMtc2VyaWYiPkFuZGhlcmkgKFcpLCBNdW1iYWkgLSA0MDAgMDUzPC9mb250PjwvZGl2PjxkaXY-PHNwYW4gc3R5bGU9InRleHQtYWxpZ246Y2VudGVyIj5UZWw6wqA8L3NwYW4-PHNwYW4gc3R5bGU9InRleHQtYWxpZ246Y2VudGVyIj4wMjIgNDYwODgwOTY8L3NwYW4-PGZvbnQgZmFjZT0idGFob21hLCBzYW5zLXNlcmlmIj48YnI-PC9mb250PjwvZGl2PjxkaXY-PGZvbnQgZmFjZT0idGFob21hLCBzYW5zLXNlcmlmIj5Nb2I6IDxzcGFuIHN0eWxlPSJ0ZXh0LWFsaWduOmNlbnRlciI-OTg5Mjg4NTM0Ni8gNzAyMTQ5NDQ3Njwvc3Bhbj48L2ZvbnQ-PC9kaXY-PC9kaXY-PC9kaXY-PGRpdj48ZGl2IGRpcj0ibHRyIj48ZGl2PjxzcGFuIHN0eWxlPSJjb2xvcjpyZ2IoNjgsNjgsNjgpIj48Zm9udCBmYWNlPSJ0YWhvbWEsIHNhbnMtc2VyaWYiPjxmb250IHNpemU9IjIiPnd3dy5kaXJlY3RvcnNpZnRkYTwvZm9udD4uY29tIHwgRmFjZWJvb2svZGlyZWN0b3JzaWZ0ZGE8L2ZvbnQ-PGJyPjxicj48c3BhbiBzdHlsZT0iZm9udC1zaXplOjEyLjhweCI-PGI-PGk-PHU-PHNwYW4gc3R5bGU9ImZvbnQtc2l6ZTo4cHQiIGxhbmc9IkVOLVVTIj5ESVNDTEFJTUVSOjwvc3Bhbj48L3U-PC9pPjwvYj48c3BhbiBzdHlsZT0iZm9udC1zaXplOjhwdCI-PC9zcGFuPjxzcGFuIHN0eWxlPSJmb250LXNpemU6OHB0O2JhY2tncm91bmQ6d2hpdGUgbm9uZSByZXBlYXQgc2Nyb2xsIDAlIDAlIj4gVGhpcw0KIGUtbWFpbCBtYXkgYmUgcHJpdmlsZWdlZCBhbmQvb3IgY29uZmlkZW50aWFsLCBhbmQgdGhlIHNlbmRlciBkb2VzIG5vdCANCndhaXZlIGFueSByZWxhdGVkIHJpZ2h0cyBhbmQgb2JsaWdhdGlvbnMuIEFueSBkaXN0cmlidXRpb24sIHVzZSBvciANCmNvcHlpbmcgb2YgdGhpcyBlLW1haWwgb3IgdGhlIGluZm9ybWF0aW9uIGl0IGNvbnRhaW5zIGJ5IG90aGVyIHRoYW4gYW4gDQppbnRlbmRlZCByZWNpcGllbnQocykgaXMgdW5hdXRob3JpemVkLiBJZiB5b3UgcmVjZWl2ZWQgdGhpcyBlLW1haWwgaW4gDQplcnJvciwgcGxlYXNlIGFkdmlzZSB1cyAoYnkgcmV0dXJuIGUtbWFpbCBvciBvdGhlcndpc2UpIGltbWVkaWF0ZWx5IGFuZCANCmRlbGV0ZSB0aGlzIGUtbWFpbC48L3NwYW4-PHNwYW4gc3R5bGU9ImZvbnQtc2l6ZTo4cHQiPjwvc3Bhbj48L3NwYW4-PC9zcGFuPjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2Pg0K"
+                        }
+                    }
+                ]
+            },
+            {
+                "partId": "1",
+                "mimeType": "image/jpeg",
+                "filename": "condolences Mr. JALAJ DHIR.jpg",
+                "headers": [
+                    {
+                        "name": "Content-Type",
+                        "value": "image/jpeg; name=\"condolences Mr. JALAJ DHIR.jpg\""
+                    },
+                    {
+                        "name": "Content-Disposition",
+                        "value": "inline; filename=\"condolences Mr. JALAJ DHIR.jpg\""
+                    },
+                    {
+                        "name": "Content-Transfer-Encoding",
+                        "value": "base64"
+                    },
+                    {
+                        "name": "Content-ID",
+                        "value": "<ii_m3wx0j9c0>"
+                    },
+                    {
+                        "name": "X-Attachment-Id",
+                        "value": "ii_m3wx0j9c0"
+                    }
+                ],
+                "body": {
+                    "attachmentId": "ANGjdJ8zCv8y-pC8ecFP7IVoYsW4sTkayvcSepjUwiCDIrSUTJPBua7Ack8YvSuFPYcM-np_PeWRTl_wq5dYpyz3k6Hc5e-tp4RZjT6t2ITBDpkqJhPn17eR36sVN41ojkJ63Wzd9HD60LDk3o4GiL8C6iyMlsHMhadvOXKYK5zITlU9mvtdMoye6fofiiLwRsoNuT9dEpBbk9r2gh-vmuYpbu0waybh_EKefEejDvuOl7cIFU1Y3Dv8kxWYQwtZ1kx1LhHF24TzisZjvBaF6cno1T6f9aTdNd3x5T0_YsgJNilfU3xdWbCYilbVACU8axpIN5mC9CwhD45ogliA9kLadSpig8KmW9MNJJxr1vMnOAI4G-zr1SAhkzm6ZXcHllk9WpncS8RRRaLilLxo",
+                    "size": 1736748
+                }
+            }
+        ]
+    },
+    "sizeEstimate": 2385712,
+    "historyId": "527104",
+    "internalDate": "1732532330000"
+}

goog/models/api_call.py

@@ -0,0 +1,132 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Wednesday, 30th Oct., 2024.
+
+    OBJECTIVE:
+
+        To provide a data model for describing the API response from Google's APIs.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For making data behaviour_models:
+from pydantic import BaseModel, Field, field_validator, model_validator
+from typing import Optional, Literal, Union, Dict, List, Any
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class GoogleApiResponse(BaseModel):
+
+    serviceName: str = Field(frozen = True, default = None)
+    action: str = Field(frozen = True, default = None)
+
+    url: str = Field(frozen = True)
+    method: str = Field(frozen = True)
+    response: Any = None
+    httpCode: int = None
+
+    success: bool = False
+    message: str = None
+    data: Any = None
+
+    exception: Any = None
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "forbid"
+
+    # ┏┓           ┏┓
+    # ┃ ┓┏┏╋┏┓┏┳┓  ┣ ┓┏┏┓┏┏
+    # ┗┛┗┻┛┗┗┛┛┗┗  ┻ ┗┻┛┗┗┛
+
+    def to_markdown(self):
+        if self.exception: message = "❌ *GOOGLE API EXCEPTION:* ❌\n\n"
+        else: message = "*GOOGLE API RESPONSE:*\n\n"
+        message += f"*SERVICE:*\n`{self.serviceName}`\n\n"
+        message += f"*ACTION:*\n`{self.action}`\n\n"
+        message += f"*URL:*\n`{self.url}`\n\n"
+        message += f"*METHOD:*\n`{self.method}`\n\n"
+        message += f"*RESPONSE:*\n`{self.response}`\n\n"
+        message += f"*MESSAGE:*\n`{self.message}`\n\n"
+        message += f"*EXCEPTION:*\n`{self.exception.__class__.__name__}: {str(self.exception)}`\n\n"
+        return message
+
+    async def get_json(self):
+        try: return self.response.json()
+        except: return {}
+
+    async def get_content(self):
+        try: return self.response.content
+        except: return b""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

goog/models/auth_tokens.py

@@ -0,0 +1,243 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Wednesday, 30th Oct., 2024.
+
+    OBJECTIVE:
+
+        To provide a data model for describing the tokens to be used for Google's APIs.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For making data behaviour_models:
+from pydantic import BaseModel, Field, field_validator, model_validator, AwareDatetime
+from typing import Optional, Literal, Union, Dict, List, Any
+
+# Related to Google:
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# To work with date and time:
+import datetime
+import dateparser
+
+# To make API calls:
+import httpx
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class GoogleAuthTokens(BaseModel):
+
+    accessToken: str = Field(description = "the main 'bearer' token")
+    refreshToken: str = Field(description = "token to be used to refresh the access token")
+    expiresAt: AwareDatetime = Field(description = "the time (utc) at which the token will expire")
+    scopes: List[str] = Field(description = "the list of permissions", default = [])
+    email: str | None = Field(description = "the email id of the user", default = None)
+    displayName: str | None = Field(description = "the display name of the user", default = None)
+    displayPictureUrl: str | None = Field(description = "the url to the display picture of the user", default = None)
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "allow"
+
+    # ┓┏  ┓• ┓   •
+    # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
+    # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
+
+    @field_validator("expiresAt", mode = "before")
+    def parse_dates(cls, value):
+        if not isinstance(value, datetime.datetime):
+            parsed = date_time.parse_date_time(value, date_formats = ["%Y%m%d", "%Y-%m-%d"])
+            value = parsed if isinstance(parsed, datetime.datetime) else dateparser.parse(value)
+        if isinstance(value, datetime.datetime): value = date_time.to_timezone(value, date_time.TIMEZONE_UTC)
+        return value
+
+    # ┏┓           •
+    # ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
+    # ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
+    #       ┛
+
+    @property
+    def client_user_id(self):
+        return {"email": self.email}
+
+    @property
+    def expired(self):
+        return True if date_time.get_current_utc_date_time() >= self.expiresAt else False
+
+    @property
+    def ttl(self):
+        return (self.expiresAt - date_time.get_current_utc_date_time()).total_seconds()
+
+    # ┏┓           ┏┓
+    # ┃ ┓┏┏╋┏┓┏┳┓  ┣ ┓┏┏┓┏┏
+    # ┗┛┗┻┛┗┗┛┛┗┗  ┻ ┗┻┛┗┗┛
+
+    def refresh(
+            self,
+            client_id: str,
+            client_secret: str,
+            force_refresh: bool = False
+    ) -> bool:
+
+        """
+        Synchronously (blocking) refreshes the existing access tokens in place.
+        :param client_id: The id of the client app (OAuth JSON) for which these tokens were granted.
+        :param client_secret: The secret of the client app (OAuth JSON) for which these tokens were granted.
+        :param force_refresh: Whether you want to refresh the token even if it hasn't expired.
+        :return: True if refreshed, else False.
+        """
+
+        # Start by assuming failure:
+        success = False
+
+        try:
+
+            # Go ahead only if either the token has expired,
+            # or the user has asked to forcefully refresh the tokens:
+            if self.expired or force_refresh:
+
+                # Create the credentials:
+                credentials = Credentials.from_authorized_user_info(
+                    info = {
+                        "client_id": client_id,
+                        "client_secret": client_secret,
+                        "refresh_token": self.refreshToken,
+                        "expires_at": self.expiresAt
+                    }
+                )
+
+                # Request a refresh:
+                credentials.refresh(Request())
+
+                # Note down the new credentials:
+                if credentials.token != self.accessToken:
+                    success = True
+                    self.accessToken = credentials.token
+                    self.refreshToken = credentials.refresh_token
+                    self.expiresAt = date_time.as_if_timezone(credentials.expiry, timezone = date_time.TIMEZONE_UTC)
+
+        # In case something goes wrong:
+        except Exception as exception:
+            success = False
+
+        # Done here:
+        return success
+
+    async def arefresh(
+            self,
+            http_client: httpx.AsyncClient,
+            client_id: str,
+            client_secret: str,
+            force_refresh: bool = False
+    ) -> bool:
+
+        """
+        Asynchronously refreshes the existing access tokens in place.
+        :param http_client: The HTTP client to use to make the refresh request.
+        :param client_id: The id of the client (OAuth JSON) for which these tokens were granted.
+        :param client_secret: The secret of the client (OAuth JSON) for which these tokens were granted.
+        :param force_refresh: Whether you want to refresh the token even if it hasn't expired.
+        :return: True if refreshed, else False.
+        """
+
+        # Currently we don't really know how to refresh tokens through low-level API calls,
+        # so we will pass on the intent to the regular, synchronous function.
+        return self.refresh(
+            client_id = client_id,
+            client_secret = client_secret,
+            force_refresh = force_refresh
+        )
+
+    def has_scopes(self, scopes: List[str]) -> bool:
+
+        """
+        Checks if all the specified scopes were granted.
+        :param scopes: The list of scopes to check. These are the permissions you need.
+        :return: True if all specified scoped are present, else False.
+        """
+
+        # Start by assuming success:
+        has_scopes = True
+
+        # Now loop through the needed scopes and check:
+        for scope in scopes:
+            if scope not in self.scopes:
+                has_scopes = False
+                break
+
+        # Done here:
+        return has_scopes
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

image/scanner/scanner.py

@@ -0,0 +1,410 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 19th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a way to easily scan documents like printouts and visiting cards.
+
+    REFERENCES:
+
+        01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085
+        02. https://github.com/JaidedAI/EasyOCR
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+
+from diffusers.utils.import_utils import candidates
+
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To work with image processing:
+import cv2
+import imutils
+import numpy as np
+
+# To run OCR:
+import easyocr
+
+# To download images from the web:
+import requests
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.ai.object_detection.yolo import YoloDetect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DocumentScanner:
+
+    def __init__(
+            self,
+            layout_detection_yolo = None,
+            whitelisted_yolo_classes = None,
+            ocr_languages = None
+    ):
+
+        self.__yolo = YoloDetect(
+            model_file = layout_detection_yolo,
+            debug = False
+        )
+        self.__whitelisted_yolo_classes = whitelisted_yolo_classes or []
+        self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"])
+        pass
+
+    @staticmethod
+    def open_image(image):
+
+        """
+        Opens an image from various sources.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :return: The image as a CV2 object.
+        """
+
+        # When the image is provided as a string,
+        # it could be either as a URL or a path to a local file:
+        if isinstance(image, str):
+            if image.startswith("https://") or image.startswith("http://"):
+                image = io.BytesIO(requests.get(image).content)
+            else: image = cv2.imread(image)
+
+        # If the image is provided as a io.BytesIO buffer:
+        if isinstance(image, io.BytesIO):
+            image.seek(0)
+            image = np.asarray(bytearray(image.read()), dtype = np.uint8)
+            image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+
+        # Done here:
+        return image
+
+    @staticmethod
+    def polygon_area(points):
+
+        """
+        Computes the area occupied by a shape described by the array of points.
+        Example input: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
+        :return: The area of the shape.
+        """
+
+        n = len(points)
+        area = 0.0
+        for i in range(n):
+            j = (i + 1) % n  # next vertex
+            area += points[i][0] * points[j][1]
+            area -= points[j][0] * points[i][1]
+        return abs(area) / 2.0
+
+    @staticmethod
+    def show(image, title = "Preview", wait = True):
+
+        """
+        Just a quick wrapper to show the image in a window.
+        :param image: The image that you want to show.
+        :param title: The title of the window.
+        :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many
+            windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the
+            very last one.
+        :return: None.
+        """
+
+        cv2.imshow(title, image)
+        if wait:
+            cv2.waitKey()
+            cv2.destroyAllWindows()
+
+    @staticmethod
+    def enhance(image):
+
+        """
+        Applies a form of contrast boost to make the edges more easily visible.
+        :param image: The OpenCV image that needs to be enhanced.
+        :return: The enhanced CV2 image.
+        """
+
+        return cv2.detailEnhance(
+            src = image,
+            sigma_s = 20,
+            sigma_r = 0.15
+        )
+
+    @staticmethod
+    def get_edges(image):
+
+        """
+        Gets the edges in an image.
+        :param image: A CV2 image.
+        :return: The image with the edges detected.
+        """
+
+        # Denoise the image:
+        processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        processed_image = cv2.GaussianBlur(
+            processed_image,
+            ksize = (5, 5),
+            sigmaX = 0
+        )
+
+        # Detect edges in the image:
+        processed_image = cv2.Canny(
+            processed_image,
+            threshold1 = 50,
+            threshold2 = 200
+        )
+
+        # Close small gaps in the edges:
+        kernel = np.ones((5, 5), np.uint8)
+        processed_image = cv2.dilate(processed_image, kernel, iterations = 1)
+        # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel)
+        # processed_image = cv2.erode(processed_image, kernel, iterations = 1)
+
+        # Done here:
+        return processed_image
+
+    def scan(self, image, min_area = 0.125):
+
+        """
+        Looks for a rectangle in an image and flattens it out. No colour filters are applied here.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_area: The minimum area occupied by the document in the image.
+        :return: The flattened image if any, else None.
+        """
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        if image is None: return None
+        height, width = image.shape[:2]
+        image_area = height * width
+
+        # Enhance the image to have better visibility of edges:
+        # processed_image = self.enhance(image.copy())
+
+        # Detect edges in the image:
+        processed_image = self.get_edges(image.copy())
+
+        # Find the contours in the edges,
+        # and sort them in ascending order:
+        contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+        contours = imutils.grab_contours(contours)
+        contours = sorted(contours, key = cv2.contourArea, reverse = True)
+
+        # We try to get clean and passable contours.
+        # For our purposes, 'clean' is when it can be approximated to exactly 4 sides,
+        # and 'passable' is when the approximation comes close to it:
+        clean_rects = []
+        passable_rects = []
+        for contour in contours:
+            perimeter = cv2.arcLength(contour, closed = True)
+            approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True)
+            if len(approximation) == 4: clean_rects.append(approximation)
+            elif len(approximation) <= 32:
+                candidate_contour = cv2.minAreaRect(approximation)
+                candidate_contour = cv2.boxPoints(candidate_contour)
+                candidate_contour = np.intp(candidate_contour)
+                passable_rects.append(candidate_contour)
+        clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True)
+        passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True)
+
+        # Now we decide between the best candidate:
+        document_outline = None
+        if len(clean_rects) > 0:
+            candidate_rect = np.array([p[0] for p in clean_rects[0]])
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+        if document_outline is None and len(passable_rects) > 0:
+            candidate_rect = cv2.minAreaRect(passable_rects[0])
+            candidate_rect = cv2.boxPoints(candidate_rect)
+            candidate_rect = np.intp(candidate_rect)
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+
+        # If there is no rectangular contour,
+        # we exit with failure:
+        if document_outline is None: return None
+
+        # Since we did get the best candidate for the document,
+        # we figure out which point n the contour is which corner of the document:
+        document_rectangle = np.zeros((4, 2), dtype = "float32")
+        sum_points = document_outline.sum(axis = 1)
+        document_rectangle[0] = document_outline[np.argmin(sum_points)]
+        document_rectangle[2] = document_outline[np.argmax(sum_points)]
+        diff_points = np.diff(document_outline, axis = 1)
+        document_rectangle[1] = document_outline[np.argmin(diff_points)]
+        document_rectangle[3] = document_outline[np.argmax(diff_points)]
+        (top_left, top_right, bottom_right, bottom_left) = document_rectangle
+
+        # let's compute the dimensions of the document in the image:
+        left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
+        right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
+        top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
+        bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
+        max_height = max(int(left_height), int(right_height))
+        max_width = max(int(top_width), int(bottom_width))
+
+        # We compute the destination of the transform:
+        destination = np.array(
+            object = [
+                [0, 0],
+                [max_width - 1, 0],
+                [max_width - 1, max_height - 1],
+                [0, max_height - 1]
+            ],
+            dtype = "float32"
+        )
+
+        # We apply the transform that flattens out the document:
+        matrix = cv2.getPerspectiveTransform(document_rectangle, destination)
+        flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height))
+
+        # Done here:
+        return flattened_image
+
+    def extract_text(
+            self,
+            image,
+            min_confidence = 0.5,
+            margin = 0.05
+    ):
+
+        """
+        To run OCR on the input image.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_confidence: The minimum amount of confidence for detected text to be considered.
+        :param margin: The extra pixels to include when cropping into a section for OCR.
+        :return: The extracted text.
+        """
+
+        # Make a variable that will hold the final result:
+        extracted_text = {}
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        height, width = image.shape[:2]
+
+        # Run OCR on the entire canvas:
+        ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence]
+        extracted_text["fullDoc"] = " ".join(ocr_result)
+
+        # Get the doc layout in the image:
+        document_sections = self.__yolo.predict(image)
+        document_sections = document_sections["boxes"]
+
+        # For each section, we run the OCR process individually:
+        section_results = []
+        for section in document_sections:
+            if section["class"] in self.__whitelisted_yolo_classes:
+                section_width = section["x2"] - section["x1"]
+                section_height = section["y2"] - section["y1"]
+                x1 = max(int(section["x1"] - (margin * section_width)), 0)
+                y1 = max(int(section["y1"] - (margin * section_height)), 0)
+                x2 = min(int(section["x2"] + (margin * section_width)), width)
+                y2 = min(int(section["y2"] + (margin * section_height)), height)
+                sub_image = image[y1:y2, x1:x2]
+                ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence]
+                section_results.append({
+                    "text": " ".join(ocr_result),
+                    "class": section["class"],
+                    "className": section["className"],
+                    "x1": x1,
+                    "y1": y1,
+                    "x2": x2,
+                    "y2": y2
+                })
+        extracted_text["bySection"] = section_results
+
+        # Done here:
+        return extracted_text
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import time
+
+    my_scanner = DocumentScanner(
+        layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
+        whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
+        # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
+        ocr_languages = ["en"]
+    )
+
+    image = my_scanner.open_image(r"/home/developer/Downloads/talkaholics_card.jpg")
+    document_image = my_scanner.scan(image)
+    if document_image is not None:
+        cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image)
+        start_time = time.time()
+        document_text = my_scanner.extract_text(
+            document_image,
+            min_confidence = 0.5
+        )
+        print("OCR RESULT:")
+        print(document_text)
+
+        print(f"FINISHED IN {time.time() - start_time} SECONDS!")
+
+    else: print("No image")

image/scanner/scanner_v2.py

@@ -0,0 +1,379 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 24th Sept., 2024
+
+    OBJECTIVE:
+
+        To provide a way to easily scan documents like printouts and visiting cards.
+
+    REFERENCES:
+
+        01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085
+        02. https://github.com/JaidedAI/
+        03. https://github.com/criistian14/flutter_document_scanner/blob/master/flutter_document_scanner_android/android/src/main/kotlin/com/christian/flutterDocumentScanner/OpenCVPlugin.kt
+        04. https://www.geeksforgeeks.org/python-bilateral-filtering/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# To work with image processing:
+import cv2
+import imutils
+import numpy as np
+
+# To run OCR:
+import easyocr
+
+# To download images from the web:
+import requests
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.ai.object_detection.yolo import YoloDetect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class DocumentScanner:
+
+    def __init__(
+            self,
+            layout_detection_yolo = None,
+            whitelisted_yolo_classes = None,
+            ocr_languages = None
+    ):
+
+        self.__yolo = YoloDetect(
+            model_file = layout_detection_yolo,
+            debug = False
+        )
+        self.__whitelisted_yolo_classes = whitelisted_yolo_classes or []
+        self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"])
+        pass
+
+    @staticmethod
+    def open_image(image):
+
+        """
+        Opens an image from various sources.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :return: The image as a CV2 object.
+        """
+
+        # When the image is provided as a string,
+        # it could be either as a URL or a path to a local file:
+        if isinstance(image, str):
+            if image.startswith("https://") or image.startswith("http://"):
+                image = io.BytesIO(requests.get(image).content)
+            else: image = cv2.imread(image)
+
+        # If the image is provided as a io.BytesIO buffer:
+        if isinstance(image, io.BytesIO):
+            image.seek(0)
+            image = np.asarray(bytearray(image.read()), dtype = np.uint8)
+            image = cv2.imdecode(image, cv2.IMREAD_COLOR)
+
+        # Done here:
+        return image
+
+    @staticmethod
+    def show(image, title = "Preview", wait = True):
+
+        """
+        Just a quick wrapper to show the image in a window.
+        :param image: The image that you want to show.
+        :param title: The title of the window.
+        :param wait: Set this to True when you want to show the window(s). This is useful when you want to show many
+            windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the
+            very last one.
+        :return: None.
+        """
+
+        cv2.imshow(title, image)
+        if wait:
+            cv2.waitKey()
+            cv2.destroyAllWindows()
+
+    @staticmethod
+    def get_edges(image):
+
+        """
+        Apply all the preprocessing filters on the image before sending it off for contour-finding.
+        :param image: The image to pre-process.
+        :return: The pre-processed image.
+        """
+
+        # Denoise the image:
+        processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+        processed_image = cv2.bilateralFilter(
+            processed_image,
+            d = 9,
+            sigmaColor = 75,
+            sigmaSpace = 75
+        )
+
+        # Detect edges in the image:
+        processed_image = cv2.Canny(
+            processed_image,
+            threshold1 = 75,
+            threshold2 = 200
+        )
+
+        # Close small gaps in the edges:
+        kernel = np.ones((5, 5), np.uint8)
+        processed_image = cv2.dilate(processed_image, kernel, iterations = 1)
+        # processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel)
+        # processed_image = cv2.erode(processed_image, kernel, iterations = 1)
+
+        # Done here:
+        return processed_image
+
+    def scan(self, image, min_area = 0.125):
+
+        """
+        Looks for a rectangle in an image and flattens it out. No colour filters are applied here.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_area: The minimum area occupied by the document in the image.
+        :return: The flattened image if any, else None.
+        """
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        if image is None: return None
+        height, width = image.shape[:2]
+        image_area = height * width
+
+        # image = self.enhance(image)
+        processed_image = self.get_edges(image)
+        # self.show(processed_image)
+
+        # Find the contours in the edges,
+        # and sort them in ascending order:
+        contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+        contours = imutils.grab_contours(contours)
+        contours = sorted(contours, key = cv2.contourArea, reverse = True)
+
+        # We try to get clean and passable contours.
+        # For our purposes, 'clean' is when it can be approximated to exactly 4 sides,
+        # and 'passable' is when the approximation comes close to it:
+        clean_rects = []
+        passable_rects = []
+        for contour in contours:
+            perimeter = cv2.arcLength(contour, closed = True)
+            approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True)
+            if len(approximation) == 4: clean_rects.append(approximation)
+            elif len(approximation) <= 32:
+                candidate_contour = cv2.minAreaRect(approximation)
+                candidate_contour = cv2.boxPoints(candidate_contour)
+                candidate_contour = np.intp(candidate_contour)
+                passable_rects.append(candidate_contour)
+        clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True)
+        passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True)
+
+        # Now we decide between the best candidate:
+        document_outline = None
+        if len(clean_rects) > 0:
+            candidate_rect = np.array([p[0] for p in clean_rects[0]])
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+        if document_outline is None and len(passable_rects) > 0:
+            candidate_rect = cv2.minAreaRect(passable_rects[0])
+            candidate_rect = cv2.boxPoints(candidate_rect)
+            candidate_rect = np.intp(candidate_rect)
+            rectangle_area = cv2.contourArea(candidate_rect)
+            if rectangle_area / image_area >= min_area: document_outline = candidate_rect
+
+        # If there is no rectangular contour,
+        # we exit with failure:
+        if document_outline is None: return None
+
+        # Since we did get the best candidate for the document,
+        # we figure out which point n the contour is which corner of the document:
+        document_rectangle = np.zeros((4, 2), dtype = "float32")
+        sum_points = document_outline.sum(axis = 1)
+        document_rectangle[0] = document_outline[np.argmin(sum_points)]
+        document_rectangle[2] = document_outline[np.argmax(sum_points)]
+        diff_points = np.diff(document_outline, axis = 1)
+        document_rectangle[1] = document_outline[np.argmin(diff_points)]
+        document_rectangle[3] = document_outline[np.argmax(diff_points)]
+        (top_left, top_right, bottom_right, bottom_left) = document_rectangle
+
+        # let's compute the dimensions of the document in the image:
+        left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
+        right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
+        top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
+        bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
+        max_height = max(int(left_height), int(right_height))
+        max_width = max(int(top_width), int(bottom_width))
+
+        # We compute the destination of the transform:
+        destination = np.array(
+            object = [
+                [0, 0],
+                [max_width - 1, 0],
+                [max_width - 1, max_height - 1],
+                [0, max_height - 1]
+            ],
+            dtype = "float32"
+        )
+
+        # We apply the transform that flattens out the document:
+        matrix = cv2.getPerspectiveTransform(document_rectangle, destination)
+        flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height))
+
+        # Done here:
+        return flattened_image
+
+    def extract_text(
+            self,
+            image,
+            min_confidence = 0.5,
+            margin = 0.05
+    ):
+
+        """
+        To run OCR on the input image.
+        :param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
+        :param min_confidence: The minimum amount of confidence for detected text to be considered.
+        :param margin: The extra pixels to include when cropping into a section for OCR.
+        :return: The extracted text.
+        """
+
+        # Make a variable that will hold the final result:
+        extracted_text = {}
+
+        # Open the image and read the data:
+        image = self.open_image(image)
+        height, width = image.shape[:2]
+
+        # Run OCR on the entire canvas:
+        ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence]
+        extracted_text["fullDoc"] = " ".join(ocr_result)
+
+        # Get the doc layout in the image:
+        document_sections = self.__yolo.predict(image)
+        document_sections = document_sections["boxes"]
+
+        # For each section, we run the OCR process individually:
+        section_results = []
+        for section in document_sections:
+            if section["class"] in self.__whitelisted_yolo_classes:
+                section_width = section["x2"] - section["x1"]
+                section_height = section["y2"] - section["y1"]
+                x1 = max(int(section["x1"] - (margin * section_width)), 0)
+                y1 = max(int(section["y1"] - (margin * section_height)), 0)
+                x2 = min(int(section["x2"] + (margin * section_width)), width)
+                y2 = min(int(section["y2"] + (margin * section_height)), height)
+                sub_image = image[y1:y2, x1:x2]
+                ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence]
+                section_results.append({
+                    "text": " ".join(ocr_result),
+                    "class": section["class"],
+                    "className": section["className"],
+                    "x1": x1,
+                    "y1": y1,
+                    "x2": x2,
+                    "y2": y2
+                })
+        extracted_text["bySection"] = section_results
+
+        # Done here:
+        return extracted_text
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import time
+
+    my_scanner = DocumentScanner(
+        layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
+        whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
+        # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
+        ocr_languages = ["en"]
+    )
+
+    image = my_scanner.open_image(r"/home/developer/Downloads/sushmita_card.jpg")
+    # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card.jpg")
+    # image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card_2.jpg")
+    # image = my_scanner.open_image(r"/home/developer/Downloads/card_square.jpg")
+    document_image = my_scanner.scan(image)
+    if document_image is not None:
+        cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image)
+        start_time = time.time()
+        document_text = my_scanner.extract_text(
+            document_image,
+            min_confidence = 0.5
+        )
+        print("OCR RESULT:")
+        print(document_text)
+
+        print(f"FINISHED IN {time.time() - start_time} SECONDS!")
+
+    else: print("No image")

logging/context.py

@@ -0,0 +1,468 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 2nd Aug., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to log all system activities by way of managing the context of what is going on.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+    NOTES:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To manage context:
+import contextvars
+from contextlib import contextmanager, asynccontextmanager
+
+# To make decorators:
+from functools import wraps
+
+# For system-level activities:
+import os
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+from utils_v2.security import sanitizers
+from utils_v2.api.codes import StatusCodes, HttpCodes
+from utils_v2.api.log import APILogModel
+from utils_v2.api.response import ResponseModel
+
+# The needed data models:
+from utils_v2.logging.model import GeneralLogModel
+
+# To work with date and time:
+import time
+
+# For random strings:
+import random
+
+# For debugging:
+import traceback
+import string
+
+# To work with datatypes:
+from types import NoneType
+import pandas as pd
+
+# To work with Pydantic objects:
+from pydantic import BaseModel
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Chars to choose from for random strings:
+ALPHANUMERIC_CHARS = string.ascii_letters + string.digits
+
+# To capture system information:
+PROCESS_ID = os.getppid()
+PARENT_PROCESS_ID = os.getppid()
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  EXCEPTIONS                                                   ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def describe_exception(exc):
+
+    """
+    Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire
+    traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it
+    to the dev team over some service like WhatsApp/Telegram.
+    :param exc: The exception that occurred.
+    :return: The dict that explains the exception.
+    """
+
+    exc_desc = {
+        "type": type(exc).__name__,
+        "msg": str(exc),
+        "tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)]
+    }
+
+    return exc_desc
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   Classes                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncMongoLogger:
+
+    def __init__(self, db_conn,  collection = "logs"):
+
+        """
+        This class uses an instance of 'AsyncMongo' and makes it usable as a logger.
+        :param db_conn: The instance of 'AsyncMongo' to use.
+        :param collection: The collection to write the log into.
+        """
+
+        self.__db_conn = db_conn
+        self.__collection = collection
+
+    async def log(self, log_json):
+
+        """
+        Log something to the database using the connection provided when the object was made.
+        :param log_json: The dict to log.
+        :return: True if logged successfully, else False.
+        """
+
+        asyncio.create_task(self.__db_conn.insert_one(
+            collection = self.__collection,
+            document = log_json,
+            raise_exception = False
+        ))
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class AsyncLoggerContext:
+
+    # Create the context-aware variable(s):
+    logger = contextvars.ContextVar("logger", default = None)
+    log_chain = contextvars.ContextVar("log_chain", default = None)
+
+    @classmethod
+    @asynccontextmanager
+    async def logging_context(cls, logger, log_chain = None):
+
+        """
+        This function makes the context manager that makes the value of the log chain available to everything that is
+        called within the scope of the context.
+        :param logger: The object which is to be used to write the log. It should have a 'log' method which should take
+            in a dict as its input.
+        :param log_chain: The value of the log chain to be made available within the scope.
+        :return: None.
+        """
+
+        # Set the context:
+        token_logger = cls.logger.set(logger)
+        token_log_chain = cls.log_chain.set(log_chain)
+
+        # Make the objects available within the context:
+        try: yield
+
+        # Release the objects when the context is over:
+        finally:
+            cls.logger.reset(token_logger)
+            cls.log_chain.reset(token_log_chain)
+
+    @staticmethod
+    def generate_log_id(count = 8):
+        return "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(min(8, count)))
+
+    @classmethod
+    def get_logger(cls):
+        return cls.logger.get()
+
+    @classmethod
+    def get_log_chain(cls):
+        return cls.log_chain.get()
+
+    @staticmethod
+    def summarize(
+            value,
+            str_limit = 100,
+            expand: bool | int = False,
+            sensitive_keys: list[str] = None
+    ):
+
+        """
+        To summarize an input value to capture the essence without hoarding to much data.
+        :param value: Anything that you want to summarize.
+        :param str_limit: The max. no. of chars of a string to retain.
+        :param expand: Set to True for full expansion, False for no expansion, and an integer for a specific level of
+            expansion. Applicable on iterables and dicts. The smaller this number, the more concise the summary will be,
+            and vice versa.
+        :param sensitive_keys: The list of keys (of a dict) to obscure when summarizing.
+        :return: The summarized version of the input.
+        """
+
+        # If the input is a Pydantic class:
+        if isinstance(value, BaseModel): value = value.model_dump()
+
+        # Check the sensitive keys:
+        if sensitive_keys is None: sensitive_keys = []
+
+        # Handle datatypes that you don't want to modify:
+        if isinstance(value, (int, float, bool, NoneType)): pass
+
+        # When the value is a list or similar iterable:
+        elif isinstance(value, (list, tuple, set)):
+            if expand:
+                if not isinstance(expand, bool): expand -= 1
+                value = [AsyncLoggerContext.summarize(
+                    v,
+                    expand = expand,
+                    sensitive_keys = sensitive_keys
+                ) for v in value]
+            else: value = f"array of {len(value)} item(s)"
+
+        # If the value is a dict:
+        elif isinstance(value, dict):
+            if expand:
+                if not isinstance(expand, bool): expand -= 1
+                value = {
+                    k: AsyncLoggerContext.summarize(
+                        v,
+                        expand = expand,
+                        sensitive_keys = sensitive_keys
+                    ) if k not in sensitive_keys else "********"
+                    for k, v in value.items()
+                }
+            else: value = f"object of {len(value.keys())} field(s) [{', '.join(value.keys())}]"
+
+        # When a dataframe is passed:
+        elif isinstance(value, pd.DataFrame):
+            cols = value.columns.to_list()
+            value = f"table with {len(cols)} col(s) [{', '.join(cols)}] and {len(value)} row(s)"
+            str_limit = 999
+
+        # If the input is some form of non-standard object:
+        else: value = str(value)
+
+        # Handle strings:
+        if isinstance(value, str):
+            if len(value) > str_limit: value = value[:str_limit] + "..."
+
+        # Done here:
+        return value
+
+    @classmethod
+    def log_it(
+            cls,
+            api_version: str = None,
+            project: str = None,
+            log_type: str = None,
+            operation: str = None,
+            log_input: bool | int = True,
+            log_output: bool | int = True,
+            sensitive_keys: list = None
+    ):
+
+        """
+        A decorator factor that can be used to log the results of functions automatically.
+        :param api_version: A string that indicates the version code of the function being decorated.
+        :param project: A hint about which project is being worked on.
+        :param log_type: A hint about which module is being worked on.
+        :param operation: A hint about which action in a particular module is being worked on.
+        :param log_input: Set to True to capture everything that went into the function, False to capture the least
+            info, and set it to an integer to capture a certain depth of the input (applicable on iterables and dicts.
+        :param log_output: The same as 'log_input', but applicable to the response from the function.
+        :param sensitive_keys: Keys of a dict whose values must be obscured even if that depth is being captured.
+        :return: A decorator with the configuration.
+        """
+
+        def decorator(func):
+
+            @wraps(func)
+            async def wrapper(*args, **kwargs):
+
+                # Make variables and extract available info.:
+                exception = None
+                response = None
+                request_ts = date_time.get_current_utc_date_time()
+                start_ts = time.perf_counter()
+                cpu_start_ts = time.process_time()
+
+                # Execute the function that is being wrapped:
+                try: response = await func(*args, **kwargs)
+                except Exception as exc: exception = exc
+
+                # Do the next steps only if within the logging context:
+                if cls.get_logger() is not None:
+
+                    # Create the log:
+                    if not args: args = None
+                    if not kwargs: kwargs = None
+                    func_log = GeneralLogModel(
+                        pid = PROCESS_ID,
+                        ppid = PARENT_PROCESS_ID,
+                        project = project,
+                        log = log_type or func.__name__,
+                        operation = operation or func.__name__,
+                        apiVer = api_version,
+                        logId = cls.generate_log_id(),
+                        logChain = cls.get_log_chain(),
+                        ts = request_ts,
+                        tat = time.perf_counter() - start_ts,
+                        cpuTime = time.process_time() - cpu_start_ts,
+                        func = func.__name__,
+                        args = cls.summarize(args, expand = log_input, sensitive_keys = sensitive_keys),
+                        kwargs = cls.summarize(kwargs, expand = log_input, sensitive_keys = sensitive_keys),
+                        exception = None if exception is None else describe_exception(exception),
+                        response = cls.summarize(response, expand = log_output, sensitive_keys = sensitive_keys),
+                    ).model_dump()
+
+                    # Write the log:
+                    await cls.get_logger().log(func_log)
+
+                # Done here:
+                if exception is not None: raise exception
+                return response
+
+            return wrapper
+
+        return decorator
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    from utils_v2.database.async_mongo_v2 import AsyncMongo
+
+    @AsyncLoggerContext.log_it(
+        api_version = "0.0.1",
+        project = "testProj",
+        log_type = "work",
+        operation = "someWork",
+        log_input = True,
+        log_output = True,
+        sensitive_keys = None
+    )
+    async def some_work(*args, **kwargs):
+        print("SOME WORK:", AsyncLoggerContext.get_log_chain())
+        await asyncio.sleep(max(2.0 * random.random(), 1.0))
+        total = sum(args)
+        return total
+
+    @AsyncLoggerContext.log_it(
+        api_version = "0.0.1",
+        project = "testProj",
+        log_type = "work",
+        operation = "moreWork",
+        log_input = True,
+        log_output = True,
+        sensitive_keys = ["password", "sessionToken"]
+    )
+    async def more_work(*args, **kwargs):
+        print("MORE WORK:", AsyncLoggerContext.get_log_chain())
+        await asyncio.sleep(max(2.0 * random.random(), 1.0))
+        return {"success": True, "sessionToken": "1234567890"}
+
+    @AsyncLoggerContext.log_it(
+        api_version = "0.0.1",
+        project = "testProj",
+        log_type = "work",
+        operation = "moreWork",
+        log_input = True,
+        log_output = True,
+        sensitive_keys = None
+    )
+    async def last_work(*args, **kwargs):
+        print("LAST WORK:", AsyncLoggerContext.get_log_chain())
+        await asyncio.sleep(max(2.0 * random.random(), 1.0))
+
+    async def main(chain = None):
+
+        # Connect to MongoDB:
+        mongo = AsyncMongo(
+            connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
+            database_name = "converse",
+            max_connections = 10,
+            debug = True
+        )
+
+        # Convert the connection to a logger instance that can be injected
+        # into the context as a dependency:
+        mongo_logger = AsyncMongoLogger(
+            db_conn = mongo,
+            collection = "logs"
+        )
+
+        # Initialize the context:
+        async with AsyncLoggerContext.logging_context(
+                logger = mongo_logger,
+                log_chain = chain
+        ):
+
+            # Run some functions within the context:
+            await some_work(1, 2, 3, 4, 5)
+            await more_work(username = "john.doe@domain.com", password = "mySecretPass")
+
+        # Run something outside the context:
+        await last_work()
+
+    # async def multi_main():
+    #     tasks = [
+    #         main(chain = "kPRwXdItb1"),
+    #         main(chain = "456")
+    #     ]
+    #     await asyncio.gather(*tasks)
+
+    asyncio.run(main(chain = "00wGHRFYPY123"))

logging/model.py

@@ -0,0 +1,131 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 11th Oct., 2024
+
+    OBJECTIVE:
+
+        To have a structure for the logs maintained for regular function calls.
+        This is different from the logs maintained for API calls.
+
+    REFERENCES:
+
+        01. https://chatgpt.com/share/6708b6c9-6ba4-800f-9ea9-00ec35067512
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# System-level activities:
+import distro
+import socket
+import platform
+
+# For data-modelling:
+from pydantic import BaseModel, Field
+from typing import Any, Optional, List, Literal
+
+# To work with date and time:
+import datetime
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Info for logging that will stay constant during runtime:
+SERVER_HOSTNAME = str(socket.gethostname())
+PLATFORM_INFO = platform.uname()
+HOST_OS = str(distro.name(True))
+HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})"
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  CLASSES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class GeneralLogModel(BaseModel):
+
+    # To identify the machine the code is running on.
+    # DO NOT MODIFY THESE:
+    hostname: str = SERVER_HOSTNAME
+    os: str = HOST_OS
+    cpu: str = HOST_CPU
+    # Can modify these:
+    pid: Optional[Any] = None
+    ppid: Optional[Any] = None
+
+    # To identify the project and actions:
+    project: Optional[str] = None
+    log: str
+    operation: Optional[str] = None
+    apiVer: Optional[str] = None
+    logId: Optional[str] = None
+    logChain: Optional[str] = None
+
+    # Timing metrics:
+    ts: datetime.datetime
+    tat: float
+    cpuTime: float
+
+    # To understand the inputs:
+    func: str
+    args: Optional[Any] = None
+    kwargs: Optional[Any] = None
+
+    # To understand the outputs:
+    exception: Optional[Any] = None
+    response: Optional[Any] = None
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

mail/async_smpt.py

@@ -0,0 +1,443 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 16th Jul, 2024
+
+    OBJECTIVE:
+
+        To be able to send out mails from code.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For working with mails:
+import aiosmtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.image import MIMEImage
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email import encoders
+
+# For random strings:
+import string
+import random
+
+# For system-level activities:
+import os
+
+# For working with files in RAM:
+import io
+
+# For debugging:
+from icecream import IceCreamDebugger
+
+# To work with datatypes:
+from typing import List
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class SMTPMessage:
+
+    def __init__(
+            self,
+            to_email: str,
+            subject: str,
+            cc_emails: List[str] = None,
+            bcc_emails: List[str] = None
+    ):
+
+        """
+        Create an instance of the message that you would like to send.
+        :param to_email: The EMail ID of th recipient.
+        :param subject: The subject of the mail.
+        :param cc_emails: A list of recipients to add to the CC section.
+        :param bcc_emails: A list of recipients to add to the BCC section.
+        """
+
+        self.message = MIMEMultipart()
+        self.message["To"] = to_email
+        self.message["Subject"] = subject
+        if cc_emails: self.message["CC"] = ",".join(cc_emails)
+        if bcc_emails: self.message["BCC"] = ",".join(bcc_emails)
+
+    def add_text(self, text):
+
+        """
+        Add plain-text to the mail body.
+        :param text: The text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(text, "plain"))
+
+    def add_html(self, html_text):
+
+        """
+        Add HTML text to the mail body.
+        :param html_text: The HTML text to add to the mail body.
+        :return: None.
+        """
+
+        self.message.attach(MIMEText(html_text, "html"))
+
+    def add_inline_image(self, image_file, content_id = None):
+
+        """
+        Add an inline image to the body of the mail.
+        NOTE: This is NOT the same as sending an image as an attachment.
+        :param image_file: The image data to attach to the mail body.
+        :param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not
+            specified, I will generate a random string. You may write a custom value here if you know what you are
+            doing. For most use cases, please ignore this field.
+        :return: None.
+        """
+
+        # Read the image as bytes:
+        image_bytes = None
+        if type(image_file) is str:
+            with open(image_file, "rb") as opened_image_file:
+                image_bytes = opened_image_file.read()
+        if type(image_file) is io.BytesIO:
+            image_file.seek(0)
+            image_bytes = image_file.getvalue()
+
+        # Declare the part to be attached to the multipart message:
+        if image_bytes is not None:
+
+            # Create the HTML block if the image pointer is blank:
+            if content_id is None:
+                content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+                self.add_html(f"""
+                <html>
+                    <body>
+                        <p><img src="cid:{content_id}"></p>
+                    </body>
+                </html>
+                """)
+
+            # Then add the image:
+            image_part = MIMEImage(image_bytes)
+            image_part.add_header("Content-ID", f"<{content_id}>")
+            self.message.attach(image_part)
+
+    def add_attachment(self, attachment_file, file_name = None):
+
+        # Declare the part to be attached to the multipart message:
+        part = MIMEBase("application", "octet-stream")
+
+        # If the attachment is a file stored in the local disk:
+        if isinstance(attachment_file, str):
+            file_name = file_name or os.path.split(attachment_file)[-1]
+            with open(attachment_file, "rb") as attachment:
+                part.set_payload(attachment.read())
+
+        # If the file is held in RAM:
+        elif isinstance(attachment_file, io.BytesIO):
+            attachment_file.seek(0)
+            part.set_payload(attachment_file.read())
+
+        # Encode and attach the file:
+        encoders.encode_base64(part)
+        part.add_header(
+            "Content-Disposition",
+            f"attachment; filename= {file_name}",
+        )
+        self.message.attach(part)
+
+    def get_message(self):
+        return self.message
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class AsyncSMTPClient:
+
+    # Constants:
+    SMTP_TLS_PORT = 587
+    SMTP_SSL_PORT = 465
+
+    # variables:
+    __smtp = None
+
+    def __init__(
+            self,
+            email,
+            password,
+            server,
+            port = 587,
+            rate_limiters = None,
+            wait_for_turn = True,
+            debug = True,
+            debug_prefix = "Mail (C) | "
+    ):
+
+        """
+        Set up the mail client.
+        :param email: The Email ID to use when sending out mails.
+        :param password: The password of the EMail ID that is being used.
+        :param server: The EMail server.
+        :param port: The port number to connect to the host.
+        :param rate_limiters: The rate limiters to use. Must have "get_turn" and "has_turn" methods. "get_turn" method
+            must wait for the turn, and "has_turn" method must only check if a turn is available.
+        :param wait_for_turn: To wait for turn if the rate limit has been exceeded, or to return with failure.
+        :param debug: Whether, or not, you want to print debugging messages.
+        :param debug_prefix: The prefix to identify the debugging messages.
+        """
+
+        # Initialize the debugger:
+        self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self.__printer.disable()
+
+        # Note down the credentials and other details:
+        self.__email = email
+        self.__password = password
+        self.__server = server
+        self.__port = port
+        self.__rate_limiters = rate_limiters if type(rate_limiters) is list else ([rate_limiters] if rate_limiters is not None else [])
+        self.__wait_for_turn = wait_for_turn
+
+    def enable_debug(self):
+        self.__printer.enable()
+
+    def disable_debug(self):
+        self.__printer.disable()
+
+    async def login(self):
+
+        """
+        To connect to the mail server and authenticate the user.
+        :return: True if authenticated, else False.
+        """
+
+        # Initialize the SMTP connection,
+        # and return with success if all goes well:
+        try:
+            self.__smtp = aiosmtplib.SMTP(
+                hostname = self.__server,
+                port = self.__port,
+                use_tls = False,
+                start_tls = False
+            )
+            await self.__smtp.connect()
+            await self.__smtp.starttls()
+            await self.__smtp.login(self.__email, self.__password)
+            return True
+
+        # Return with failure if something goes wrong:
+        except Exception as exception:
+            self.__printer(exception)
+            try: await self.__smtp.quit()
+            except Exception as exception: self.__printer(exception)
+            self.__smtp = None
+            return False
+
+    async def ensure_connection(self):
+
+        """
+        Can be run before the sending operation to confirm that we are yet connected to the server.
+        If not connected, this code will reattempt to connect and log-in.
+        :return: True if connected, else False.
+        """
+
+        # If the login had failed initially, the object will be set to null.
+        # In such a case, we make an attempt to login:
+        if self.__smtp is None:
+            return await self.login()
+
+        # If the login was successful, we check if the connection is active.
+        # If not, we try to re-login:
+        if self.__smtp.is_connected:
+            return True
+        else:
+            try:
+                await self.__smtp.connect()
+                await self.__smtp.starttls()
+                await self.__smtp.login(self.__email, self.__password)
+                return True
+            except Exception as exception:
+                self.__printer(exception)
+                try: await self.__smtp.quit()
+                except Exception as exception: self.__printer(exception)
+                self.__smtp = None
+                return False
+
+    async def logout(self):
+
+        """
+        Closes the connection to the SMTP client.
+        :return: True by default.
+        """
+
+        if self.__smtp is not None:
+            try: await self.__smtp.quit()
+            except Exception as exception: self.__printer(exception)
+        self.__smtp = None
+        return True
+
+    async def send(self, mail: SMTPMessage):
+
+        """
+        Send out the mail.
+        :param mail: The instance of 'MailMessage' with all the content populated.
+        :return: A dict with 'success' and 'message'.
+        """
+
+        # Return with failure if we aren't connected,
+        # and our attempt to (re)connect fails:
+        if not await self.ensure_connection():
+            return {
+                "success": False,
+                "message": "login failed"
+            }
+
+        # Comply with the rate-limit:
+        for rate_limiter in self.__rate_limiters:
+            if not self.__wait_for_turn:
+                if not await rate_limiter.has_turn(): return False
+            got_turn = await rate_limiter.get_turn()
+            if not got_turn:
+                return {
+                    "success": False,
+                    "message": "rate-limit wait timeout"
+                }
+
+        # Try to send the message:
+        try:
+            mail.message["From"] = self.__email
+            response = await self.__smtp.send_message(mail.message)
+            return {
+                "success": True,
+                "message": f"mail accepted - {response[-1]}"
+            }
+
+        # If something goes wrong:
+        except Exception as exception:
+            self.__printer(exception)
+            return {
+                "success": False,
+                "message": str(exception)
+            }
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    import asyncio
+    from utils_v2.string import json
+
+    async def test():
+
+        mail_client = AsyncSMTPClient(
+            email = "sender@gmail.com",
+            password = "zcaf nmqy ncfz fave",
+            server = "smtp.gmail.com",
+            rate_limiters = None
+        )
+
+        my_mail = SMTPMessage(
+            to_email = "orangebhopli@gmail.coms",
+            subject = "Bhopli is the best!",
+            cc_emails = None,
+            bcc_emails = None
+        )
+        my_mail.add_html(
+            """
+            <!DOCTYPE html>
+            <html>
+            <head>
+                <meta charset="UTF-8">
+                <title>Sample HTML String</title>
+                <style>
+                    .heading {
+                        color: #ff9025;
+                    }
+                    .sub-heading {
+                        color: #000000;
+                    }
+                </style>
+            </head>
+            <body>
+                <h1 class="heading">Hello, Bhopli!</h1>
+                <h2 class="sub-heading">Bhopli is the best, most well-behaved cat in the known universe.</h2>
+            </body>
+            </html>
+            """
+        )
+        my_mail.add_text("This is how you should pet her 👇")
+        my_mail.add_inline_image(r"/path/to/image/cat_petting.png")
+        my_mail.add_attachment(r"/path/to/file/sample_label.pdf")
+
+        await mail_client.login()
+        result = await mail_client.send(my_mail)
+        print("MAIL RESULT:", json.to_string(result))
+        await mail_client.logout()
+
+    asyncio.run(test())

mail/mail_parser.py

@@ -0,0 +1,266 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Tuesday, 10th Dec., 2024
+
+    OBJECTIVE:
+
+        To parse raw mail bodies and give a structure that is suitable for storing in No-SQL databases like MongoDB. The
+        raw mail's text is expected to be compliant with standard defined in RFC 5322, RFC 2045, and maybe a few more.
+
+    REFERENCES:
+
+        1. GitHub: https://github.com/SpamScope/mail-parser
+        2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
+        3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
+        4. StackOverflow: https://stackoverflow.com/questions/17874360/python-how-to-parse-the-body-from-a-raw-email-given-that-raw-email-does-not
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# To work with mails:
+import email
+from email.message import Message
+from email.utils import parsedate_tz
+from email.utils import parseaddr
+
+# To parse the HTML content in the mail:
+from bs4 import BeautifulSoup
+
+# To work with datatypes:
+from typing import Any, Dict, List, Literal
+
+# To work with various encodings:
+import base64
+import quopri
+
+# To work with date and time:
+import datetime
+import pytz
+import time
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def parse_addr(addr_header: str) -> List[Dict[str, str]]:
+
+    # If the field is null, we return null:
+    if addr_header is None: return []
+
+    # Create an empty variable that will hold the results:
+    addrs = []
+
+    # Iterate through the addresses and parse them:
+    for a in addr_header.split(","):
+        n, e = parseaddr(a.strip())
+        addrs.append({
+            "name": n.strip() or e.strip(),
+            "email": e.strip()
+        })
+
+    # Done here:
+    return addrs
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def parse_date(date_header: str) -> datetime.datetime | None:
+
+    # Try to parse the date header:
+    date_tuple = parsedate_tz(date_header)
+
+    # If the date header was parsed successfully, we assemble
+    # the parts to get an aware object in UTC timezone:
+    if date_tuple:
+        dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60)))
+        dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC)
+        return dt
+
+    # In case of an invalid date header:
+    else: return None
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def decode_payload(
+        raw_payload: str | bytes,
+        content_main_type: str,
+        content_charset: str | None,
+        content_transfer_encoding: Literal[None, "base64", "quoted-printable"]
+) -> str | bytes:
+
+    # Start by assuming nothing needs to be done:
+    payload = raw_payload
+
+    # We decode various kinds of parts:
+    match content_transfer_encoding:
+
+        # This is just unencoded plaintext:
+        case None:
+            pass
+
+        # Typically see with attachments:
+        case "base64":
+            charset = content_charset or "utf-8"
+            payload = raw_payload
+            payload = base64.b64decode(payload)
+            if content_main_type == "text": payload = payload.decode(charset)
+
+        # Typically seen with HTML parts:
+        case "quoted-printable":
+            charset = content_charset or "utf-8"
+            payload = raw_payload.encode(charset)
+            payload = quopri.decodestring(payload)
+            if content_main_type == "text": payload = payload.decode(charset)
+
+    # Done here:
+    return payload
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def parse_part(
+        part: Message | List[Message]
+) -> Dict[str, Any]:
+
+    # Start by extracting basic details:
+    part_json = {
+        "boundary": part.get_boundary(),
+        "contentType": part.get_content_type(),
+        "contentMainType": part.get_content_maintype(),
+        "contentSubType": part.get_content_subtype(),
+        "contentCharset": part.get_content_charset(),
+        "contentTransferEncoding": part.get("Content-Transfer-Encoding"),
+        "contentDisposition": part.get_content_disposition(),
+        "filename": part.get_filename(),
+        "contentId": part.get("Content-ID")
+    }
+
+    # Process the payload of this part:
+    if part_json["contentMainType"] == "multipart":
+        part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)]
+    else:
+        part_json["payload"] = decode_payload(
+            raw_payload = part.get_payload(decode = False),
+            content_main_type = part_json["contentMainType"],
+            content_charset = part_json["contentCharset"],
+            content_transfer_encoding = part_json["contentTransferEncoding"]
+        )
+
+    # Done here;
+    return part_json
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def parse(raw_mail: str | bytes) -> Dict[str, Any]:
+
+    """
+    To parse the raw mail text to a usable JSON that can even be stored on a No-SQL database like MongoDB.
+    DOCUMENTATION:
+        1. GitHub: https://github.com/SpamScope/mail-parser
+        2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
+        3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
+    :param raw_mail: The raw mail body that adheres to RFC 5322 and RFC 2045 (among others).
+    :return: The parsed JSON format (dict) of the mail.
+    """
+
+    # Parse the raw format:
+    if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
+    else: parsed_mail = email.message_from_bytes(raw_mail)
+
+    # Extract the most basic details:
+    mail_json = {
+        "ts": parse_date(parsed_mail["Date"]),
+        "headers": {k: v for k, v in parsed_mail.items()},
+        "from": parse_addr(parsed_mail["From"]),
+        "to": parse_addr(parsed_mail["To"]),
+        "cc": parse_addr(parsed_mail["Cc"]),
+        "bcc": parse_addr(parsed_mail["Bcc"]),
+        "subject": parsed_mail["Subject"],
+        "payload": None
+    }
+
+    # Iterate through each part of the mail for multipart mails:
+    if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail)
+
+    # When the mails are not multipart, just plaintext:
+    else: mail_json["payload"] = parsed_mail.get_payload()
+
+    # Done here:
+    return mail_json
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    from utils_v2.system import files
+
+    mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
+    parse_results = parse(mail_string_raw)
+
+    print(json.to_string(parse_results, default = str))

network/ip.py

@@ -0,0 +1,173 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 12th Jul., 2024
+
+    OBJECTIVE:
+
+        To provide an easy way to get geolocation information of an IP address.
+
+    REFERENCES:
+
+        1) https://medium.com/@tubelwj/how-to-retrieve-ip-geolocation-information-in-python-929e15041e3e
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For working with IP Addresses:
+import ipaddress
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def ipv4_to_int(ip_string):
+
+    """
+    Converts an IP (v4) string to an integer value.
+    :param ip_string: The IP address (v4) that you want to convert to integer format.
+    :return: An integer representation of the IP (v4) address.
+    """
+
+    ip_numerical = int(ipaddress.IPv4Address(ip_string))
+    return ip_numerical
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def int_to_ipv4(ip_numerical):
+
+    """
+    Interprets the IP (v4) value from the given integer value.
+    :param ip_numerical: The integer value that represents an IP (v4) address.
+    :return:
+    """
+
+    ip_string = str(ipaddress.IPv4Address(ip_numerical))
+    return ip_string
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def ipv4_to_bin(ip_string):
+
+    """
+    Converts an input IP (v4) address to the binary string that represents the 32 bits.
+    :param ip_string: The IP (v4) string in a format like "192.168.0.1"
+    :return: The binary representation (as a string) of the input IP address.
+    """
+
+    ip_binary = bin(int(ipaddress.IPv4Address(ip_string)))[2:].zfill(32)
+    return ip_binary
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def bin_to_ipv4(ip_binary):
+
+    """
+    Interprets the IP (v4) value from the given binary string.
+    :param ip_binary: The string of 1s and 0s that represents the IP (v4) address.
+    :return: The IP (v4) address as a string.
+    """
+
+    ip_string = str(ipaddress.IPv4Address(int(ip_binary, 2)))
+    return ip_string
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def get_ipv4_range(ip_string, as_string = True):
+
+    """
+    Given a network description in the format "88.95.100.128/25", this function tells you the first and last IP
+    addresses of that network. Useful for determining if an IP address lies in a network.
+    :param ip_string: The input network description in the format "88.95.100.128/25"
+    :param as_string: To select between integer and string formats for the IP range output.
+    :return: The first and last IP addresses of the input network, and the count.
+    """
+
+    # Extract the components of the string:
+    ip_components = ip_string.split("/")
+    ip_addr = ipv4_to_int(ip_components[0])
+    ip_bits = int(ip_components[1])
+
+    # Convert the mask number to binary representation:
+    ip_mask = (1 << ip_bits) - 1
+    ip_mask = ip_mask << (32 - ip_bits)
+    inv_ip_mask = (~ip_mask) & 0xFFFF
+
+    # Figure out the start and end IP addresses:
+    start_ip = ip_addr & ip_mask
+    end_ip = ip_addr | inv_ip_mask
+    count = end_ip - start_ip + 1
+
+    # If the IPs are needed as strings, we perform the conversion:
+    if as_string:
+        start_ip = int_to_ipv4(start_ip)
+        end_ip = int_to_ipv4(end_ip)
+
+    # Done here:
+    return start_ip, end_ip, count
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    print(ipv4_to_int("255.255.255.255"))
+    print(ipv4_to_int("x.x.x.x"))

network/ping.py

@@ -0,0 +1,201 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 28th Jun, 2024
+
+    OBJECTIVE:
+
+        To provide a way to ping a server and get the traceroute dump.
+
+    REFERENCES:
+
+        1) https://www.geeksforgeeks.org/traceroute-implementation-on-python/
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# Utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# For networking:
+import socket
+from scapy.all import *
+
+# For running the script from the terminal:
+import argparse
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+def get_name_and_addr(destination):
+
+    # If the user provided the IP address:
+    if (
+            regex.match(destination, regex.REGEX_IPV4) or
+            regex.match(destination, regex.REGEX_IPV6)
+    ):
+        try: destination_name = socket.gethostbyaddr(destination)[0]
+        except Exception as exception: destination_name = "*"
+        destination_ip = destination
+
+    # If the provided destination was the domain name:
+    else:
+        destination_name = destination
+        try: destination_ip = socket.gethostbyname(destination)
+        except Exception as exception: destination_ip = "*"
+
+    # Done here:
+    return destination_name, destination_ip
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def tracert(
+        destination,
+        max_hops = 30,
+        timeout = 2.0,
+        port = 33434
+):
+
+    # Initialize the variables:
+    destination_name, destination_ip = get_name_and_addr(destination)
+    full_trace = []
+    ttl = 1
+
+    # Keep noting hops till the limit is reached:
+    while ttl <= max_hops:
+
+        # Create a JSON for this stage:
+        this_hop = {
+            "destAddr": destination_ip,
+            "destName": destination_name,
+            "hopNo": ttl - 1,
+            "isDest": False,
+            "hopAddr": None,
+            "hopName": None,
+            "ts": None
+        }
+
+        # Create the IP and UDP headers and combine them:
+        ip_packet = IP(dst = destination, ttl = ttl)
+        udp_packet = UDP(dport = port)
+        trace_packet = ip_packet / udp_packet
+
+        # Send the packet and receive a reply and note down the timestamp:
+        reply = sr1(trace_packet, timeout = timeout, verbose = 0)
+        this_hop["ts"] = date_time.get_current_utc_date_time(as_string = True)
+
+        # No response:
+        if reply is None: this_hop["hopAddr"] = this_hop["hopName"] = "*"
+
+        # If some response was received, we note the values and break out if this was the destination hop:
+        else:
+            this_hop["hopName"], this_hop["hopAddr"] = get_name_and_addr(f"{reply.src}")
+            if reply.type == 3:
+                this_hop["isDest"] = True
+                full_trace.append(this_hop)
+                break
+
+        # Carry on to the next hop:
+        full_trace.append(this_hop)
+        ttl += 1
+
+    # Done here:
+    return full_trace
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser(description = "Traceroute Implementation in Python!")
+
+    parser.add_argument(
+        "dest",
+        help = "Destination (Name or IP address)."
+    )
+    parser.add_argument(
+        "-m",
+        "--max-hops",
+        type = int,
+        default = 30,
+        help =
+        "Maximum number of hops (default: 30)."
+    )
+    parser.add_argument(
+        "-t",
+        "--timeout",
+        type = float,
+        default = 2.0,
+        help = "Timeout for each packet in seconds (default: 2.0)."
+    )
+    parser.add_argument(
+        "-p",
+        "--port",
+        type = int,
+        default = 33434,
+        help = "The port to probe at the destination (default: 33434)."
+    )
+
+    args = parser.parse_args()
+
+    trace = tracert(
+        destination = args.dest,
+        max_hops = args.max_hops,
+        timeout = args.timeout,
+        port = args.port
+    )
+    print("TRACE:", json.to_string(trace))

nse/controllers/base.py

@@ -0,0 +1,281 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a base class for common behaviour of NSE's APIs.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.date_time import date_time
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# For working with datatypes:
+from typing import Literal, List
+
+# For debugging:
+from icecream import IceCreamDebugger
+import inspect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncNSEBase:
+
+    def __init__(
+            self,
+            base_url: str,
+            data_url: str,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE | ",
+            debug_only_errors = True
+    ):
+
+        """
+        To be used as a base class for all sorts of NSE scraping.
+        :param base_url: The base URL (that you open in your browser) for your targeted module.
+        :param data_url: The internal URL (that NSE's script calls by itself) for your targeted module.
+        :param http_client: An HTTP client to use to make API calls.
+        :param cookies_refresh_interval: The no. of seconds after which the cookies should get refreshed.
+        :param debug: Whether, or not, you want to show debugging messages.
+        :param debug_prefix: The prefix string to use to recognize the module that is printing the debugging text.
+        :param debug_only_errors: Whether you want to show all debugging messages, or just error messages.
+        """
+
+        # Prepare the debugging utility:
+        self._debug_prefix = debug_prefix
+        self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self._printer.disable()
+        self._debug_only_errors = debug_only_errors
+
+        # Accept the input configuration:
+        self._base_url = base_url
+        self._data_url = data_url
+        self._http_client = http_client
+
+        # Make some provisions for later:
+        self._cookies = None
+        self._cookies_refresh_interval = cookies_refresh_interval
+        self._last_cookies_refresh = datetime.datetime.fromtimestamp(0, tz = date_time.TIMEZONE_UTC)
+
+    def enable_debug(self):
+        self._printer.enable()
+
+    def disable_debug(self):
+        self._printer.disable()
+
+    def debug_only_errors(self):
+        self._debug_only_errors = True
+
+    def debug_everything(self):
+        self._debug_only_errors = False
+
+    # ┏┓    ┓ •     ┳┳┓  ┓ •
+    # ┃ ┏┓┏┓┃┏┓┏┓┏  ┃┃┃┏┓┃┏┓┏┓┏┓
+    # ┗┛┗┛┗┛┛┗┗┗ ┛  ┛ ┗┗┻┛┗┗┛┗┗┫
+    #                          ┛
+
+    @property
+    def seconds_since_cookies_refreshed(self):
+        return (date_time.get_current_utc_date_time() - self._last_cookies_refresh).total_seconds()
+
+    async def refresh_cookies(self) -> bool:
+
+        """
+        To get cookies for the given base URL. NSE has some sort of strict cookies and origin policy that is beyond my
+        current understanding. But I have noticed that if you furnish the right base URL and then use that, NSE will
+        respond just fine.
+        USAGE: Call this right after you initialize your class and then call it every so often after that.
+        :return: The cookies in the 'data' field of the
+        """
+
+        # Start by assuming failure:
+        success = False
+
+        # Make the API call:
+        api_response = await self.get(
+            url = self._base_url,
+            headers = {
+                "Accept": "*/*",
+                "Accept-Encoding": "gzip, deflate, br",
+                "Accept-Language": "en-US,en;q=0.9",
+                "Connection": "keep-alive",
+                "Upgrade-Insecure-Requests": "1",
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
+            },
+            refresh_cookies = False
+        )
+
+        # If the API call succeeds:
+        if api_response.httpCode in [200]:
+            self._last_cookies_refresh = date_time.get_current_utc_date_time(as_string = False)
+            self._cookies = api_response.response.cookies
+
+        # If it failed, alert on the terminal:
+        else: self._printer("Cookies Refresh FAILED!")
+
+        # Done here:
+        return success
+
+    # ┏┓┏┓┳  ┏┓  ┓┓•
+    # ┣┫┃┃┃  ┃ ┏┓┃┃┓┏┓┏┓
+    # ┛┗┣┛┻  ┗┛┗┻┗┗┗┛┗┗┫
+    #                  ┛
+
+    async def get(
+            self,
+            url: str = None,
+            headers: dict = None,
+            params: dict = None,
+            cookies: httpx.Cookies | dict = None,
+            refresh_cookies: bool = True
+    ) -> NSEApiResponse:
+
+        """
+        To call an API using the GET method.
+        :param url: A custom URL to hit. If not specified, the data URL will be hit.
+        :param headers: Custom headers to use. If not specified, default values will be used.
+        :param params: The params to send in the query string itself.
+        :param cookies: Custom cookies to send, else the ones fetched from the base URL will be used.
+        :param refresh_cookies: Set this to False is you would like to block the auto refreshing of cookies. Remember
+            that the refreshing is controlled through the interval specified in the constructor.
+        :return: A structured response that includes the raw response, the exception (if any), and so on.
+        """
+
+        # Select and refresh the cookies as needed:
+        if cookies is None and refresh_cookies:
+            if self.seconds_since_cookies_refreshed > self._cookies_refresh_interval: await self.refresh_cookies()
+            use_cookies = self._cookies
+        else: use_cookies = cookies
+
+        # Prepare the structure of the response:
+        api_response = NSEApiResponse(
+            action = inspect.stack()[1].function,
+            url = url or self._data_url,
+            method = "GET"
+        )
+
+        try:
+
+            # Make the API call:
+            response = await self._http_client.get(
+                url = url or self._data_url,
+                headers = headers or {
+                    "Accept": "*/*",
+                    "Accept-Encoding": "gzip, deflate, br",
+                    "Accept-Language": "en-US,en;q=0.9",
+                    "Connection": "keep-alive",
+                    "Host": "www.nseindia.com",
+                    "Referer": self._base_url,
+                    "Sec-Fetch-Dest": "empty",
+                    "Sec-Fetch-Mode": "cors",
+                    "Sec-Fetch-Site": "same-origin",
+                    "Sec-GPC": "1",
+                    "Upgrade-Insecure-Requests": "1",
+                    "TE": "trailers",
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
+                },
+                params = params,
+                cookies = use_cookies
+            )
+
+            # Note down the results:
+            api_response.response = response
+            api_response.httpCode = response.status_code
+            api_response.message = response.reason_phrase
+
+        # If something goes wrong:
+        except Exception as exception:
+            api_response.exception = exception
+            api_response.message = str(exception)
+            self._printer(exception, api_response.url, api_response.method, headers, params)
+
+        # Done here:
+        return api_response
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

nse/controllers/calendar/corporate_events.py

@@ -0,0 +1,232 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates of important events like financial-results, stock-splits, fund-raising, etc.
+        from NSE's portal.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSECorporateEventCalendar(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            debug = True,
+            debug_prefix = "NSE (CECal) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/companies-listing/corporate-filings-event-calendar",
+            data_url = r"https://www.nseindia.com/api/event-calendar",
+            http_client = http_client,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get()
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = True),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: List[dict],
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                {
+                    "scrapeTs": timestamp,
+                    "symbol": event["symbol"],
+                    "company": event["company"],
+                    "eventDate": date_time.to_timezone(
+                        date_time.parse_date_time(
+                            event["date"],
+                            timezone = date_time.TIMEZONE_IST
+                        ),
+                        timezone = date_time.TIMEZONE_UTC
+                    ),
+                    "eventType": event["purpose"],
+                    "brief": event["bm_desc"]
+                } for event in raw_json
+            ]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSECorporateEventCalendar(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("CORPORATE EVENT CALENDAR:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/calendar/examples/20241128_formatted_nse_corporate_event_calendar.json

@@ -0,0 +1,178 @@
+[
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "AAREYDRUGS",
+        "company": "Aarey Drugs & Pharmaceuticals Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Fund Raising/Other business matters",
+        "brief": "To consider Fund Raising and other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "ASTERDM",
+        "company": "Aster DM Healthcare Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters pertaining to Preferential Issue."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "EROSMEDIA",
+        "company": "Eros International Media Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Financial Results/Other business matters",
+        "brief": "To consider and approve the financial results for the quarter and year ended March 31, 2024 and other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "MUFIN",
+        "company": "Mufin Green Finance Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Fund Raising",
+        "brief": "To consider Fund Raising by Issue of Secured Unlisted Non Convertible Debenture."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "ONMOBILE",
+        "company": "OnMobile Global Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Fund Raising",
+        "brief": "To consider Fund Raising"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "PRIVISCL",
+        "company": "Privi Speciality Chemicals Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider and Approve \"Privi Employee Stock Option Scheme - 2024\""
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "SOUTHBANK",
+        "company": "The South Indian Bank Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider, decide on exercise of call option of Bank s Non-convertible, fully paid-up, unsecured, perpetual, Basel III Compliant, Tier I Bonds, listed in BSE"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "eventDate": "2024-11-29 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "eventDate": "2024-11-29 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "To consider and approve the financial results for the period ended Jun 30, 2024"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "SIKKO",
+        "company": "Sikko Industries Limited",
+        "eventDate": "2024-11-29 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider discuss and decide terms and conditions such as determination of the rights issue price, rights entitlement ratio, record date and other matters incidental orconnected therewith."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "AGIIL",
+        "company": "Agi Infra Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Stock Split",
+        "brief": "To consider stock split of equity shares of the Company"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "MANAKCOAT",
+        "company": "Manaksia Coated Metals & Industries Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Fund Raising",
+        "brief": "To consider Fund Raising"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider the enclosed business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "PAR",
+        "company": "Par Drugs and Chemicals Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "FIBERWEB",
+        "company": "Fiberweb (India) Limited",
+        "eventDate": "2024-12-02 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider and discuss about the expansion plans"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "IITL",
+        "company": "Industrial Investment Trust Limited",
+        "eventDate": "2024-12-02 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "SWIGGY",
+        "company": "Swiggy Limited",
+        "eventDate": "2024-12-02 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "To consider and approve the financial results for the quarter and half year ended September 30, 2024"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "GBGLOBAL",
+        "company": "GB Global Limited",
+        "eventDate": "2024-12-03 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "MOTOGENFIN",
+        "company": "The Motor & General Finance Limited",
+        "eventDate": "2024-12-03 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "Intimation Regarding Independent Directors Meeting will be held on 04.12.2024."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "DIL",
+        "company": "Debock Industries Limited",
+        "eventDate": "2024-12-04 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "DIL : 05-Dec-2024 :  The Company has informed the Exchange that a Board meeting to be held on November 27, 2024 has been re-scheduled. Further, the Company has informed the Exchange that the meeting of the Board of Directors of the Company will be held on December 05, 2024,  To consider and approve the financial results for the period ended September 30, 2024"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "eventDate": "2024-12-08 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider enclosed business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "EXIDEIND",
+        "company": "Exide Industries Limited",
+        "eventDate": "2025-01-27 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "To consider and approve the financial results for the period ended December 31, 2024"
+    }
+]

nse/controllers/calendar/examples/20241128_raw_nse_corporate_event_calendar.json

@@ -0,0 +1,156 @@
+[
+    {
+        "symbol": "AAREYDRUGS",
+        "company": "Aarey Drugs & Pharmaceuticals Limited",
+        "purpose": "Fund Raising/Other business matters",
+        "bm_desc": "To consider Fund Raising and other business matters",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "ASTERDM",
+        "company": "Aster DM Healthcare Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters pertaining to Preferential Issue.",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "EROSMEDIA",
+        "company": "Eros International Media Limited",
+        "purpose": "Financial Results/Other business matters",
+        "bm_desc": "To consider and approve the financial results for the quarter and year ended March 31, 2024 and other business matters",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "MUFIN",
+        "company": "Mufin Green Finance Limited",
+        "purpose": "Fund Raising",
+        "bm_desc": "To consider Fund Raising by Issue of Secured Unlisted Non Convertible Debenture.",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "ONMOBILE",
+        "company": "OnMobile Global Limited",
+        "purpose": "Fund Raising",
+        "bm_desc": "To consider Fund Raising",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "PRIVISCL",
+        "company": "Privi Speciality Chemicals Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider and Approve \"Privi Employee Stock Option Scheme - 2024\"",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "SOUTHBANK",
+        "company": "The South Indian Bank Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider, decide on exercise of call option of Bank s Non-convertible, fully paid-up, unsecured, perpetual, Basel III Compliant, Tier I Bonds, listed in BSE",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "30-Nov-2024"
+    },
+    {
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "To consider and approve the financial results for the period ended Jun 30, 2024",
+        "date": "30-Nov-2024"
+    },
+    {
+        "symbol": "SIKKO",
+        "company": "Sikko Industries Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider discuss and decide terms and conditions such as determination of the rights issue price, rights entitlement ratio, record date and other matters incidental orconnected therewith.",
+        "date": "30-Nov-2024"
+    },
+    {
+        "symbol": "AGIIL",
+        "company": "Agi Infra Limited",
+        "purpose": "Stock Split",
+        "bm_desc": "To consider stock split of equity shares of the Company",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "MANAKCOAT",
+        "company": "Manaksia Coated Metals & Industries Limited",
+        "purpose": "Fund Raising",
+        "bm_desc": "To consider Fund Raising",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider the enclosed business matters",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "PAR",
+        "company": "Par Drugs and Chemicals Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "FIBERWEB",
+        "company": "Fiberweb (India) Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider and discuss about the expansion plans",
+        "date": "03-Dec-2024"
+    },
+    {
+        "symbol": "IITL",
+        "company": "Industrial Investment Trust Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "03-Dec-2024"
+    },
+    {
+        "symbol": "SWIGGY",
+        "company": "Swiggy Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "To consider and approve the financial results for the quarter and half year ended September 30, 2024",
+        "date": "03-Dec-2024"
+    },
+    {
+        "symbol": "GBGLOBAL",
+        "company": "GB Global Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "04-Dec-2024"
+    },
+    {
+        "symbol": "MOTOGENFIN",
+        "company": "The Motor & General Finance Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "Intimation Regarding Independent Directors Meeting will be held on 04.12.2024.",
+        "date": "04-Dec-2024"
+    },
+    {
+        "symbol": "DIL",
+        "company": "Debock Industries Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "DIL : 05-Dec-2024 :  The Company has informed the Exchange that a Board meeting to be held on November 27, 2024 has been re-scheduled. Further, the Company has informed the Exchange that the meeting of the Board of Directors of the Company will be held on December 05, 2024,  To consider and approve the financial results for the period ended September 30, 2024",
+        "date": "05-Dec-2024"
+    },
+    {
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider enclosed business matters",
+        "date": "09-Dec-2024"
+    },
+    {
+        "symbol": "EXIDEIND",
+        "company": "Exide Industries Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "To consider and approve the financial results for the period ended December 31, 2024",
+        "date": "28-Jan-2025"
+    }
+]

nse/controllers/calendar/examples/20241205_formatted_nse_trading_holiday_calendar.json

@@ -0,0 +1,542 @@
+[
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-01-21 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Special Holiday",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-01-25 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Republic Day",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-02-18 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Chatrapati Shivaji Maharaj Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-07 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Mahashivratri",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-24 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Holi",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-28 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Good Friday",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-31 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Annual Bank Closing",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-08 18:30:00+00:00",
+        "weekDay": "Tuesday",
+        "event": "Gudi Padwa",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-10 18:30:00+00:00",
+        "weekDay": "Thursday",
+        "event": "Id-Ul-Fitr (Ramadan Id)",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-13 18:30:00+00:00",
+        "weekDay": "Sunday",
+        "event": "Dr.Baba Saheb Ambedkar Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-16 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Shri Ram Navami",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-20 18:30:00+00:00",
+        "weekDay": "Sunday",
+        "event": "Mahavir Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-30 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Maharashtra Day",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-05-19 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "General Parliamentary Elections",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-05-22 18:30:00+00:00",
+        "weekDay": "Thursday",
+        "event": "Buddha Pournima",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-06-16 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Bakri Id",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-07-16 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Muharram",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-08-14 18:30:00+00:00",
+        "weekDay": "Thursday",
+        "event": "Independence Day/Parsi New Year",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-09-06 18:30:00+00:00",
+        "weekDay": "Saturday",
+        "event": "Ganesh Chaturthi",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-09-15 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Id-E-Milad",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-10-01 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Mahatma Gandhi Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-10-11 18:30:00+00:00",
+        "weekDay": "Saturday",
+        "event": "Dussehra",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-11-01 18:30:00+00:00",
+        "weekDay": "Saturday",
+        "event": "Balipratipada",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-11-14 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Prakash Gurpurb Sri Guru Nanak Dev",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-11-19 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Assembly Elections in Maharashtra",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-12-24 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Christmas",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2023-12-31 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "New year",
+        "morningSession": "Open",
+        "eveningSession": "Closed",
+        "holidayType": [
+            "COM"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-10-31 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Diwali Laxmi Pujan",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "MF",
+            "NDM",
+            "NTRP"
+        ]
+    }
+]

nse/controllers/calendar/trading_holidays.py

@@ -0,0 +1,301 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 5th Dec., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve the list of holidays from NSE and to check if today is a holiday.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+from time import timezone
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+# For debugging:
+import inspect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSETradingHolidayCalendar(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            debug = True,
+            debug_prefix = "NSE (Hol) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/resources/exchange-communication-holidays",
+            data_url = r"https://www.nseindia.com/api/holiday-master?type=trading",
+            http_client = http_client,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the holiday calendar.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get()
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = True),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Start by creating an empty dict:
+            formatted_data = {}
+
+            # Process each category in the original data
+            for category, events in raw_json.items():
+                for event in events:
+                    trading_date = event["tradingDate"]
+
+                    # If the tradingDate is not in the dictionary, initialize it
+                    if trading_date not in formatted_data:
+                        formatted_data[trading_date] = {
+                            "scrapeTs": timestamp,
+                            "date": date_time.to_timezone(
+                                date_time.as_if_timezone(
+                                    datetime.datetime.strptime(trading_date, "%d-%b-%Y"),
+                                    timezone = date_time.TIMEZONE_IST
+                                ),
+                                timezone = date_time.TIMEZONE_UTC
+                            ),
+                            "weekDay": event["weekDay"],
+                            "event": event["description"],
+                            "morningSession": event["morning_session"],
+                            "eveningSession": event["evening_session"],
+                            "holidayType": []
+                        }
+
+                    # Add the current category to the "type" list
+                    formatted_data[trading_date]["holidayType"].append(category)
+
+            # Now we get rid of the unnecessary keys and make it a list:
+            formatted_data = [v for v in formatted_data.values()]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+    async def is_holiday(
+            self,
+            timestamp: datetime.datetime
+    ) -> NSEApiResponse:
+
+        """
+        Checks if a given date is a trading holiday, or not.
+        :param timestamp: The datetime instance that you want to check for it being a holiday. Preferably make it an
+            aware instance. If a naive instance is passed, it will be assumed to be in UTC.
+        :return: True if it is a holiday, False if it isn't. Can be None if something goes wrong in fetching the data.
+            The value will be in the 'data' field of the response model.
+        """
+
+        # We first prepare UTC and IST versions of the incoming datetime:
+        timestamp_utc = date_time.to_timezone(timestamp, timezone = date_time.TIMEZONE_UTC)
+        timestamp_ist = date_time.to_timezone(timestamp_utc, timezone = date_time.TIMEZONE_IST)
+
+        # If this day is a weekend, it is a holiday by default:
+        ist_dow = timestamp_ist.weekday()
+        if ist_dow in [5, 6]: return NSEApiResponse(
+            action = inspect.stack()[0].function,
+            url = None,
+            method = None,
+            success = True,
+            data = True,
+            message = f"Weekend ({'Saturday' if ist_dow == 5 else 'Sunday'})"
+        )
+
+        # First we fetch the data from NSE:
+        api_response = await self.get_data(return_raw = False)
+        api_response.action = inspect.stack()[0].function
+        if not api_response.success: return api_response
+
+        # Now we check if our date to check is in the list:
+        holidays_list = [h["date"].date() for h in api_response.data]
+        api_response.data = True if timestamp_utc.date() in holidays_list else False
+
+        # Done here:
+        return api_response
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSETradingHolidayCalendar(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("HOLIDAY CALENDAR:", json.to_string(api_response.data, default = str))
+        print("\n---\n\n")
+
+        # Check for a holiday:
+        date_to_check = date_time.parse_date_time(
+            input_value = "2024-12-25 00:00:00",
+            date_formats = ["%Y-%m-%d %H:%M:%S"],
+            timezone = date_time.TIMEZONE_IST
+        )
+        api_response = await my_nse.is_holiday(date_to_check)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        print("DATETIME:", date_to_check)
+        print("IS HOLIDAY:", api_response.data)
+
+    asyncio.run(main())

nse/controllers/indices/constituents.py

@@ -0,0 +1,301 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 16th Dec., 2024
+
+    OBJECTIVE:
+
+        This file aims to fetch the current details about the constituents of various indices. This gives you not only
+        the constituent stocks of the selected index, but also that stock's current activity in the market.
+        https://www.nseindia.com/market-data/live-equity-market?symbol=NIFTY%2050
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEIndexConstituents(AsyncNSEBase):
+
+    # Broad Market Indices:
+    INDEX_NIFTY_50 = "NIFTY 50"
+    INDEX_NIFTY_NEXT_50 = "NIFTY NEXT 50"
+    INDEX_NIFTY_MIDCAP_50 = "NIFTY MIDCAP 50"
+    INDEX_NIFTY_MIDCAP_100 = "NIFTY MIDCAP 100"
+    INDEX_NIFTY_MIDCAP_150 = "NIFTY MIDCAP 150"
+    INDEX_NIFTY_SMALLCAP_50 = "NIFTY SMALLCAP 50"
+    INDEX_NIFTY_SMALLCAP_100 = "NIFTY SMALLCAP 100"
+    INDEX_NIFTY_SMALLCAP_250 = "NIFTY SMALLCAP 250"
+    INDEX_NIFTY_MIDSMALLCAP_400 = "NIFTY MIDSMALLCAP 400"
+    INDEX_NIFTY_100 = "NIFTY 100"
+    INDEX_NIFTY_200 = "NIFTY 200"
+    INDEX_NIFTY_500_MULTICAP_50_25_25 = "NIFTY500 MULTICAP 50:25:25"
+    INDEX_NIFTY_LARGEMIDCAP_250 = "NIFTY LARGEMIDCAP 250"
+    INDEX_NIFTY_MIDCAP_SELECT = "NIFTY MIDCAP SELECT"
+    INDEX_NIFTY_TOTAL_MARKET = "NIFTY TOTAL MARKET"
+    INDEX_NIFTY_MICROCAP_250 = "NIFTY MICROCAP 250"
+    INDEX_NIFTY_500 = "NIFTY 500"
+    INDEX_NIFTY_500_LARGEMIDSMALL_EQUAL_CAP_WEIGHTED = "NIFTY500 LARGEMIDSMALL EQUAL-CAP WEIGHTED"
+
+    # Sectoral Indices:
+    INDEX_NIFTY_AUTO = "NIFTY AUTO"
+    INDEX_NIFTY_BANK = "NIFTY BANK"
+    INDEX_NIFTY_ENERGY = "NIFTY ENERGY"
+    INDEX_NIFTY_FINANCIAL_SERVICES = "NIFTY FINANCIAL SERVICES"
+    INDEX_NIFTY_FINANCIAL_SERVICES_25_50 = "NIFTY FINANCIAL SERVICES 25/50"
+    INDEX_NIFTY_FMCG = "NIFTY FMCG"
+    INDEX_NIFTY_IT = "NIFTY IT"
+    INDEX_NIFTY_MEDIA = "NIFTY MEDIA"
+    INDEX_NIFTY_METAL = "NIFTY METAL"
+    INDEX_NIFTY_PHARMA = "NIFTY PHARMA"
+    INDEX_NIFTY_PSU_BANK = "NIFTY PSU BANK"
+    INDEX_NIFTY_REALTY = "NIFTY REALTY"
+    INDEX_NIFTY_PRIVATE_BANK = "NIFTY PRIVATE BANK"
+    INDEX_NIFTY_HEALTHCARE_INDEX = "NIFTY HEALTHCARE INDEX"
+    INDEX_NIFTY_CONSUMER_DURABLES = "NIFTY CONSUMER DURABLES"
+    INDEX_NIFTY_OIL_GAS = "NIFTY OIL & GAS"
+    INDEX_NIFTY_MIDSMALL_HEALTHCARE = "NIFTY MIDSMALL HEALTHCARE"
+    INDEX_NIFTY_FINANCIAL_SERVICES_EX_BANK = "NIFTY FINANCIAL SERVICES EX-BANK"
+    INDEX_NIFTY_MIDSMALL_FINANCIAL_SERVICES = "NIFTY MIDSMALL FINANCIAL SERVICES"
+    INDEX_NIFTY_MIDSMALL_IT_TELECOM = "NIFTY MIDSMALL IT & TELECOM"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            debug = True,
+            debug_prefix = "NSE (IdxCons) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/live-equity-market",
+            data_url = r"https://www.nseindia.com/api/equity-stockIndices",
+            http_client = http_client,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            index_name: str,
+            return_raw: bool = False,
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param index_name: The value held in the 'indexName' field of the formatted output of the Index Master.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(params = {"index": index_name})
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    index_name = index_name,
+                    timestamp = date_time.get_current_utc_date_time(as_string = True),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            index_name: str,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param index_name: The value that you had used to fetch the raw data in the first place.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                {
+                    "scrapeTs": timestamp,
+                    "ts": date_time.to_timezone(
+                        date_time.parse_date_time(
+                            input_value = symbol["lastUpdateTime"],
+                            date_formats = ["%d-%b-%Y %H:%M:%S"],
+                            timezone = date_time.TIMEZONE_IST
+                        ),
+                        timezone = date_time.TIMEZONE_UTC
+                    ),
+                    "indexName": index_name,
+                    "symbol": symbol["symbol"],
+                    "name": symbol["meta"]["companyName"],
+                    "industry": symbol["meta"]["industry"],
+                    "isFNOSec": symbol["meta"]["isFNOSec"],
+                    "isSuspended": symbol["meta"]["isSuspended"],
+                    "isin": symbol["meta"]["isin"],
+                    "open": symbol["open"],
+                    "high": symbol["dayHigh"],
+                    "low": symbol["dayLow"],
+                    "close": symbol["lastPrice"],
+                    "totTradedVol": symbol["totalTradedVolume"],
+                    "totTradedVal": symbol["totalTradedValue"],
+                    "prevClose": symbol["previousClose"],
+                    "change": symbol["change"],
+                    "pctChange": symbol["pChange"],
+                    "yearHigh": symbol["yearHigh"],
+                    "yearLow": symbol["yearLow"],
+                    "pctChange30d": symbol["perChange30d"],
+                    "pctChange365d": symbol["perChange365d"],
+                    "ffmc": symbol["ffmc"]
+                } for symbol in raw_json["data"] if symbol["priority"] == 0
+            ]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEIndexConstituents(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(
+            index_name = NSEIndexConstituents.INDEX_NIFTY_50,
+            return_raw = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("INDEX CONSTITUENTS:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/indices/master.py

@@ -0,0 +1,238 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        The "Index Master" contains information about just the indices, and not the component symbols of those indices.
+        This script provides a way to get the data that is available on the screen on the following URL:
+        https://www.nseindia.com/market-data/live-market-indices
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEIndexMaster(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            debug = True,
+            debug_prefix = "NSE (IdxMstr) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/live-market-indices",
+            data_url = r"https://www.nseindia.com/api/allIndices",
+            http_client = http_client,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get()
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = True),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                {
+                    "scrapeTs": timestamp,
+                    "indexType": idx["key"],
+                    "indexName": idx["index"],
+                    "indexSymbol": idx["indexSymbol"],
+                    "open": idx["open"],
+                    "high": idx["high"],
+                    "low": idx["low"],
+                    "close": idx["last"],
+                    "prevClose": idx["previousClose"],
+                    "pctChange": idx["percentChange"],
+                    "yearHigh": idx["yearHigh"],
+                    "yearLow": idx["yearLow"],
+                    "advances": idx.get("advances"),
+                    "declines": idx.get("declines"),
+                    "unchanged": idx.get("unchanged"),
+                    "pctChange30d": idx["perChange30d"],
+                    "pctChange365d": idx["perChange365d"]
+                } for idx in raw_json["data"]
+            ]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEIndexMaster(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("INDEX MASTER:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/market/pre_market.py

@@ -0,0 +1,273 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 5th Dec., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve pre-market data from NSE. This is typically available by 9:10 AM.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEPreMarket(AsyncNSEBase):
+
+    # Symbol names:
+    PRE_MARKET_KEY_NIFTY = "NIFTY"
+    PRE_MARKET_KEY_BANK_NIFTY = "BANKNIFTY"
+    PRE_MARKET_KEY_SME = "SME"
+    PRE_MARKET_KEY_FO = "FO"
+    PRE_MARKET_KEY_OTHERS = "OTHERS"
+    PRE_MARKET_KEY_ALL = "ALL"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            debug = True,
+            debug_prefix = "NSE (CECal) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/pre-open-market-cm-and-emerge-market",
+            data_url = r"https://www.nseindia.com/api/market-data-pre-open",
+            http_client = http_client,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            key: str,
+            return_raw: bool = False,
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the pre-open market trading. Useful for finding gaps and expected unusual activity in the
+        trading hours.
+        :param key: The type of pre-market data that you want. Choose from the class variables.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(params = {"key": key})
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = True),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Start by extracting basic data:
+            formatted_data = {
+                "scrapeTs": timestamp,
+                "ts": date_time.to_timezone(
+                    date_time.as_if_timezone(
+                        date_time.parse_date_time(
+                            input_value = raw_json["timestamp"],
+                            date_formats = ["%d-%b-%Y %H:%M:%S"]
+                        ),
+                        timezone = date_time.TIMEZONE_IST
+                    ),
+                    timezone = date_time.TIMEZONE_UTC
+                ),
+                "advances": raw_json["advances"],
+                "declines": raw_json["declines"],
+                "unchanged": raw_json["unchanged"],
+                "totalMarketCap": raw_json["totalmarketcap"],
+                "totalTradedValue": raw_json["totalTradedValue"],
+                "totalTradedVolume": raw_json["totalTradedVolume"],
+                "data": []
+            }
+
+            # Now we iterate through the symbol-wise data and extract what we need:
+            for raw_symbol_data in raw_json["data"]:
+                raw_symbol_metadata = raw_symbol_data["metadata"]
+                raw_symbol_detail = raw_symbol_data["detail"]["preOpenMarket"]
+                formatted_data["data"].append({
+                    "symbol": raw_symbol_metadata["symbol"],
+                    "marketCap": raw_symbol_metadata["marketCap"],
+                    "trigger": raw_symbol_metadata["purpose"],
+                    "yearHigh": raw_symbol_metadata["yearHigh"],
+                    "yearLow": raw_symbol_metadata["yearLow"],
+                    "prevClose": raw_symbol_metadata["previousClose"],
+                    "premarketPrice": raw_symbol_metadata["iep"],
+                    "chg": raw_symbol_metadata["change"],
+                    "pctChg": raw_symbol_metadata["pChange"],
+                    "totalTradedVolume": raw_symbol_detail["totalTradedVolume"],
+                    "totalBuyVolume": raw_symbol_detail["totalBuyQuantity"],
+                    "totalSellVolume": raw_symbol_detail["totalSellQuantity"],
+                })
+
+            # Data sorting (descending order of percent change):
+            formatted_data["data"] = sorted(
+                formatted_data["data"],
+                key = lambda x: x["pctChg"],
+                reverse = True
+            )
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 9.9  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEPreMarket(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(key = my_nse.PRE_MARKET_KEY_FO, return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("PRE-MARKET DATA:", json.to_string(api_response.data, default = str))
+        if api_response.exception: raise api_response.exception
+
+    asyncio.run(main())