(20241209) Many changes to the SMS section.

api/blueprints/sms/send.py

@@ -0,0 +1,265 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 5th Dec., 2024
+
+    OBJECTIVE:
+
+        To receive auth details for various SMS client APIs.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+    NOTES:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For using Quart:
+from quart import Blueprint, current_app, request
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.api.codes import StatusCodes, HttpCodes
+from utils_v2.api.response import ResponseModel
+from utils_v2.api.async_quart import (
+    set_api_version,
+    read_input,
+    get_session_info,
+    log_request_to_mongo,
+    log_chain_to_mongo,
+    should_not_be_under_maintenance,
+    only_whitelisted_ips,
+    limit_rate,
+    validate_input,
+    handle_cancelled_request
+)
+
+# SMS-related utils:
+from utils_v2.sms.nimbus.async_nimbus import AsyncNimbusSMS
+from utils_v2.sms.savvy_bulk_sms.async_savvy_bulk_sms import AsyncSavvyBulkSMS
+
+# Common:
+from shared import constants
+
+# Data Models:
+from models.data.api.sms.auth import SMSAuthRequestHeaders, SMSAuthRequestData
+from models.data.core.auth_token import CoreAuthTokenModel
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# Related to Quart:
+sms_auth_bp = Blueprint("sms_auth", __name__)
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+@sms_auth_bp.record_once
+def init(blueprint_setup_state):
+
+    # This gets called when the blueprint is registered.
+    # Consider this to be a one-time setup for the whole blueprint:
+    pass
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+@sms_auth_bp.route("/auth", methods = ["POST"])
+@set_api_version(api_version = "1.0.0")
+@read_input(sanitize_headers = False, sanitize_data = False)
+@get_session_info(key = "X-Session-Token", session_coro = "get_session")
+@log_request_to_mongo(
+    attr_name = "logs_mongo",
+    project = constants.PROJECT_NAME,
+    log_type = constants.MODULE_NAME,
+    operation = "smsAuthApi",
+    log_input = True,
+    log_output = True,
+    sensitive_keys = ["sessionToken", "X-Session-Token"]
+)
+@log_chain_to_mongo(attr_name = "logs_mongo")
+@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
+@validate_input(
+    header_validator = lambda x: SMSAuthRequestHeaders(**x).model_dump(),
+    data_validator = lambda x: SMSAuthRequestData(**x)
+)
+@handle_cancelled_request()
+async def request_oauth_authorization_url(
+        inbound_headers: dict | SMSAuthRequestHeaders = None,
+        inbound_data: dict | SMSAuthRequestData = None,
+        inbound_files: dict = None,
+        **kwargs
+):
+
+    """
+    Use this when a user wants to register a third-party SMS client with your service.
+    :param inbound_headers: auto-extracted by the decorators.
+    :param inbound_data: auto-extracted by the decorators.
+    :param inbound_files: auto-extracted by the decorators.
+    :param kwargs: Any number of extra inputs supplied by the decorators.
+    :return: A standard response structure.
+    """
+
+    # ┏┓
+    # ┃┃┏┓┏┓┏┓┏┓┏┓┏┏┓┏┏
+    # ┣┛┛ ┗ ┣┛┛ ┗┛┗┗ ┛┛
+    #       ┛
+
+    # If the session token is invalid/expired:
+    if kwargs.get("session_info") is None:
+        return ResponseModel(
+            status_code = StatusCodes.FAILED,
+            http_code = HttpCodes.UNAUTHORIZED
+        )
+
+    # Start by assuming failure:
+    token_id = None
+
+    # ┏┓      ┳┓•   ┓      ┏┓┳┳┓┏┓  ┳   ┓•
+    # ┣ ┏┓┏┓  ┃┃┓┏┳┓┣┓┓┏┏  ┗┓┃┃┃┗┓  ┃┏┓┏┫┓┏┓
+    # ┻ ┗┛┛   ┛┗┗┛┗┗┗┛┗┻┛  ┗┛┛ ┗┗┛  ┻┛┗┗┻┗┗┻
+
+    if inbound_data.smsClient == "nimbusSmsIndia":
+
+        token_id = await current_app.sms_auth_model.set(
+            db_conn = current_app.sql_writer,
+            mongo_conn = current_app.data_mongo,
+            auth_token = CoreAuthTokenModel(
+                serviceType = "sms",
+                client = inbound_data.smsClient,
+                authType = "auth",
+                auth = inbound_data.auth.model_dump(),
+                user = kwargs.get("session_info"),
+                clientUserId = {
+                    "userId": inbound_data.auth.userId,
+                    "senderId": inbound_data.auth.senderId,
+                    "entityId": inbound_data.auth.entityId
+                },
+                status = "active",
+                syncFreq = 60
+            ),
+            # user_info = kwargs["session_info"],
+            # client_user_id = {
+            #     "userId": inbound_data.auth.userId,
+            #     "senderId": inbound_data.auth.senderId,
+            #     "entityId": inbound_data.auth.entityId
+            # },
+            # auth = inbound_data.auth.model_dump(),
+            # token = None,
+            # service_client = inbound_data.smsClient,
+            # auth_type = "auth",
+            # sync_freq = 300,
+            session_token = inbound_headers["X-Session-Token"]
+        )
+
+    # ┏┓      ┏┓          ┳┓  ┓┓   ┏┓┳┳┓┏┓  ┓┏┓
+    # ┣ ┏┓┏┓  ┗┓┏┓┓┏┓┏┓┏  ┣┫┓┏┃┃┏  ┗┓┃┃┃┗┓  ┃┫ ┏┓┏┓┓┏┏┓
+    # ┻ ┗┛┛   ┗┛┗┻┗┛┗┛┗┫  ┻┛┗┻┗┛┗  ┗┛┛ ┗┗┛  ┛┗┛┗ ┛┗┗┫┗┻
+    #                  ┛                            ┛
+
+    elif inbound_data.smsClient == "savvyBulkSmsKenya":
+
+        token_id = await current_app.sms_auth_model.set(
+            db_conn = current_app.sql_writer,
+            mongo_conn = current_app.data_mongo,
+            auth_token=CoreAuthTokenModel(
+                serviceType = "sms",
+                client = inbound_data.smsClient,
+                authType = "auth",
+                auth = inbound_data.auth.model_dump(),
+                user = kwargs.get("session_info"),
+                clientUserId = {
+                    "partnerId": inbound_data.auth.partnerId,
+                    "shortCode": inbound_data.auth.shortCode
+                },
+                status = "active",
+                syncFreq = 60
+            ),
+            # user_info = kwargs["session_info"],
+            # client_user_id = {
+            #     "partnerId": inbound_data.auth.partnerId,
+            #     "shortCode": inbound_data.auth.shortCode
+            # },
+            # auth = inbound_data.auth.model_dump(),
+            # token = None,
+            # service_client = inbound_data.smsClient,
+            # auth_type = "auth",
+            # sync_freq = 300,
+            session_token = inbound_headers["X-Session-Token"]
+        )
+
+    # ┳┓
+    # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
+    # ┛┗┗ ┛┣┛┗┛┛┗┛┗
+    #      ┛
+
+    # Done here:
+    return ResponseModel(
+        status_code = StatusCodes.OK if token_id else StatusCodes.FAILED,
+        http_code = HttpCodes.SUCCESS if token_id else HttpCodes.INTERNAL_SERVER_ERROR,
+        data = {
+            "smsClient": inbound_data.smsClient,
+            "authorized": True
+        }
+    )
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

models/behaviour/mail/sync_v3.py

@@ -0,0 +1,593 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        tuesday, 3rd Dec., 2024
+
+    OBJECTIVE:
+
+        From here we sync all mails between the mail client's server and TheCAOffice's database.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For Quart:
+from quart import current_app
+
+# My async utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+from utils_v2.database.async_mysql_v2 import AsyncMySQL
+from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
+
+# Mail Clients:
+from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
+from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
+
+# Base model:
+from models.behaviour.base import BaseModel
+
+# Data models:
+from models.data.api.mail.sync import MailSyncOneResult, MailSyncManyResults
+
+# To work with MongoDB:
+from bson import ObjectId
+from pymongo import InsertOne, UpdateOne, ReplaceOne
+
+# To work with LLMs:
+from models.behaviour.ai.llm.open_ai import LLMOpenAI
+from models.data.api.ai.llm import LLMInput
+
+# To work with datatypes:
+from typing import Literal, List, Dict, Any
+
+# To make deep-copies:
+import copy
+
+# To work with base-64 encoding:
+import base64
+
+# To work with date and time:
+import datetime
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class MailSyncModel(BaseModel):
+
+    # For MongoDB:
+    AUTH_COLLECTION = "_authTokens"
+    MAIL_COLLECTION = "_messages"
+
+    # For AI Magic through LLMs:
+    PROMPT_TEMPLATE = [
+        {
+            "role": "system",
+            "content": (
+                "You're a mail summary expert that summarizes mails in 150 chars or less. "
+                "If available, show login info like username and OTPs in your summary."
+                "If no login info is provided, please don't worry; just summarize what you see."
+            )
+        }
+    ]
+
+    # ┏┓     ┓
+    # ┣┫╋╋┏┓┏┣┓┏┳┓┏┓┏┓╋┏
+    # ┛┗┗┗┗┻┗┛┗┛┗┗┗ ┛┗┗┛
+
+    @staticmethod
+    async def __save_one_attachment(
+            session_token: str,
+            attachment: Dict[str, Any],
+            attachment_tags: List[str],
+            attachment_metadata: dict,
+            retry_count: int = 1,
+            retry_delay: int = 1,
+            backoff_multiplier: float = 1.1
+    ) -> Dict[str, Any]:
+
+        """
+        Saves one attachment and generates a URL that can be later used to retrieve it.
+        :param session_token: The session token of the uer who is trying to upload this file.
+        :param attachment: The JSON that describes the attachment.
+        :param attachment_tags: Any tags to put on the file for easy search later.
+        :param attachment_metadata: Any metadata to put on the file for easy search later.
+        :param retry_count: How many max. retries to do in case of failure.
+        :param retry_delay: The interval between the delays.
+        :param backoff_multiplier: By what rate the delay between 2 attempts must change.
+        :return: The JSON that describes the same attachment, except that the payload's data is replaced by the id and
+            url of where to find the attachment.
+        """
+
+        # Make a deep-copy of the attachment JSON,
+        # and process the payload in advance:
+        attachment_copy = copy.deepcopy(attachment)
+        attachment_payload = attachment_copy.pop("payload").encode()
+        if attachment_copy.pop("contentTransferEncoding", "?").strip().lower() == "base64":
+            attachment_payload = base64.b64decode(attachment_payload)
+
+        # Start by assuming failure,
+        # and retry as many times as asked:
+        attachment_copy["id"] = None
+        attachment_copy["url"] = None
+        for _ in range(retry_count):
+
+            # Make the upload:
+            api_response = await current_app.http_client.post(
+                url = current_app.script_data["fileUpload"]["url"],
+                headers = {
+                    "X-Session-Token": session_token,
+                    "X-File-Name": attachment["filename"],
+                    "X-File-Private": "false",
+                    "X-File-Tags": json.to_string(attachment_tags, no_space = True),
+                    "X-File-Metadata": json.to_string(attachment_metadata, no_space = True)
+                },
+                data = attachment_payload
+            )
+
+            # If the upload was successful:
+            if api_response.status_code in [200]:
+                api_data = api_response.json()["data"]
+                attachment_copy["id"] = api_data["id"]
+                attachment_copy["url"] = api_data["url"]
+                break
+
+        # Done here:
+        return attachment_copy
+
+    async def __save_many_attachments(
+            self,
+            session_token: str,
+            attachments: List[Dict[str, Any]],
+            attachment_tags: List[str],
+            attachment_metadata: dict,
+            retry_count: int = 1,
+            retry_delay:int = 1,
+            backoff_multiplier: float = 1.1
+    ) -> List[Dict[str, Any]]:
+
+        """
+        Saves all the attachments received in the mail (whether inline or otherwise) and makes them available through
+        simple download URLs.
+        :param session_token: The session token of the uer who is trying to upload this file.
+        :param attachments: The JSON that describes the attachments.
+        :param attachment_tags: Any tags to put on the file for easy search later.
+        :param attachment_metadata: Any metadata to put on the file for easy search later.
+        :param retry_count: How many max. retries to do in case of failure.
+        :param retry_delay: The interval between the delays.
+        :param backoff_multiplier: By what rate the delay between 2 attempts must change.
+        :return: The JSON that describes the same attachments, except that the payload's data is replaced by the id and
+            url of where to find each attachment.
+        """
+
+        # Create and fire all the tasks
+        # needed to save the files:
+        tasks = [
+            self.__save_one_attachment(
+                session_token = session_token,
+                attachment = attachment,
+                attachment_tags = attachment_tags,
+                attachment_metadata = attachment_metadata,
+                retry_count = retry_count,
+                retry_delay = retry_delay,
+                backoff_multiplier = backoff_multiplier
+            ) for attachment in attachments
+        ]
+        uploaded_attachments = await asyncio.gather(*tasks)
+
+        # Done here:
+        return uploaded_attachments
+
+    # ┏┓      ┏┓┳┳┓  •┓
+    # ┣ ┏┓┏┓  ┃┓┃┃┃┏┓┓┃
+    # ┻ ┗┛┛   ┗┛┛ ┗┗┻┗┗
+
+    async def __sync_one_gmail(
+            self,
+            session_token: str,
+            user_info: dict,
+            mongo_conn: AsyncMongo,
+            mail_client: AsyncGMailClient,
+            tokens: GoogleAuthTokens,
+            message_id: str,
+            llm: LLMOpenAI = None,
+            force_sync: bool = False
+    ) -> MailSyncOneResult:
+
+        """
+        Sync on mail from GMail.
+        :param session_token: The session token of the uer who is trying to upload this file.
+        :param user_info: The information of the user (derived from his session token).
+        :param mongo_conn: The instance of the connection to the database to use.
+        :param mail_client: The instance of the mail client to use to perform the action.
+        :param tokens: The tokens to use to fetch the mails.
+        :param message_id: The id that Google uses to identify this mail. This will be received in the 'list_messages'
+            method.
+        :param llm: The instance of the LLM to use to summarize the mail's content.
+        :param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
+            database.
+        :return:
+        """
+
+        # Start by assuming failure:
+        sync_result = MailSyncOneResult()
+
+        # If we've not been forced to re-sync the mail message,
+        # we first check if the mail already exists in our database:
+        if not force_sync:
+            mail_record = await mongo_conn.find_one(
+                collection = self.MAIL_COLLECTION,
+                filter = mongo_conn.dict_to_dot_notation({
+                    "payload": {
+                        "messageId": message_id
+                    },
+                    "user_info": {
+                        "entityId": user_info["entityId"],
+                        "billingAccountId": user_info["billingAccountId"]
+                    }
+                }),
+                projection = {
+                    "_id": False,
+                    "readTs": "payload.readTs"
+                }
+            )
+            if mail_record:
+                sync_result.success = True
+                sync_result.message = f"gmail message '{message_id}' already sync'd on '{mail_record['readTs']} (UTC)'"
+                return sync_result
+
+        # Now that we know that we have to fetch the mail from GMail:
+        client_response = await mail_client.get_message(
+            tokens = tokens,
+            message_id = message_id,
+            return_raw = False
+        )
+
+        # If we didn't get the mail from GMail;
+        if not client_response.success:
+            sync_result.message = f"gmail (messageId: '{message_id}'): {client_response.message}"
+            return sync_result
+
+        # We upload the attachments:
+        client_response.data["attachments"] = await self.__save_many_attachments(
+            session_token = session_token,
+            attachments = client_response.data["attachments"],
+            attachment_tags = [
+                "email",
+                "gmail",
+                client_response.data["from"][0]["name"],
+                client_response.data["from"][0]["email"],
+                tokens.email,
+            ],
+            attachment_metadata = {
+                "project": "tcaoff",
+                "serviceType": "email",
+                "client": "gmail",
+                "from": client_response.data["from"][0]["email"],
+                "to": tokens.email
+            },
+            retry_count = 3
+        )
+
+        # Give a quick indicator of whether this mail is an inbox mail or sent mail:
+        all_recipients = []
+        for field in ["to", "cc", "bcc"]: all_recipients += [item["email"] for item in client_response.data[field]]
+        if tokens.email in all_recipients: client_response.data["isInbox"] = True
+        else: client_response.data["isInbox"] = False
+
+        # If an LLM is given,
+        # we add an AI summary:
+        llm_json = None
+        if llm:
+
+            # Invoke the LLM:
+            llm_response = response = await llm.invoke(
+                mongo_conn = mongo_conn,
+                user_info = user_info,
+                llm_input = LLMInput(
+                    messages = self.PROMPT_TEMPLATE + [
+                        {
+                            "role": "human",
+                            "content": f"Please summarize this mail: \"\"\"{client_response.data['unformattedText']}\"\"\""
+                        }
+                    ]
+                )
+            )
+
+            # Format the response:
+            llm_json = {
+                "ts": llm_response.ts,
+                "snippet": llm_response.output,
+                "tokens": llm_response.tokens.model_dump()
+            }
+
+        # Add the LLM's response to the main data:
+        client_response.data["aiSnippet"] = llm_json
+
+        # Done here:
+        sync_result.success = True
+        sync_result.mailMessage = client_response.data
+        return sync_result
+
+    async def __sync_many_gmail(
+            self,
+            session_token: str,
+            user_info: dict,
+            mongo_conn: AsyncMongo,
+            token_id: ObjectId,
+            mail_client: AsyncGMailClient,
+            tokens: GoogleAuthTokens,
+            llm: LLMOpenAI = None,
+            force_sync: bool = False,
+            start_date: datetime.datetime = None,
+            end_date: datetime.datetime = None,
+            max_count: int = 100
+    ) -> MailSyncManyResults:
+
+        """
+        Sync many mails from GMail in one shot.
+        :param session_token: The session token of the uer who is trying to upload this file.
+        :param user_info: The information of the user (derived from his session token).
+        :param mongo_conn: The instance of the connection to the database to use.
+        :param token_id: The id of the document in the database that holds the tokens to access the account.
+            Needed only for refreshing the tokens and saving them.
+        :param mail_client: The instance of the mail client to use to perform the action.
+        :param tokens: The tokens to use to fetch the mails.
+        :param llm: The instance of the LLM to use to summarize the mail's content.
+        :param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
+            database.
+        :param start_date: The starting date (inclusive) from when to sync the mails.
+        :param end_date: The ending date (inclusive) from when to sync the mails.
+        :param max_count: The max. no. of mails to sync.
+        :return: The result of the sync'ing.
+        """
+
+        # Start by assuming failure:
+        sync_results = MailSyncManyResults()
+
+        # Refresh the tokens (if needed):
+        tokens_refreshed = await tokens.arefresh(
+            http_client = current_app.http_client,
+            client_id = mail_client.client_id,
+            client_secret = mail_client.client_secret
+        )
+        if tokens_refreshed: await current_app.mail_oauth_model.set_token(
+            db_conn = current_app.sql_writer,
+            mongo_conn = mongo_conn,
+            token_id = token_id,
+            client_user_id = tokens.client_user_id,
+            token = tokens,
+            session_token = session_token
+        )
+
+        # Let's build the query:
+        sub_queries = []
+        if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
+        if end_date: sub_queries.append((end_date + datetime.timedelta(days = 1)).strftime("before:%Y/%m/%d"))
+        query_string = " ".join(sub_queries)
+
+        # Let's enlist all the mails that fall in the date range:
+        client_response = await mail_client.list_messages(
+            tokens = tokens,
+            max_count = max_count,
+            query = query_string
+        )
+        if not client_response.success:
+            sync_results["message"] = f"gmail: {client_response.message}"
+            return sync_results
+        messages_list = client_response.data["messages"]
+
+        # Now, for every mail in the list, we fetch the mail and note the results:
+        tasks = [
+            self.__sync_one_gmail(
+                session_token = session_token,
+                user_info = user_info,
+                mongo_conn = mongo_conn,
+                mail_client = mail_client,
+                tokens = tokens,
+                message_id = v["id"],
+                llm = llm,
+                force_sync = force_sync
+            ) for v in messages_list.values()
+        ]
+        individual_sync_results = await asyncio.gather(*tasks)
+
+        # Now we create operations for each mail,
+        # and maintain success/failure counters:
+        sync_results.totalCount = len(individual_sync_results)
+        mongo_operations = []
+        for result in individual_sync_results:
+            if result.success: sync_results.successCount += 1
+            else: sync_results.failureCount += 1
+            if result.mailMessage: mongo_operations.append(ReplaceOne(
+                filter = {
+                    "serviceType": "email",
+                    "$or": [
+                        {
+                            "client": "gmail",
+                            "payload.messageId": result.mailMessage["messageId"]
+                        }
+                    ]
+                },
+                replacement = {
+                    "version": "1.0.0",
+                    "tokenId": ObjectId(token_id),
+                    "serviceType": "email",
+                    "client": "gmail",
+                    "payload": result.mailMessage
+                },
+                upsert = True
+            ))
+
+        # Make the bulk write:
+        if mongo_operations:
+            mongo_count = await mongo_conn.bulk_write(
+                collection = self.MAIL_COLLECTION,
+                requests = mongo_operations
+            )
+
+        # Apply the labels to the read messages:
+        try:
+            client_response = await mail_client.modify_messages(
+                tokens = tokens,
+                message_ids = [v["id"] for v in messages_list.values()],
+                add_label_ids = [tokens.labels.get("TCAOFF", {}).get("id")]
+            )
+        except Exception as exception:
+            self._printer(exception)
+
+        # Done here:
+        sync_results.message = f"{sync_results.successCount}/{sync_results.totalCount} mail(s) sync'd from gmail"
+        return sync_results
+
+    # ┳┓
+    # ┣┫┏┓┓┏╋┏┓┏┓
+    # ┛┗┗┛┗┻┗┗ ┛
+
+    async def sync(
+            self,
+            session_token: str,
+            user_info: dict,
+            mongo_conn: AsyncMongo,
+            token_id: ObjectId,
+            llm: LLMOpenAI = None,
+            force_sync: bool = False,
+            start_date: datetime.datetime = None,
+            end_date: datetime.datetime = None,
+            max_count: int = 100
+    ) -> MailSyncManyResults:
+
+        """
+        Sync many mails at once from many types of clients. Use this as a common entry point after which you internally
+        route the request to the appropriate clients.
+        :param session_token: The session token of the uer who is trying to upload this file.
+        :param user_info: The information of the user (derived from his session token).
+        :param mongo_conn: The instance of the connection to the database to use.
+        :param token_id: The id of the document in the database that holds the tokens to access the account.
+            Needed only for refreshing the tokens and saving them.
+        :param llm: The instance of the LLM to use to summarize the mail's content.
+        :param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
+            database.
+        :param start_date: The starting date (inclusive) from when to sync the mails.
+        :param end_date: The ending date (inclusive) from when to sync the mails.
+        :param max_count: The max. no. of mails to sync.
+        :return: The result of the sync'ing.
+        """
+
+        # Start by assuming failure:
+        sync_results = MailSyncManyResults()
+
+        # ┏┓    ┓   ┏┳┓  ┓
+        # ┣ ┏┓╋┏┣┓   ┃ ┏┓┃┏┏┓┏┓┏
+        # ┻ ┗ ┗┗┛┗   ┻ ┗┛┛┗┗ ┛┗┛
+
+        # We first load the authorization tokens:
+        auth_json = await current_app.mail_oauth_model.get_token(
+            mongo_conn = mongo_conn,
+            token_id = token_id,
+        )
+
+        # If we failed to load the authorization tokens:
+        if not auth_json:
+            sync_results.message = f"no such token id '{token_id}'"
+            return sync_results
+
+        # ┏┓      ┏┓┳┳┓  •┓
+        # ┣ ┏┓┏┓  ┃┓┃┃┃┏┓┓┃
+        # ┻ ┗┛┛   ┗┛┛ ┗┗┻┗┗
+
+        if auth_json["client"] == "gmail":
+            return await self.__sync_many_gmail(
+                session_token = session_token,
+                user_info = user_info,
+                mongo_conn = mongo_conn,
+                token_id = token_id,
+                mail_client = current_app.gmail_client,
+                tokens = GoogleAuthTokens(**auth_json["token"]),
+                llm = llm,
+                force_sync = force_sync,
+                start_date = start_date,
+                end_date = end_date,
+                max_count = max_count
+            )
+
+        # ┳      ┓• ┓  ┏┓┓•
+        # ┃┏┓┓┏┏┓┃┓┏┫  ┃ ┃┓┏┓┏┓╋
+        # ┻┛┗┗┛┗┻┗┗┗┻  ┗┛┗┗┗ ┛┗┗
+
+        # If we haven't been able to sync mail due to not entering any 'if' condition:
+        sync_results.message = f"no such mail client '{auth_json['client']}'"
+        return sync_results
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

models/behaviour/sms/send.py

@@ -6,12 +6,12 @@
 
     DATE:
 
-        Monday, 2nd Dec., 2024
+        ORIGINAL: Thursday, 5th Dec., 2024
+        UPGRADED: Monday, 9th Dec., 2024
 
     OBJECTIVE:
 
-        To define the interaction between the UI layer and the database connectivity in one place. Here we shall handle
+        To work with auth details of SMS clients like Nimbus SMS (India) and Savvy Bulk SMS (Kenya).
-        all the activities for OAuth2.0 authorization requests for all the users of our service.
 
     REFERENCES:
 
@@ -44,6 +44,9 @@ from utils_v2.database.async_mongo_v2 import AsyncMongo
 # Base model:
 from models.behaviour.base import BaseModel
 
+# Data models:
+from models.data.core.auth_token import CoreAuthTokenModel
+
 # To work with MongoDB:
 from bson import ObjectId
 
@@ -91,35 +94,23 @@ import copy
 #   *****************************************************************************************************************
 
 
-class MailOAuthModel(BaseModel):
+class SMSAuthModel(BaseModel):
 
     AUTH_COLLECTION = "_authTokens"
 
-    async def get_token_id(
+    async def set(
             self,
             db_conn: AsyncMySQL,
             mongo_conn: AsyncMongo,
-            user_info: dict,
+            auth_token: CoreAuthTokenModel,
-            client_user_id: dict,
-            auth: dict,
-            service_client: Literal["gmail"],
-            auth_type: Literal["oauth"],
-            sync_freq: Literal[60, 300, 900] = 300,
             session_token: str = None
-    ) -> ObjectId:
+    ) -> ObjectId | None:
 
         """
-        Stores params from the session info and gives an identifier to use in the authorization URL. Use this when the
+        To store auth/tokens for a particular service to the database.
-        user requests an authorization URL to link your service to another service (like GMail).
         :param db_conn: The database connection (MariaDB) to use to perform the action.
         :param mongo_conn: The database connection (MongoDB) to use to perform the action.
-        :param user_info: The dictionary that has the user's session information.
+        :param auth_token: An instance of the core auth-token model that holds data in the database.
-        :param client_user_id: The way the third-party client recognizes your user.
-        :param auth: The authentication details of the account.
-        :param service_client: The name of the company or brand that is providing this service that is being integrated.
-        :param auth_type: To identify the type of authentication being done here. This could indicate simple password
-            authentication, more advance OAuth2.0 authentication, etc.
-        :param sync_freq: The time interval in which mails need to be sync'd. Specify this in seconds.
         :param session_token: The session token of the user who requested this service.
         :return: An ObjectId to later store the granted tokens.
         """
@@ -128,34 +119,35 @@ class MailOAuthModel(BaseModel):
         request_ts = date_time.get_current_utc_date_time(as_string = False)
 
         # Get the identifier from the database:
+        # BE CAREFUL WITH THE KEYS HERE, THEY SHOULD MATCH THE FIELDS OF THE CORE AUTH-TOKEN MODEL:
         mongo_json = await mongo_conn.find_one_and_update(
-            collection = MailOAuthModel.AUTH_COLLECTION,
+            collection = self.AUTH_COLLECTION,
             filter = mongo_conn.dict_to_dot_notation({
-                "serviceType": "email",
+                "serviceType": auth_token.serviceType,
                 "user": {
-                    "entityId": user_info["entityId"],
+                    "entityId": auth_token.user.entityId,
-                    "billingAccountId": user_info["billingAccountId"]
+                    "billingAccountId": auth_token.user.billingAccountId
                 },
-                "clientUserId": client_user_id
+                "clientUserId": auth_token.clientUserId
             }),
             update = {
                 "$set": {
-                    "lastRequestTs": request_ts,
+                    "lastRequestTs": auth_token.lastRequestTs,
-                    "status": "active",
+                    "status": auth_token.status,
-                    "syncFreq": max(sync_freq, 60)
+                    "syncFreq": auth_token.syncFreq
                 },
                 "$setOnInsert": {
-                    "version": "1.1.1",
+                    "version": auth_token.version,
-                    "serviceType": "email",
+                    "serviceType": auth_token.serviceType,
-                    "client": service_client,
+                    "client": auth_token.client,
-                    "authType": auth_type,
+                    "authType": auth_token.authType,
-                    "user": user_info,
+                    "user": auth_token.user.model_dump(),
-                    "clientUserId": client_user_id,
+                    "clientUserId": auth_token.clientUserId,
-                    "auth": auth,
+                    "auth": auth_token.auth,
-                    "token": None,
+                    "token": auth_token.token,
-                    "firstRefreshTs": None,
+                    "firstRefreshTs": auth_token.firstRefreshTs,
-                    "lastRefreshTs": None,
+                    "lastRefreshTs": auth_token.lastRefreshTs,
-                    "firstRequestTs": request_ts,
+                    "firstRequestTs": auth_token.firstRequestTs or request_ts
                 }
             },
             projection = {
@@ -168,19 +160,20 @@ class MailOAuthModel(BaseModel):
         # Tell MariaDB that an authorization request was initiated:
         db_json = {}
         if mongo_json is not None:
+            token_notes = auth_token.clientUserId
             db_json = await self.call_procedure(
                 db_conn = db_conn,
                 proc_name = "entity_integration_save",
                 proc_args = (
-                    user_info["entityId"],  # ............................................ 'p_entity_id'
+                    auth_token.user.entityId,  # ..................................... 'p_entity_id'
-                    service_client,  # ................................................... 'p_provider'
+                    auth_token.client,  # ............................................ 'p_provider'
-                    "Pending",  # ........................................................ 'p_current_status'
+                    auth_token.status,  # ............................................ 'p_current_status'
-                    "Auth Requested",  # ................................................. 'p_last_action'
+                    "Auth Details Accepted",  # ...................................... 'p_last_action'
-                    None,  # ............................................................. 'p_display_name'
+                    None,  # ......................................................... 'p_display_name'
-                    None,  # ............................................................. 'p_display_picture'
+                    None,  # ......................................................... 'p_display_picture'
-                    str(mongo_json["_id"]),  # ........................................... 'p_token_id'
+                    str(mongo_json["_id"]),  # ....................................... 'p_token_id'
-                    json.to_string(python_data = {"email": None}, no_space = True),  # ... 'p_notes'
+                    json.to_string(python_data = token_notes, no_space = True),  # ... 'p_notes'
-                    user_info["userId"]  # ............................................... 'p_created_by'
+                    auth_token.user.userId  # ........................................ 'p_created_by'
                 ),
                 session_token = session_token
             )
@@ -188,96 +181,7 @@ class MailOAuthModel(BaseModel):
         # Done here:
         return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
 
-    async def set_token(
+    async def get(
-            self,
-            db_conn: AsyncMySQL,
-            mongo_conn: AsyncMongo,
-            token_id: ObjectId | str,
-            client_user_id: dict,
-            token: dict,
-            session_token: str = None
-    ) -> bool:
-
-        """
-        This method is to be called when the end user authorizes your service to connect to his third-party account. For
-        example, when the end user allows you to access his GMail account. USE THIS FOR UPDATING (REFRESHING) TOKENS
-        ALSO.
-        :param db_conn: The database connection (MariaDB) to use to perform the action.
-        :param mongo_conn: The database connection (MongoDB) to use to perform the action.
-        :param token_id: The identifier granted by the 'get_token_id' method.
-        :param client_user_id: The way the third-party client recognizes your user. These details should match the
-            details furnished while requesting the authorization through 'get_token_id' method.
-        :param token: The token granted by the third-party service.
-        :param session_token: The session token of the user who requested this service.
-        :return: True if saved, False if failed.
-        """
-
-        # Start by assuming failure:
-        token_saved = False
-
-        # Note down the timestamp at which this event occurred:
-        request_ts = date_time.get_current_utc_date_time(as_string = False)
-
-        # Save the token to MongoDB:
-        mongo_json = await mongo_conn.find_one_and_update(
-            collection = MailOAuthModel.AUTH_COLLECTION,
-            filter = mongo_conn.dict_to_dot_notation({
-                "_id": ObjectId(token_id),
-                "clientUserId": client_user_id
-            }),
-            update = [{
-                "$set": {
-                    "token": token,
-                    "status": "active",
-                    "lastRefreshTs": request_ts,
-                    "firstRefreshTs": {
-                        "$cond": {
-                            "if": {
-                                "$or": [
-                                    {"$eq": ["$firstRefreshTs", None]},
-                                    {"$eq": [{"$type": "$firstRefreshTs"}, "missing"]}
-                                ]
-                            },
-                            "then": request_ts,
-                            "else": "$firstRefreshTs"
-                        }
-                    }
-                }
-            }],
-            projection = {"token": False},
-            return_updated = True,
-            upsert = False
-        )
-
-        # Tell MariaDB that the token was saved:
-        if mongo_json is not None:
-            token_notes = {
-                "email": token["email"],
-                "displayName": token.get("displayName"),
-                "displayPictureUrl": token.get("displayPictureUrl"),
-            }
-            db_json = await self.call_procedure(
-                db_conn = db_conn,
-                proc_name = "entity_integration_save",
-                proc_args = (
-                    mongo_json["user"]["entityId"],  # ............................... 'p_entity_id'
-                    mongo_json["client"],  # ......................................... 'p_provider'
-                    "Active",  # ..................................................... 'p_current_status'
-                    "Auth Granted",  # ............................................... 'p_last_action'
-                    token["displayName"],  # ......................................... 'p_display_name'
-                    token["displayPictureUrl"],  # ................................... 'p_display_picture'
-                    token_id,  # ..................................................... 'p_token_id'
-                    json.to_string(python_data = token_notes, no_space = True),  # ... 'p_notes'
-                    mongo_json["user"]["userId"]  # .................................. 'p_created_by'
-                ),
-                session_token = session_token
-            )
-            if db_json["status"] == 1: token_saved = True
-
-        # Done here:
-        return token_saved
-
-    async def get_token(
             self,
             mongo_conn: AsyncMongo,
             token_id: ObjectId | str = None,
@@ -285,9 +189,9 @@ class MailOAuthModel(BaseModel):
     ) -> dict | None:
 
         """
-        To retrieve stored tokens from the database.
+        To retrieve stored auth/tokens from the database.
         :param mongo_conn: The database connection (MongoDB) to use to perform the action.
-        :param token_id: The identifier granted by the 'get_token_id' method.
+        :param token_id: The identifier granted providing auth details for the first time in 'set_token'.
         :param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
             info, the client, the type of authentication used, or even the kind of service.
         :return: The retrieved record that has the token, and information about the service and client if found, else
@@ -301,21 +205,15 @@ class MailOAuthModel(BaseModel):
         # If there is no search criteria, we exit with failure:
         if not filter_json: return None
 
-        # If there is some filtering possible,
+        # If there is some filtering possible, we fetch the token:
-        # we fetch and return the token:
+        token = await mongo_conn.find_one(
-        return await mongo_conn.find_one(
             collection = self.AUTH_COLLECTION,
             filter = filter_json,
-            projection = {
-                "_id": True,
-                "serviceType": True,
-                "authType": True,
-                "client": True,
-                "clientUserId": True,
-                "token": True
-            }
         )
 
+        # Done here:
+        return CoreAuthTokenModel(**token) if token else None
+
 
 #   *****************************************************************************************************************
 #  *****                                                                                                          ****

models/data/api/sms/send.py

@@ -0,0 +1,196 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 5th Dec., 2024.
+
+    OBJECTIVE:
+
+        To provide a structure to receive auth details of various SMS providers.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For making data behaviour_models:
+from pydantic import BaseModel, Field, field_validator, PastDatetime
+from typing import Optional, Literal, Union
+
+# My utils:
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# To work with date and time:
+import datetime
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# RegEx Patterns:
+REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NimbusSMSIndiaAuth(BaseModel):
+
+    entityId: str = Field(
+        description = "the entity id as registered with DLT",
+        min_length = 1,
+        frozen = True
+    )
+
+    senderId: str = Field(
+        description = "the 6-char code that you see in your SMS inbox",
+        min_length = 1,
+        frozen = True,
+        examples = ["HDFCBK", "NSESMS", "ZRODHA"]
+    )
+
+    userId: str = Field(
+        description = "the 6-digit id that Nimbus has assigned to you",
+        min_length = 1,
+        frozen = True
+    )
+
+    apiKey: str = Field(
+        description = "the key generated through Nimbus's portal",
+        min_length = 1,
+        frozen = True
+    )
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "forbid"
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class SavvyBulkSMSKenyaAuth(BaseModel):
+
+    apiKey: str = Field(
+        description = "the key generated through Savvy's portal",
+        min_length = 1,
+        frozen = True
+    )
+
+    partnerId: str = Field(
+        description = "the key generated through Savvy's portal",
+        min_length = 1,
+        frozen = True
+    )
+
+    shortCode: str = Field(
+        description = "your short code with Savvy",
+        min_length = 1,
+        frozen = True
+    )
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "forbid"
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class SMSAuthRequestHeaders(BaseModel):
+
+    sessionToken: str = Field(
+        description = "the session token of the user who is requesting the service",
+        pattern = REGEX_SESSION_TOKEN,
+        frozen = True,
+        alias = "X-Session-Token"
+    )
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "allow"
+
+    def model_dump(self, *args, **kwargs):
+        return super().model_dump(*args, by_alias = True, **kwargs)
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+class SMSAuthRequestData(BaseModel):
+
+    smsClient: Literal["nimbusSmsIndia", "savvyBulkSmsKenya"] = Field(alias = "client")
+    auth: Union[NimbusSMSIndiaAuth, SavvyBulkSMSKenyaAuth]
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "forbid"
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

utils_v2/api/async_quart.py

@@ -606,10 +606,10 @@ def get_session_info(
                     if hasattr(current_app, "printer"): getattr(current_app, "printer")(exception)
 
             # We note down whatever we got:
+            session_info = summarize_variable(session_info, expand = True, sensitive_keys = sensitive_keys)
             if session_info and isinstance(get, str):
                 for subkey in get.split("."):
                     session_info = session_info.get(subkey, {}) if isinstance(session_info, dict) else {}
-            session_info = summarize_variable(session_info, expand = True, sensitive_keys = sensitive_keys)
             kwargs["session_info"] = session_info
 
             # If no session info was found, but it was mandatory:

utils_v2/mail/mail_parser.py

@@ -55,8 +55,9 @@ from bs4 import BeautifulSoup
 # To work with datatypes:
 from typing import Any, Dict
 
-# To work with base-64 encoding:
+# To work with various encodings:
 import base64
+import quopri
 
 
 #   *****************************************************************************************************************
@@ -86,6 +87,51 @@ import base64
 #   *****************************************************************************************************************
 
 
+def from_quoted_printable(text: str) -> str:
+
+    decoded_text = ""
+    decoded_bytes = quopri.decodestring(text)
+    for encoding in ["utf-8", "utf-16", "utf-32", "latin1"]:
+        try: text = decoded_bytes.decode(encoding)
+        except: text = ""
+        if text.find("From") >= 0:
+            decoded_text = text
+            break
+    return decoded_text
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
+def find_in_raw_mail(
+        raw_mail: str,
+        text: str
+) -> int:
+
+    # We first treat it as un-encoded text:
+    offset = raw_mail.find(text)
+    if offset >= 0: return offset
+
+    # Then we try Base64 encoding:
+    offset = raw_mail.find(base64.b64encode(text.encode("utf-8")).decode("utf-8"))
+    if offset >= 0: return offset
+
+    # then we try Quoted-Printable encoding:
+    offset = from_quoted_printable(raw_mail).find(text)
+    print("MAIL:")
+    print(raw_mail)
+    print("\n\n\n---\n\n\n")
+    print("TEXT:")
+    print(quopri.encodestring(text.encode("utf-8")).decode("utf-8"))
+    if offset >= 0: return offset
+
+    # Done here, even if nothing worked:
+    return offset
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+
+
 def parse(raw_mail: str | bytes) -> Dict[str, Any]:
 
     """
@@ -102,6 +148,9 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
     if isinstance(raw_mail, str): parsed_mail = mailparser.parse_from_string(raw_mail)
     else: parsed_mail = mailparser.parse_from_bytes(raw_mail)
 
+    # print(parsed_mail.mail_json)
+    # return
+
     # Format the attachments:
     message_attachments = [
         {
@@ -117,46 +166,43 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
         } for attachment in parsed_mail.attachments
     ]
 
-    # Figure out which entity (text and HTML) came in which sequence.
+    # print("PRINTING PARTS")
-    # The library doesn't give us any sequence info so we do some custom string processing here to figure out the order
+    # print("LIBRARY PARTS:", type(parsed_mail))
-    # in which to render the contents of the page.
+
-    parts = [
+    # # Figure out which entity (text and HTML) came in which sequence.
-        {
+    # # The library doesn't give us any sequence info so we do some custom string processing here to figure out the order
-            "partNo": None,
+    # # in which to render the contents of the page.
-            "offset": max(
+    # parts = [
-                parsed_mail.message_as_string.find(t),
+    #     # {
-                parsed_mail.message_as_string.find(base64.b64encode(t.encode()).decode())
+    #     #     "partNo": None,
-            ),
+    #     #     "offset": max(
-            "type": "text/plain",
+    #     #         parsed_mail.message_as_string.find(t),
-            "data": t
+    #     #         parsed_mail.message_as_string.find(base64.b64encode(t.encode()).decode())
-        } for t in parsed_mail.text_plain
+    #     #     ),
-    ]
+    #     #     "type": "text/plain",
-    parts = parts + [
+    #     #     "data": t
-        {
+    #     # } for t in parsed_mail.text_plain
-            "partNo": None,
+    # ]
-            "offset": max(
+    # parts = parts + [
-                parsed_mail.message_as_string.find(h),
+    #     {
-                parsed_mail.message_as_string.find(base64.b64encode(h.encode()).decode())
+    #         "partNo": None,
-            ),
+    #         "offset": find_in_raw_mail(raw_mail = parsed_mail.message_as_string, text = h),
-            "type": "text/html",
+    #         "type": "text/html",
-            "data": h
+    #         "data": h
-        } for h in parsed_mail.text_html
+    #     } for h in parsed_mail.text_html
-    ]
+    # ]
-    parts = sorted(parts, key = lambda x: x["offset"])
+    # parts = sorted(parts, key = lambda x: x["offset"])
-    for i, p in enumerate(parts): p["partNo"] = i
+    # for i, p in enumerate(parts): p["partNo"] = i
 
     # Get the unformatted text from everything in the mail:
     unformatted_text = []
-    for p in parts:
+    for p in parsed_mail.text_html:
-        if p["type"] == "text/html":
+        html_parser = BeautifulSoup(p, "html.parser")
-            html_parser = BeautifulSoup(p["data"], "html.parser")
         unformatted_text.append(html_parser.get_text())
-        else: unformatted_text.append(p["data"])
 
     # Put everything together:
     return {
         "ts": date_time.to_timezone(parsed_mail.date, timezone = date_time.TIMEZONE_UTC),
-        "readTs": date_time.get_current_utc_date_time(as_string = False),
         "headers": parsed_mail.headers,
         "from": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers["From"]],
         "to": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers["To"]],
@@ -165,7 +211,7 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
         "subject": parsed_mail.headers["Subject"],
         "text": parsed_mail.text_plain,
         "html": parsed_mail.text_html,
-        "parts": parts,
+        # "parts": parts,
         "unformattedText": "\n".join(unformatted_text),
         "attachments": message_attachments,
         "isInbox": None
@@ -181,4 +227,9 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
 
 if __name__ == "__main__":
 
-    pass
+    from utils_v2.system import files
+
+    mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail.txt")
+    parse_results = parse(mail_string_raw)
+
+    print(json.to_string(parse_results, default = str))

utils_v2/sms/models/behaviour/nimbus/async_nimbus.py

@@ -43,6 +43,9 @@ sys.path.append("..")
 # To make API Calls:
 import httpx
 
+# Data models:
+from utils_v2.sms.models.data.sms_message import SentSMSMessageModel
+
 # For debugging:
 from icecream import IceCreamDebugger
 
@@ -172,12 +175,12 @@ class AsyncNimbusSMS:
 
     async def send_sms(
             self,
-            template_id,
+            template_id: str,
-            recipient_number,
+            recipient_number: str,
-            message,
+            message: str,
-            message_type = MESSAGE_TYPE_REGULAR,
+            message_type: int = MESSAGE_TYPE_REGULAR,
-            flash = MESSAGE_TYPE_NOT_FLASH
+            flash: int = MESSAGE_TYPE_NOT_FLASH
-    ):
+    ) -> SentSMSMessageModel:
 
         """
         Sends one SMS through Nimbus IT's system. The text of the message must match the template that had been
@@ -195,17 +198,14 @@ class AsyncNimbusSMS:
         """
 
         # Construct the basic structure of the response of this method:
-        summary = {
+        summary = SentSMSMessageModel(
-            "success": False,
+            sender = {"senderId": self.__sender_id},
-            "info": None,
+            recipient = {"recipientNo": recipient_number},
-            "sender": self.__sender_id,
+            text = message,
-            "recipient": recipient_number,
+            length = len(message),
-            "message": message,
+            isFlash = True if flash else False,
-            "length": len(message),
+            metadata = {"templateId": template_id}
-            "template_id": template_id,
+        )
-            "raw": None,
-            "isFlash": True if flash else False
-        }
 
         try:
 
@@ -231,18 +231,13 @@ class AsyncNimbusSMS:
             # For a successful API call:
             if response.status_code == 200:
                 response_json = response.json()
-                summary["success"] = True if response_json.get("STATUS", "ERROR").lower() in ["ok"] else False
+                summary.rawResponse = response_json
-                summary["info"] = response_json.get("RESPONSE", {}).get("INFO")
+                summary.success = True if response_json.get("STATUS", "ERROR").lower() in ["ok"] else False
-                summary["raw"] = {
+                if summary.success: summary.messageId = response_json.get("RESPONSE", {}).get("UID")
-                    "http_code": response.status_code,
+                else: summary.brief = response_json.get("RESPONSE", {}).get("INFO")
-                    "response": response_json,
-                }
 
             # For any other code that indicates some form of failure:
-            else: summary["raw"] = {
+            else: summary.rawResponse = response.content.decode()
-                "http_code": response.status_code,
-                "response": response.content.decode()
-            }
 
         except Exception as exception:
             self.__printer(exception)
@@ -263,21 +258,21 @@ if __name__ == "__main__":
 
     async def main():
 
-        sender = AsyncNimbusSMS(
+        client = AsyncNimbusSMS(
-            entity_id = "<your_entity_id>",
+            entity_id = "1701172465456946915",
-            sender_id = "<your_sender_id>",
+            sender_id = "TCAOFF",
-            user_id = "<your_nimbus_user_id>",
+            user_id = "210844",
-            api_key = "<your_nimbus_api_key>"
+            api_key = "92UnZwiiY7zps"
         )
 
-        response = await sender.send_sms(
+        response = await client.send_sms(
-            template_id = "<your_sms_template_id>",
+            template_id = "1707172474709021546",
-            recipient_number = "<the_number_you_want_to_send_the_message_to>",
+            recipient_number = "987039115511",
-            message = "<your_sms_message>"
+            message = "OTP for The CA Office registration request is 583920. Please enter this to verify your identity and proceed with the registration request. - TCAOFF"
         )
         print("SMS API RESPONSE:", response)
 
-        my_balance = await sender.get_balance()
+        my_balance = await client.get_balance()
         print("REMAINING BALANCE:", my_balance)
 
 

utils_v2/sms/models/behaviour/savvy_bulk_sms/async_savvy_bulk_sms.py

@@ -42,6 +42,9 @@ sys.path.append("..")
 # To make API Calls:
 import httpx
 
+# Data models:
+from utils_v2.sms.models.data.sms_message import SentSMSMessageModel
+
 # For debugging:
 from icecream import IceCreamDebugger
 
@@ -149,18 +152,19 @@ class AsyncSavvyBulkSMS:
         """
 
         # Construct the basic structure of the response of this method:
-        summary = {
+        summary = SentSMSMessageModel(
-            "success": False,
+            sender = {
-            "info": None,
-            "sender": {
                 "partnerId": self.__partner_id,
                 "shortCode": self.__short_code
             },
-            "recipient": recipient_number,
+            recipient = {
-            "message": message,
+                "recipientNo": recipient_number
-            "length": len(message),
+            },
-            "raw": None,
+            text = message,
-        }
+            length = len(message),
+            isFlash = False,
+            metadata = None
+        )
 
         try:
 
@@ -179,18 +183,15 @@ class AsyncSavvyBulkSMS:
             # For a successful API call:
             if api_response.status_code in [200]:
                 api_json = api_response.json()
-                first_desc = api_json.get("responses", [{}])[0].get("response-description", "N/A").lower().strip()
+                summary.rawResponse = api_json
-                summary["success"] = True if first_desc == "success" else False
+                first_response = api_json.get("responses", [{}])[0]
-                summary["raw"] = {
+                first_desc = first_response.get("response-description", "N/A").lower().strip()
-                    "http_code": api_response.status_code,
+                summary.success = True if first_desc == "success" else False
-                    "response": api_json,
+                if summary.success: summary.messageId = first_response.get("messageid")
-                }
+                else: summary.brief = first_response.get("response-description")
 
             # For any other code that indicates some form of failure:
-            else: summary["raw"] = {
+            else: summary.rawResponse = api_response.content.decode()
-                "http_code": api_response.status_code,
-                "response": api_response.content.decode()
-            }
 
         # If something goes wrong along the way:
         except Exception as exception:
@@ -214,14 +215,14 @@ if __name__ == "__main__":
     async def main():
 
         sender = AsyncSavvyBulkSMS(
-            api_key = "<your_api_key>",
+            api_key = "19250bdd74050f7cec980c71e75f851a",
-            partner_id = "<your_partner_id>",
+            partner_id = "7462",
-            short_code = "<your_short_code>"
+            short_code = "HTL TV-NET"
         )
 
         response = await sender.send_sms(
-            recipient_number = "<target_recipient>",
+            recipient_number = "254748877373 123",
-            message = "<message_for_recipient>"
+            message = "Test 123"
         )
         print("SMS API RESPONSE:", response)
 

utils_v2/sms/models/data/sms_message.py

@@ -0,0 +1,157 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 9th Dec., 2024.
+
+    OBJECTIVE:
+
+        To define how messages will be stored in the database.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# For making data behaviour_models:
+from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
+from typing import Optional, Literal, Union, Any
+
+# My utils:
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# To work with MongoDB:
+from bson.objectid import ObjectId
+
+# To work with date and time:
+import datetime
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class SentSMSMessageModel(BaseModel):
+
+    ts: AwareDatetime = Field(
+        description = "the time (utc) at which this message was sent by the sender",
+        frozen = True,
+        default_factory = lambda: date_time.get_current_utc_date_time(as_string = False)
+    )
+
+    sender: dict = Field(
+        description = "the sender of this message",
+        frozen = True
+    )
+
+    recipient: dict = Field(
+        description = "the recipient of this message",
+        frozen = True
+    )
+
+    text: str = Field(
+        description = "the actual text that was sent",
+        frozen = True
+    )
+
+    length: int = Field(
+        description = "the no. of chars in this message",
+        frozen = True
+    )
+
+    isFlash: bool = Field(
+        description = "to know whether this message is a flash message or a regular message",
+        frozen = True
+    )
+
+    success: bool = Field(
+        description = "to know whether this message was sent successfully or not",
+        default = False
+    )
+
+    brief: str | None = Field(
+        description = "a brief message about what happened; useful when something goes wrong",
+        default = None
+    )
+
+    metadata: dict | None = Field(
+        description = "any extra data about this message",
+        frozen = True
+    )
+
+    rawResponse: Any | None = Field(
+        description = "the raw response from the third-party client",
+        default = None
+    )
+
+    messageId: int | str | None = Field(
+        description = "how the client recognizes this message",
+        frozen = False,
+        default = None
+    )
+
+    # ┏┓    ┏•
+    # ┃ ┏┓┏┓╋┓┏┓
+    # ┗┛┗┛┛┗┛┗┗┫
+    #          ┛
+
+    class Config:
+        extra = "allow"
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass