(20250116) Day-end push.

controllers/core/ai/llm.py

@@ -128,7 +128,7 @@ class CoreLLMController(BaseModel):
 
     async def invoke(
             self,
-            mongo_conn: AsyncMongo,
+            mongo_data_conn: AsyncMongo,
             user_info: CoreUserInfoModel,
             llm_input: LLMInput
     ) -> LLMOutput:
@@ -158,7 +158,7 @@ class CoreLLMController(BaseModel):
         # Store this into MongoDB:
         mongo_document = {"user": user_info.model_dump()}
         for k, v in llm_response.model_dump().items(): mongo_document[k] = v
-        inserted_id = await mongo_conn.insert_one(
+        inserted_id = await mongo_data_conn.insert_one(
             collection = self.AI_USAGE_COLLECTION,
             document = mongo_document
         )

controllers_v2/message/mail/base.py

@@ -42,6 +42,7 @@ from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
 
 # Controllers:
 from controllers_v2.core.message import CoreMessageController
+from controllers.core.ai.llm import CoreLLMController
 
 # Models:
 from models.core.user import CoreUserInfoModel
@@ -50,7 +51,11 @@ from models.api.message.mail.oauth import (
     OAuthMailAuthorizationRequestHeaders,
     OAuthMailAuthorizationRequestData
 )
-from models.message.mail.oauth import OAuthMailGetAuthorizationURLResponse
+from models.message.mail.oauth import OAuthMailGetAuthorizationURLResponse, OAuthMailHandleCallbackResponse
+from models.core.message import CoreMessageModel
+from models.core.ai.llm import LLMInput, LLMOutput, LLMInputMessage
+from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
+from models.message.mail.send import MailSendOneResult
 
 # Mail Client(s):
 from utils_v2.goog.controllers.gmail.gmail_client import AsyncGMailClient
@@ -61,6 +66,15 @@ from typing import List, Any
 # To make HTTP requests:
 import httpx
 
+# To parse the HTML content in the mail:
+from bs4 import BeautifulSoup
+
+# To work with MongoDB:
+from bson import ObjectId
+
+# To work with date and time:
+import datetime
+
 # To make abstract classes:
 from abc import ABC, abstractmethod
 
@@ -104,6 +118,45 @@ from abc import ABC, abstractmethod
 
 class MailController(CoreMessageController, ABC):
 
+    # ┏┓┓      ┓┏    •  ┓ ┓
+    # ┃ ┃┏┓┏┏  ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
+    # ┗┛┗┗┻┛┛  ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
+
+    # For AI Magic through LLMs:
+    RECEIVED_MAIL_SUMMARIZATION_PROMPT_TEMPLATE = [
+        LLMInputMessage(
+            role = "system",
+            content = (
+                "You're an expert mail summary program. "
+                "Provide the response in a structured JSON format with two fields: \"summary\" and \"senderType\". "
+                "The summary should be 150 chars or less. "
+                "If available, show login info like username and OTPs in your summary."
+                "If no login info is provided, please don't worry; just summarize what you see."
+                "\"senderType\" must be one of [\"Vendor\", \"Client\", null].\n"
+                "Example output 1: "
+                "{\"summary\":\"Sagar Supplies has shipped your materials. They are expected to reach by Thursday. "
+                "Use OTP 346780 when the delivery agent asks.\",\"senderType\":\"Vendor\"}\n"
+                "Example output 2: "
+                "{\"summary\":\"Mr. Mehta is enquiring about the submission of his tax filings, which are to be done "
+                "today.\",\"senderType\":\"Client\"}\n"
+                "Example output 3: "
+                "{\"summary\":\"JustDial's marketing message. They're offering a 35% discount to new accounts.\","
+                "\"senderType\":null}\n"
+                "Remember to respond only with the raw JSON string, nothing else. Good luck :)"
+            )
+        )
+    ]
+    SENT_MAIL_SUMMARIZATION_PROMPT_TEMPLATE = [
+        LLMInputMessage(
+            role = "system",
+            content = (
+                "You're an expert mail summary assistant. That summarizes sent mails in 150 chars or less. "
+                "The objective of your user is to be able to recollect what a mail they sent was about from a brief "
+                "summary. Reply in a simple string, no formatting is allowed except emojis. Good luck :)"
+            )
+        )
+    ]
+
     # ┏┓
     # ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
     # ┗┛┗┛┛┗┛┗┛ ┗┻┗┗┗┛┛
@@ -149,6 +202,68 @@ class MailController(CoreMessageController, ABC):
             debug_only_errors = debug_only_errors
         )
 
+    # ┓┏  ┓
+    # ┣┫┏┓┃┏┓┏┓┏┓┏
+    # ┛┗┗ ┗┣┛┗ ┛ ┛
+    #      ┛
+
+    def extract_plaintext_parts(
+            self,
+            payload: dict
+    ) -> List[str]:
+
+        """
+        A mail's body will have plaintext and HTML parts. This method extracts the plaintext parts if readily available,
+        or tries to convert the HTML parts to plaintext.
+        :param payload: The mail's full payload.
+        :return: An array of plaintext parts.
+        """
+
+        # Start with just a holder:
+        text_parts = []
+
+        # If a direct text/plain part occurs,
+        # we just add it to the list:
+        if (
+                payload["contentMainType"] == "text" and
+                payload["contentSubType"] == "plain"
+        ):
+            text_parts.append(payload["payload"])
+
+        # If a direct text/html part occurs,
+        # we just add it to the list:
+        if (
+                payload["contentMainType"] == "text" and
+                payload["contentSubType"] == "html"
+        ):
+            html_parser = BeautifulSoup(payload["payload"], "html.parser")
+            text_parts.append(html_parser.get_text())
+
+        # If a multipart/alternative part occurs,
+        # we pick just the ready plaintext part:
+        if (
+                payload["contentMainType"] == "multipart" and
+                payload["contentSubType"] == "alternative"
+        ):
+            for part in payload["payload"]:
+                if part["contentSubType"] == "plain":
+                    text_parts.append(part["payload"])
+
+        # If a multipart/mixed or multipart/related part occurs,
+        # we use recursion to look for plaintext parts nested inside:
+        if (
+                payload["contentMainType"] == "multipart" and
+                (
+                        payload["contentSubType"] == "mixed" or
+                        payload["contentSubType"] == "related"
+                )
+        ):
+            for part in payload["payload"]:
+                text_parts += self.extract_plaintext_parts(payload = part)
+
+        # Done here:
+        return text_parts
+
     # ┏┓┏┓   ┓ ┏┓ ┏┓
     # ┃┃┣┫┓┏╋┣┓┏┛ ┃┫
     # ┗┛┛┗┗┻┗┛┗┗━•┗┛
@@ -178,11 +293,70 @@ class MailController(CoreMessageController, ABC):
 
         pass
 
+    @abstractmethod
+    async def handle_authorization_callback(
+            self,
+            sql_conn: AsyncMySQL,
+            mongo_data_conn: AsyncMongo,
+            mail_client: AsyncGMailClient,
+            request_url: str,
+            inbound_data: dict,
+            session_token: str = None
+    ) -> OAuthMailHandleCallbackResponse:
+
+        """
+        To handle the authorization callback for the mail client. The user may grant or deny authorization.
+        :param sql_conn: The database connection to use to perform this task.
+        :param mongo_data_conn: The database connection to use to perform this task.
+        :param mail_client: The instance of the third-party mail client that will be used to get the URL.
+        :param request_url: The full callback URL invoked by the third-party client.
+        :param inbound_data: The data that came in with the request (API call).
+        :param session_token: The session token of the user. It is expected that this will be null in all cases.
+        :return: A structured response of the process of handling the mail callback.
+        """
+
+        pass
+
     # ┳┳┓  •┓  ┏┓            •    •
     # ┃┃┃┏┓┓┃  ┗┓┓┏┏┳┓┏┳┓┏┓┏┓┓┓┏┓╋┓┏┓┏┓
     # ┛ ┗┗┻┗┗  ┗┛┗┻┛┗┗┛┗┗┗┻┛ ┗┗┗┻┗┗┗┛┛┗
 
-    pass
+    async def summarize_mail_with_ai(
+            self,
+            mongo_data_conn: AsyncMongo,
+            user_info: CoreUserInfoModel,
+            llm: CoreLLMController,
+            message: CoreMessageModel,
+            prompt_template: List[LLMInputMessage | dict]
+    ) -> LLMOutput:
+
+        """
+        To summarize the contents of a mail. To be used along the subject line on the UI.
+        :param mongo_data_conn: The database connection to use to perform this task.
+        :param user_info: The information about the user. This is needed to track token usage and bill accordingly.
+        :param llm: The instance of the LLM that must be used to performance of the summarization.
+        :param message: The message from which the content needs to be summarized.
+        :param prompt_template: The prompt template to use for summarization.
+        :return: A standard LLM output model.
+        """
+
+        # Extract the text from the message here:
+        text_parts = self.extract_plaintext_parts(payload = message.message["payload"])
+        text = "\n".join(text_parts)
+
+        # Invoke the LLM and return the response:
+        return await llm.invoke(
+            mongo_data_conn = mongo_data_conn,
+            user_info = user_info,
+            llm_input = LLMInput(
+                messages = prompt_template + [
+                    LLMInputMessage(
+                        role = "human",
+                        content = f"Please summarize this mail: \"\"\"{text}\"\"\""
+                    )
+                ]
+            )
+        )
 
     # ┳┳┓  •┓  ┏┓     ╹•
     # ┃┃┃┏┓┓┃  ┗┓┓┏┏┓┏ ┓┏┓┏┓
@@ -192,7 +366,41 @@ class MailController(CoreMessageController, ABC):
     # To synchronize the mails on the third-party client's server and your server. You are effectively making a copy of
     # the mail on your database.
 
-    pass
+    @abstractmethod
+    async def sync_mails(
+            self,
+            sql_conn: AsyncMySQL,
+            mongo_data_conn: AsyncMongo,
+            mail_client: AsyncGMailClient,
+            token_key: ObjectId | str,
+            user_info: CoreUserInfoModel | None,
+            llm: CoreLLMController = None,
+            force_sync: bool = False,
+            start_date: datetime.datetime = None,
+            end_date: datetime.datetime = None,
+            max_count: int = 100,
+            session_token: str = None
+    ) -> MailSyncManyResults:
+
+        """
+        To fetch mails from the third-party client and store them to your database.
+        :param sql_conn: The database connection to use to perform this task.
+        :param mongo_data_conn: The database connection to use to perform this task.
+        :param mail_client: The instance of the third-party mail client that will be used to get the URL.
+        :param token_key: The key by which the auth-tokens to this account are identified.
+        :param user_info: The information about your user who is trying to use this system. Needed to note LLM token
+            usage in the process of mail summarization.
+        :param llm: The instance of the LLm that can be used to summarize the contents of the mail.
+        :param force_sync: To forcefully sync a mail even if it already exists in the database.
+        :param start_date: The starting date (inclusive) from which mails must be sync'd.
+        :param end_date: The ending date (inclusive) till which mails must be sync'd.
+        :param max_count: The max. no. of mails to sync.
+        :param session_token: TO identify a user session. This will be null if a cron script invokes this method, else
+            it will be received from the inputs of the API call.
+        :return:
+        """
+
+        pass
 
     # ┳┳┓  •┓  ┓ •  •
     # ┃┃┃┏┓┓┃  ┃ ┓┏╋┓┏┓┏┓
@@ -211,6 +419,16 @@ class MailController(CoreMessageController, ABC):
 
     pass
 
+    # ┳┳┓  •┓  ┳┳   ┓   •
+    # ┃┃┃┏┓┓┃  ┃┃┏┓┏┫┏┓╋┓┏┓┏┓
+    # ┛ ┗┗┻┗┗  ┗┛┣┛┗┻┗┻┗┗┛┗┗┫
+    #            ┛          ┛
+
+    # We cannot modify the mails themselves, but we can set/unset tags on them for internal referencing and filtering.
+    # This will help the users organize their inboxes well.
+
+    pass
+
 
 #   *****************************************************************************************************************
 #  *****                                                                                                          ****

controllers_v2/message/mail/gmail.py

@@ -53,6 +53,8 @@ from models.api.message.mail.oauth import (
     OAuthMailAuthorizationRequestData
 )
 from models.message.mail.oauth import OAuthMailGetAuthorizationURLResponse, OAuthMailHandleCallbackResponse
+from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
+from models.message.mail.send import MailSendOneResult
 
 # Mail Client(s):
 from utils_v2.goog.controllers.gmail.gmail_client import AsyncGMailClient, SCOPES_GMAIL_MAIL_MANAGEMENT
@@ -60,9 +62,19 @@ from utils_v2.goog.controllers.gmail.gmail_client import AsyncGMailClient, SCOPE
 # To work with datatypes:
 from typing import List, Any
 
+# To work with MongoDB:
+from bson import ObjectId
+
+# To work with LLMs:
+from controllers.core.ai.llm import CoreLLMController
+from models.core.ai.llm import LLMInput, LLMOutput
+
 # To make HTTP requests:
 import httpx
 
+# To work with date and time:
+import datetime
+
 # For asynchronous activities:
 import asyncio
 
@@ -222,6 +234,17 @@ class GmailController(MailController):
             session_token: str = None
     ) -> OAuthMailHandleCallbackResponse:
 
+        """
+        To handle the authorization callback for the mail client. The user may grant or deny authorization.
+        :param sql_conn: The database connection to use to perform this task.
+        :param mongo_data_conn: The database connection to use to perform this task.
+        :param mail_client: The instance of the third-party mail client that will be used to get the URL.
+        :param request_url: The full callback URL invoked by the third-party client.
+        :param inbound_data: The data that came in with the request (API call).
+        :param session_token: The session token of the user. It is expected that this will be null in all cases.
+        :return: A structured response of the process of handling the mail callback.
+        """
+
         # Start by assuming failure:
         response = OAuthMailHandleCallbackResponse()
 
@@ -353,7 +376,40 @@ class GmailController(MailController):
     # To synchronize the mails on the third-party client's server and your server. You are effectively making a copy of
     # the mail on your database.
 
-    pass
+    async def sync_mails(
+            self,
+            sql_conn: AsyncMySQL,
+            mongo_data_conn: AsyncMongo,
+            mail_client: AsyncGMailClient,
+            token_key: ObjectId | str,
+            user_info: CoreUserInfoModel | None,
+            llm: CoreLLMController = None,
+            force_sync: bool = False,
+            start_date: datetime.datetime = None,
+            end_date: datetime.datetime = None,
+            max_count: int = 100,
+            session_token: str = None
+    ) -> MailSyncManyResults:
+
+        """
+        To fetch mails from the third-party client and store them to your database.
+        :param sql_conn: The database connection to use to perform this task.
+        :param mongo_data_conn: The database connection to use to perform this task.
+        :param mail_client: The instance of the third-party mail client that will be used to get the URL.
+        :param token_key: The key by which the auth-tokens to this account are identified.
+        :param user_info: The information about your user who is trying to use this system. Needed to note LLM token
+            usage in the process of mail summarization.
+        :param llm: The instance of the LLm that can be used to summarize the contents of the mail.
+        :param force_sync: To forcefully sync a mail even if it already exists in the database.
+        :param start_date: The starting date (inclusive) from which mails must be sync'd.
+        :param end_date: The ending date (inclusive) till which mails must be sync'd.
+        :param max_count: The max. no. of mails to sync.
+        :param session_token: TO identify a user session. This will be null if a cron script invokes this method, else
+            it will be received from the inputs of the API call.
+        :return:
+        """
+
+        pass
 
     # ┳┳┓  •┓  ┓ •  •
     # ┃┃┃┏┓┓┃  ┃ ┓┏╋┓┏┓┏┓
@@ -372,6 +428,16 @@ class GmailController(MailController):
 
     pass
 
+    # ┳┳┓  •┓  ┳┳   ┓   •
+    # ┃┃┃┏┓┓┃  ┃┃┏┓┏┫┏┓╋┓┏┓┏┓
+    # ┛ ┗┗┻┗┗  ┗┛┣┛┗┻┗┻┗┗┛┗┗┫
+    #            ┛          ┛
+
+    # We cannot modify the mails themselves, but we can set/unset tags on them for internal referencing and filtering.
+    # This will help the users organize their inboxes well.
+
+    pass
+
 
 #   *****************************************************************************************************************
 #  *****                                                                                                          ****