572 lines
24 KiB
Python
572 lines
24 KiB
Python
"""
|
|
|
|
AUTHOR:
|
|
|
|
Khushal P Soonderji
|
|
|
|
DATE:
|
|
|
|
Thursday, 16th Jan., 2025.
|
|
|
|
OBJECTIVE:
|
|
|
|
To handle all mail-related behaviour from one place. The initially known client is only Gmail.
|
|
|
|
REFERENCES:
|
|
|
|
N/A
|
|
|
|
DOWNLOADS:
|
|
|
|
N/A
|
|
|
|
"""
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** IMPORT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# To make sibling directories accessible for imports:
|
|
import sys
|
|
sys.path.append(".")
|
|
sys.path.append("..")
|
|
|
|
# My async utils:
|
|
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
|
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
|
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
|
|
|
|
# Controllers:
|
|
from controllers_v2.core.message import CoreMessageController
|
|
from controllers.core.ai.llm import CoreLLMController
|
|
|
|
# Models:
|
|
from models.core.user import CoreUserInfoModel
|
|
from models.core.auth_token import CoreAuthTokenModel
|
|
from models.api.message.mail.oauth import (
|
|
OAuthMailAuthorizationRequestHeaders,
|
|
OAuthMailAuthorizationRequestData
|
|
)
|
|
from models.message.mail.oauth import OAuthMailGetAuthorizationURLResponse, OAuthMailHandleCallbackResponse
|
|
from models.core.message import CoreMessageModel
|
|
from models.core.ai.llm import LLMInput, LLMOutput, LLMInputMessage
|
|
from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
|
|
from models.message.mail.send import MailSendOneResult
|
|
|
|
# Mail Client(s):
|
|
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGmailClient
|
|
|
|
# To work with datatypes:
|
|
from typing import List, Any
|
|
|
|
# To make HTTP requests:
|
|
import httpx
|
|
|
|
# To work with MongoDB:
|
|
from bson.objectid import ObjectId
|
|
|
|
# To parse the HTML content in the mail:
|
|
from bs4 import BeautifulSoup
|
|
|
|
# To work with date and time:
|
|
import datetime
|
|
|
|
# To make abstract classes:
|
|
from abc import ABC, abstractmethod
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MACROS / ONE-TIME INIT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** VARIABLES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** FUNCTIONS ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** CLASSES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
class MailController(CoreMessageController, ABC):
|
|
|
|
# ┏┓┓ ┓┏ • ┓ ┓
|
|
# ┃ ┃┏┓┏┏ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
|
|
# ┗┛┗┗┻┛┛ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
|
|
|
|
# For AI Magic through LLMs:
|
|
RECEIVED_MAIL_SUMMARIZATION_PROMPT_TEMPLATE = [
|
|
LLMInputMessage(
|
|
role = "system",
|
|
content = (
|
|
"You're an expert mail summary program. "
|
|
"Provide the response in a structured JSON format with two fields: \"summary\" and \"senderType\". "
|
|
"The summary should be 150 chars or less. "
|
|
"If available, show login info like username and OTPs in your summary."
|
|
"If no login info is provided, please don't worry; just summarize what you see."
|
|
"\"senderType\" must be one of [\"Vendor\", \"Client\", null].\n"
|
|
"Example output 1: "
|
|
"{\"summary\":\"Sagar Supplies has shipped your materials. They are expected to reach by Thursday. "
|
|
"Use OTP 346780 when the delivery agent asks.\",\"senderType\":\"Vendor\"}\n"
|
|
"Example output 2: "
|
|
"{\"summary\":\"Mr. Mehta is enquiring about the submission of his tax filings, which are to be done "
|
|
"today.\",\"senderType\":\"Client\"}\n"
|
|
"Example output 3: "
|
|
"{\"summary\":\"JustDial's marketing message. They're offering a 35% discount to new accounts.\","
|
|
"\"senderType\":null}\n"
|
|
"Remember to respond only with the raw JSON string, nothing else. Good luck :)"
|
|
)
|
|
)
|
|
]
|
|
SENT_MAIL_SUMMARIZATION_PROMPT_TEMPLATE = [
|
|
LLMInputMessage(
|
|
role = "system",
|
|
content = (
|
|
"You're an expert mail summary assistant. That summarizes sent mails in 150 chars or less. "
|
|
"The objective of your user is to be able to recollect what a mail they sent was about from a brief "
|
|
"summary. Reply in a simple string, no formatting is allowed except emojis. Good luck :)"
|
|
)
|
|
)
|
|
]
|
|
|
|
# ┏┓
|
|
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
|
|
# ┗┛┗┛┛┗┛┗┛ ┗┻┗┗┗┛┛
|
|
|
|
def __init__(
|
|
self,
|
|
cache: AsyncRedisCache = None,
|
|
http_client: httpx.AsyncClient = None,
|
|
alert_url: str = None,
|
|
base_filter: dict = None,
|
|
debug: bool = True,
|
|
debug_prefix: str = "Mail (C) | ",
|
|
debug_only_errors: bool = True
|
|
):
|
|
|
|
"""
|
|
This is the foundational controller for all mail services. This is built on top of the core message controller,
|
|
and, in turn, all individual mail client controllers must be built on top of this.
|
|
:param cache: The object to use for caching results from database calls.
|
|
:param http_client: The HTTP client
|
|
:param base_filter: The basic filter that will be applied to all fetching/updating queries. WARNING: THE BASE
|
|
FILTER WILL ALWAYS BE APPLIED AUTOMATICALLY. SET THIS UP WISELY.
|
|
:param debug: Whether, or not, you would like to print debugging messages:
|
|
:param debug_prefix: The prefix to print with the debugging messages.
|
|
:param debug_only_errors: Whether you would like to print only error messages or all messages.
|
|
:return: None.
|
|
"""
|
|
|
|
# Prepare the combined base filter:
|
|
sms_filter = {}
|
|
for k, v in (base_filter or {}).items(): sms_filter[k] = v
|
|
sms_filter["serviceType"] = "email"
|
|
|
|
# Invoke the parents' constructor:
|
|
CoreMessageController.__init__(
|
|
self,
|
|
cache = cache,
|
|
alert_url = alert_url,
|
|
http_client = http_client,
|
|
base_filter = sms_filter,
|
|
debug = debug,
|
|
debug_prefix = debug_prefix,
|
|
debug_only_errors = debug_only_errors
|
|
)
|
|
|
|
# ┓┏ ┓
|
|
# ┣┫┏┓┃┏┓┏┓┏┓┏
|
|
# ┛┗┗ ┗┣┛┗ ┛ ┛
|
|
# ┛
|
|
|
|
def extract_plaintext_parts(
|
|
self,
|
|
payload: dict
|
|
) -> List[str]:
|
|
|
|
"""
|
|
A mail's body will have plaintext and HTML parts. This method extracts the plaintext parts if readily available,
|
|
or tries to convert the HTML parts to plaintext.
|
|
:param payload: The mail's full payload.
|
|
:return: An array of plaintext parts.
|
|
"""
|
|
|
|
# Start with just a holder:
|
|
text_parts = []
|
|
|
|
# If a direct text/plain part occurs,
|
|
# we just add it to the list:
|
|
if (
|
|
payload["contentMainType"] == "text" and
|
|
payload["contentSubType"] == "plain"
|
|
):
|
|
text_parts.append(payload["payload"])
|
|
|
|
# If a direct text/html part occurs,
|
|
# we just add it to the list:
|
|
if (
|
|
payload["contentMainType"] == "text" and
|
|
payload["contentSubType"] == "html"
|
|
):
|
|
html_parser = BeautifulSoup(payload["payload"], "html.parser")
|
|
text_parts.append(html_parser.get_text())
|
|
|
|
# If a multipart/alternative part occurs,
|
|
# we pick just the ready plaintext part:
|
|
if (
|
|
payload["contentMainType"] == "multipart" and
|
|
payload["contentSubType"] == "alternative"
|
|
):
|
|
for part in payload["payload"]:
|
|
if part["contentSubType"] == "plain":
|
|
text_parts.append(part["payload"])
|
|
|
|
# If a multipart/mixed or multipart/related part occurs,
|
|
# we use recursion to look for plaintext parts nested inside:
|
|
if (
|
|
payload["contentMainType"] == "multipart" and
|
|
(
|
|
payload["contentSubType"] == "mixed" or
|
|
payload["contentSubType"] == "related"
|
|
)
|
|
):
|
|
for part in payload["payload"]:
|
|
text_parts += self.extract_plaintext_parts(payload = part)
|
|
|
|
# Done here:
|
|
return text_parts
|
|
|
|
def drop_attachments(
|
|
self,
|
|
payload: dict
|
|
) -> dict:
|
|
|
|
"""
|
|
At the time of creating the system, we don't have a mechanism to save, organize and serve files in a
|
|
satisfactory way. This is a simple way to ignore all attachments till then by dropping them entirely.
|
|
:param payload: The mail's full payload.
|
|
:return: The same payload, but with all attachment data wiped clean.
|
|
"""
|
|
|
|
# If the part is some sort of file:
|
|
if payload["contentMainType"] not in ["multipart", "text"]:
|
|
payload["payload"] = None
|
|
payload["payloadId"] = None
|
|
payload["payloadUrl"] = None
|
|
|
|
# If the payload is of multipart type,
|
|
# we use recursion to look inside it:
|
|
elif payload["contentMainType"] == "multipart":
|
|
for part in payload["payload"]:
|
|
self.drop_attachments(part)
|
|
|
|
# Done here:
|
|
return payload
|
|
|
|
# ┏┓┏┓ ┓ ┏┓ ┏┓
|
|
# ┃┃┣┫┓┏╋┣┓┏┛ ┃┫
|
|
# ┗┛┛┗┗┻┗┛┗┗━•┗┛
|
|
|
|
@abstractmethod
|
|
async def get_authorization_url(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
user_info: CoreUserInfoModel,
|
|
inbound_data: OAuthMailAuthorizationRequestData,
|
|
session_token: str
|
|
) -> OAuthMailGetAuthorizationURLResponse:
|
|
|
|
"""
|
|
To accept an incoming request for mail integration and provide a URL that the user can use to authorize your
|
|
service to access his mail inbox.
|
|
:param sql_conn: The database connection to use to perform this task.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param mail_client: The instance of the third-party mail client that will be used to get the URL.
|
|
:param user_info: The information about your user who is trying to use this system.
|
|
:param inbound_data: The data that came in with the request (API call).
|
|
:param session_token: The session token of the user.
|
|
:return: A structure response with details about the URL generation process.
|
|
"""
|
|
|
|
pass
|
|
|
|
@abstractmethod
|
|
async def handle_authorization_callback(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
request_url: str,
|
|
inbound_data: dict,
|
|
session_token: str = None
|
|
) -> OAuthMailHandleCallbackResponse:
|
|
|
|
"""
|
|
To handle the authorization callback for the mail client. The user may grant or deny authorization.
|
|
:param sql_conn: The database connection to use to perform this task.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param mail_client: The instance of the third-party mail client that will be used to get the URL.
|
|
:param request_url: The full callback URL invoked by the third-party client.
|
|
:param inbound_data: The data that came in with the request (API call).
|
|
:param session_token: The session token of the user. It is expected that this will be null in all cases.
|
|
:return: A structured response of the process of handling the mail callback.
|
|
"""
|
|
|
|
pass
|
|
|
|
@abstractmethod
|
|
async def refresh_authorization(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
http_client: httpx.AsyncClient,
|
|
auth_token: CoreAuthTokenModel,
|
|
force_refresh: bool = False,
|
|
session_token: str = None
|
|
) -> CoreAuthTokenModel:
|
|
|
|
"""
|
|
To refresh the third-party client's access/authorization token(s) before use.
|
|
:param sql_conn: The database connection to use when storing the refreshed tokens.
|
|
:param mongo_data_conn: The database connection to use when storing the refreshed tokens.
|
|
:param mail_client: The connection of the third-party mail client.
|
|
:param http_client: The HTTP client to use to make the token refresh request.
|
|
:param auth_token: The auth-token model of the existing integration. This may get updated if a refresh is needed
|
|
(or forced).
|
|
:param force_refresh: Whether, or not, you would like to force a refresh even if the token hasn't expired yet.
|
|
:param session_token: The session token of the user. This will be null if this method is invoked by a cron
|
|
script in the background. Needed only to identify the user in case of a failure to send a timely alert.
|
|
:return: The same auth-token model instance, but maybe with updated tokens.
|
|
"""
|
|
|
|
pass
|
|
|
|
# ┳┳┓ •┓ ┏┓ • •
|
|
# ┃┃┃┏┓┓┃ ┗┓┓┏┏┳┓┏┳┓┏┓┏┓┓┓┏┓╋┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗┻┛┗┗┛┗┗┗┻┛ ┗┗┗┻┗┗┗┛┛┗
|
|
|
|
async def summarize_mail_with_ai(
|
|
self,
|
|
mongo_data_conn: AsyncMongo,
|
|
user_info: CoreUserInfoModel,
|
|
llm: CoreLLMController,
|
|
message: CoreMessageModel,
|
|
prompt_template: List[LLMInputMessage | dict]
|
|
) -> LLMOutput:
|
|
|
|
"""
|
|
To summarize the contents of a mail. To be used along the subject line on the UI.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param user_info: The information about the user. This is needed to track token usage and bill accordingly.
|
|
:param llm: The instance of the LLM that must be used to performance of the summarization.
|
|
:param message: The message from which the content needs to be summarized.
|
|
:param prompt_template: The prompt template to use for summarization.
|
|
:return: A standard LLM output model.
|
|
"""
|
|
|
|
# Extract the text from the message here:
|
|
text_parts = self.extract_plaintext_parts(payload = message.message["payload"])
|
|
text = "\n".join(text_parts)
|
|
|
|
# Invoke the LLM and return the response:
|
|
return await llm.invoke(
|
|
mongo_data_conn = mongo_data_conn,
|
|
user_info = user_info,
|
|
llm_input = LLMInput(
|
|
messages = prompt_template + [
|
|
LLMInputMessage(
|
|
role = "human",
|
|
content = f"Please summarize this mail: \"\"\"{text}\"\"\""
|
|
)
|
|
]
|
|
)
|
|
)
|
|
|
|
# ┳┳┓ •┓ ┏┓ ╹•
|
|
# ┃┃┃┏┓┓┃ ┗┓┓┏┏┓┏ ┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗┫┛┗┗ ┗┛┗┗┫
|
|
# ┛ ┛
|
|
|
|
# To synchronize the mails on the third-party client's server and your server. You are effectively making a copy of
|
|
# the mail on your database.
|
|
|
|
@abstractmethod
|
|
async def sync_mails(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
auth_token: CoreAuthTokenModel,
|
|
user_info: CoreUserInfoModel | None,
|
|
llm: CoreLLMController = None,
|
|
force_sync: bool = False,
|
|
start_date: datetime.datetime = None,
|
|
end_date: datetime.datetime = None,
|
|
max_count: int = 100,
|
|
session_token: str = None
|
|
) -> MailSyncManyResults:
|
|
|
|
"""
|
|
To fetch mails from the third-party client and store them to your database.
|
|
:param sql_conn: The database connection to use to perform this task.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param mail_client: The instance of the third-party mail client that will be used to get the URL.
|
|
:param auth_token: The credentials to the account with the third-party client.
|
|
:param user_info: The information about your user who is trying to use this system. Needed to note LLM token
|
|
usage in the process of mail summarization.
|
|
:param llm: The instance of the LLm that can be used to summarize the contents of the mail.
|
|
:param force_sync: To forcefully sync a mail even if it already exists in the database.
|
|
:param start_date: The starting date (inclusive) from which mails must be sync'd.
|
|
:param end_date: The ending date (inclusive) till which mails must be sync'd.
|
|
:param max_count: The max. no. of mails to sync.
|
|
:param session_token: TO identify a user session. This will be null if a cron script invokes this method, else
|
|
it will be received from the inputs of the API call.
|
|
:return:
|
|
"""
|
|
|
|
pass
|
|
|
|
# ┳┳┓ •┓ ┓ • •
|
|
# ┃┃┃┏┓┓┃ ┃ ┓┏╋┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗┛┗┗┛┗┗┫
|
|
# ┛
|
|
|
|
# Use these to show your users their mails once the mails are on your server. This would include activities like
|
|
# listing mails, showing full mails, showing mail trails, etc.
|
|
|
|
async def list_mails(
|
|
self,
|
|
mongo_data_conn: AsyncMongo,
|
|
token_ids: List[ObjectId | str],
|
|
limit: int = 100,
|
|
skip: int = 0,
|
|
additional_filter: dict = None
|
|
) -> List[CoreMessageModel] | None:
|
|
|
|
"""
|
|
To list mails (just previews, not full payloads).
|
|
:param mongo_data_conn: The database connection to use to fetch the data.
|
|
:param token_ids: The ids by which the mails will be identified. These are the auth-token ids of the accounts to
|
|
which the mails belong.
|
|
:param limit: The max. no. of mails to fetch. Good for pagination.
|
|
:param skip: The no. of initial mails to skip before picking mails to show. Good for pagination.
|
|
:param additional_filter: Any additional constraints.
|
|
:return: A list of message models that describe the contents of the mails.
|
|
"""
|
|
|
|
# Regardless of what additional filter is provided from outside,
|
|
# we add a mail-selecting filter here:
|
|
if additional_filter is None: additional_filter = {}
|
|
additional_filter["serviceType"] = "email"
|
|
|
|
# Simply call the core model:
|
|
return await self.get_message_previews(
|
|
mongo_data_conn = mongo_data_conn,
|
|
token_ids = token_ids,
|
|
limit = limit,
|
|
skip = skip,
|
|
additional_filter = additional_filter
|
|
)
|
|
|
|
async def get_one_mail(
|
|
self,
|
|
mongo_data_conn: AsyncMongo,
|
|
message_id: ObjectId | str
|
|
) -> CoreMessageModel | None:
|
|
|
|
"""
|
|
To get one full mail (the full payload, not just the preview).
|
|
:param mongo_data_conn: The database connection to use to fetch the data.
|
|
:param message_id: The ObjectId of the document that holds the mail.
|
|
:return: The message model containing the full payload of the mail.
|
|
"""
|
|
|
|
# Simply call the core model:
|
|
return await self.get_message(
|
|
mongo_data_conn = mongo_data_conn,
|
|
message_id = message_id
|
|
)
|
|
|
|
# ┳┳┓ •┓ ┏┓ ┓•
|
|
# ┃┃┃┏┓┓┃ ┗┓┏┓┏┓┏┫┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗ ┛┗┗┻┗┛┗┗┫
|
|
# ┛
|
|
|
|
pass
|
|
|
|
# ┳┳┓ •┓ ┳┳ ┓ •
|
|
# ┃┃┃┏┓┓┃ ┃┃┏┓┏┫┏┓╋┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┣┛┗┻┗┻┗┗┛┗┗┫
|
|
# ┛ ┛
|
|
|
|
# We cannot modify the mails themselves, but we can set/unset tags on them for internal referencing and filtering.
|
|
# This will help the users organize their inboxes well.
|
|
|
|
async def update_mail_tags(
|
|
self,
|
|
mongo_data_conn: AsyncMongo,
|
|
message_id: ObjectId | str,
|
|
unset_tags: List[str] = None,
|
|
set_tags: List[str] = None
|
|
) -> bool:
|
|
|
|
"""
|
|
To set and unset tags on a mail.
|
|
:param mongo_data_conn: The database connection to use to perform this action.
|
|
:param message_id: The ObjectId of the document in MongoDb that holds the message.
|
|
:param unset_tags: The list of tags to unset (done before setting new tags).
|
|
:param set_tags: The list of tags to set (done after unsetting old tags).
|
|
:return: True if successful, else False.
|
|
"""
|
|
|
|
# Simply call the core model:
|
|
return await self.update_message_tags(
|
|
mongo_data_conn = mongo_data_conn,
|
|
message_id = message_id,
|
|
unset_tags = unset_tags,
|
|
set_tags = set_tags
|
|
)
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MAIN PROGRAM ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
pass
|