738 lines
31 KiB
Python
738 lines
31 KiB
Python
"""
|
|
|
|
AUTHOR:
|
|
|
|
Khushal P Soonderji
|
|
|
|
DATE:
|
|
|
|
Thursday, 16th Jan., 2025.
|
|
|
|
OBJECTIVE:
|
|
|
|
To handle all mail-related behaviour for Gmail from one place.
|
|
|
|
REFERENCES:
|
|
|
|
N/A
|
|
|
|
DOWNLOADS:
|
|
|
|
N/A
|
|
|
|
"""
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** IMPORT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# To make sibling directories accessible for imports:
|
|
import sys
|
|
sys.path.append(".")
|
|
sys.path.append("..")
|
|
|
|
# My async utils:
|
|
from utils_v2.date_time import date_time
|
|
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
|
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
|
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
|
|
|
|
# Controllers:
|
|
from controllers_v2.message.mail.base import MailController
|
|
|
|
# Models:
|
|
from models.core.user import CoreUserInfoModel
|
|
from models.core.auth_token import CoreAuthTokenModel
|
|
from models.core.message import CoreMessageModel
|
|
from models.api.message.mail.oauth import (
|
|
OAuthMailAuthorizationRequestHeaders,
|
|
OAuthMailAuthorizationRequestData
|
|
)
|
|
from models.message.mail.oauth import OAuthMailGetAuthorizationURLResponse, OAuthMailHandleCallbackResponse
|
|
from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
|
|
from models.message.mail.send import MailSendOneResult
|
|
|
|
# Mail Client(s):
|
|
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGmailClient, SCOPES_GMAIL_MAIL_MANAGEMENT
|
|
from utils_v2.goog.models.auth_tokens import GoogleAuthTokens
|
|
|
|
# To work with datatypes:
|
|
from typing import List, Any
|
|
|
|
# To work with MongoDB:
|
|
from bson import ObjectId
|
|
from pymongo import InsertOne, UpdateOne, ReplaceOne
|
|
|
|
# To work with LLMs:
|
|
from controllers.core.ai.llm import CoreLLMController
|
|
from models.core.ai.llm import LLMInput, LLMOutput
|
|
|
|
# To make HTTP requests:
|
|
import httpx
|
|
|
|
# To work with date and time:
|
|
import datetime
|
|
|
|
# For asynchronous activities:
|
|
import asyncio
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MACROS / ONE-TIME INIT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** VARIABLES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** FUNCTIONS ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** CLASSES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
class GmailController(MailController):
|
|
|
|
# ┏┓
|
|
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
|
|
# ┗┛┗┛┛┗┛┗┛ ┗┻┗┗┗┛┛
|
|
|
|
def __init__(
|
|
self,
|
|
cache: AsyncRedisCache = None,
|
|
http_client: httpx.AsyncClient = None,
|
|
alert_url: str = None,
|
|
debug: bool = True,
|
|
debug_prefix: str = "Gmail (C) | ",
|
|
debug_only_errors: bool = True
|
|
):
|
|
|
|
"""
|
|
This is the controller specifically built for Gmail's services. It is built on top of the base mail controller.
|
|
:param cache: The object to use for caching results from database calls.
|
|
:param http_client: The HTTP client
|
|
:param debug: Whether, or not, you would like to print debugging messages:
|
|
:param debug_prefix: The prefix to print with the debugging messages.
|
|
:param debug_only_errors: Whether you would like to print only error messages or all messages.
|
|
:return: None.
|
|
"""
|
|
|
|
# Invoke the parent's constructor:
|
|
super().__init__(
|
|
cache = cache,
|
|
alert_url = alert_url,
|
|
http_client = http_client,
|
|
base_filter = {"client": "gmail"},
|
|
debug = debug,
|
|
debug_prefix = debug_prefix,
|
|
debug_only_errors = debug_only_errors
|
|
)
|
|
|
|
# ┏┓┏┓ ┓ ┏┓ ┏┓
|
|
# ┃┃┣┫┓┏╋┣┓┏┛ ┃┫
|
|
# ┗┛┛┗┗┻┗┛┗┗━•┗┛
|
|
|
|
async def get_authorization_url(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
user_info: CoreUserInfoModel,
|
|
inbound_data: OAuthMailAuthorizationRequestData,
|
|
session_token: str
|
|
) -> OAuthMailGetAuthorizationURLResponse:
|
|
|
|
"""
|
|
To accept an incoming request for mail integration and provide a URL that the user can use to authorize your
|
|
service to access his mail inbox.
|
|
:param sql_conn: The database connection to use to perform this task.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param mail_client: The instance of the third-party mail client that will be used to get the URL.
|
|
:param user_info: The information about your user who is trying to use this system.
|
|
:param inbound_data: The data that came in with the request (API call).
|
|
:param session_token: The session token of the user.
|
|
:return: A structure response with details about the URL generation process.
|
|
"""
|
|
|
|
# Start by assuming failure:
|
|
response = OAuthMailGetAuthorizationURLResponse()
|
|
|
|
# First, we create/update a record for this integration request:
|
|
token_key = await self.generate_token_key(
|
|
sql_conn = sql_conn,
|
|
mongo_data_conn = mongo_data_conn,
|
|
auth_token = CoreAuthTokenModel(
|
|
serviceType = "email",
|
|
client = inbound_data.mailClient,
|
|
authType = "oauth",
|
|
user = user_info,
|
|
clientUserId = {"email": inbound_data.mailId},
|
|
status = "pending",
|
|
syncFreq = inbound_data.syncFreq,
|
|
),
|
|
token_notes = {
|
|
"email": inbound_data.mailId,
|
|
"client": inbound_data.mailClient
|
|
},
|
|
display_name = inbound_data.mailId,
|
|
display_picture = None,
|
|
session_token = session_token
|
|
)
|
|
|
|
# If generating the token key fails:
|
|
if token_key is None:
|
|
response.message = "Failed to generate token key."
|
|
return response
|
|
|
|
# Now we create the URL:
|
|
response.url = await mail_client.get_authorization_url(
|
|
scopes = SCOPES_GMAIL_MAIL_MANAGEMENT,
|
|
state = str(token_key),
|
|
access_type = "offline",
|
|
approval_prompt = "force",
|
|
include_granted_scopes = "true",
|
|
user_email = inbound_data.mailId
|
|
)
|
|
response.success = True
|
|
response.message = "Please use the URL to integrate your Gmail account."
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
async def handle_authorization_callback(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
request_url: str,
|
|
inbound_data: dict,
|
|
session_token: str = None
|
|
) -> OAuthMailHandleCallbackResponse:
|
|
|
|
"""
|
|
To handle the authorization callback for the mail client. The user may grant or deny authorization.
|
|
:param sql_conn: The database connection to use to perform this task.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param mail_client: The instance of the third-party mail client that will be used to get the URL.
|
|
:param request_url: The full callback URL invoked by the third-party client.
|
|
:param inbound_data: The data that came in with the request (API call).
|
|
:param session_token: The session token of the user. It is expected that this will be null in all cases.
|
|
:return: A structured response of the process of handling the mail callback.
|
|
"""
|
|
|
|
# Start by assuming failure:
|
|
response = OAuthMailHandleCallbackResponse()
|
|
|
|
# In case the user denied access:
|
|
if inbound_data.get("error") == "access_denied":
|
|
response.action = "denied"
|
|
response.message = "The user denied authorization."
|
|
return response
|
|
|
|
# Otherwise we know that the user authorized access:
|
|
else:
|
|
response.action = "authorized"
|
|
response.message = "The user has given authorization."
|
|
|
|
pass
|
|
# We fetch the auth-token associated with this authorization loop:
|
|
auth_token = await self.get_token_from_key(
|
|
mongo_data_conn = mongo_data_conn,
|
|
token_key = inbound_data["state"]
|
|
)
|
|
if not auth_token:
|
|
response.message = "Failed to load the auth-token for this flow."
|
|
return response
|
|
|
|
# Generate the tokens from the callback. Google sends all the needed params in the callback as the URL's query
|
|
# params. We can simply use the exact URL that was hit to generate the tokens. In Quart (and Flask) this can be
|
|
# achieved by 'request.url' like this:
|
|
google_tokens = await mail_client.get_authorization_tokens(
|
|
redirect_url = request_url,
|
|
scopes = None
|
|
)
|
|
|
|
# If no tokens were generated:
|
|
if not google_tokens:
|
|
response.message = "Failed to get access token(s) from Gmail."
|
|
return response
|
|
|
|
# Try getting the user's profile from Gmail:
|
|
user_profile = await mail_client.get_user_profile(tokens = google_tokens)
|
|
if user_profile.success:
|
|
google_tokens.email = user_profile.data["emailAddress"]
|
|
google_tokens.displayName = user_profile.data["displayName"]
|
|
google_tokens.displayPictureUrl = user_profile.data["displayPictureUrl"]
|
|
else:
|
|
response.message = "Failed to get the user's profile from Gmail."
|
|
return response
|
|
|
|
# We confirm if the expected email account and the one that gave authorization are the same:
|
|
if auth_token.clientUserId["email"] != str(google_tokens.email):
|
|
response.message = (
|
|
f"We were expecting authorization from '{auth_token.clientUserId['email']}', "
|
|
f"but got authorization from '{google_tokens.email}' instead."
|
|
)
|
|
return response
|
|
|
|
# Now we try to create the standard set of Labels:
|
|
labels = [
|
|
{
|
|
"name": "TCAOFF",
|
|
"textColor": "#434343",
|
|
"backgroundColor": "#e7e7e7"
|
|
},
|
|
{
|
|
"name": "CA-Doc",
|
|
"textColor": "#434343",
|
|
"backgroundColor": "#e7e7e7"
|
|
},
|
|
{
|
|
"name": "CA-AI",
|
|
"textColor": "#434343",
|
|
"backgroundColor": "#e7e7e7"
|
|
}
|
|
]
|
|
tasks = [
|
|
mail_client.create_label(
|
|
tokens = google_tokens,
|
|
label_name = label["name"],
|
|
label_visibility = "labelShow",
|
|
message_visibility = "show",
|
|
label_text_color = label["textColor"],
|
|
label_background_color = label["backgroundColor"]
|
|
) for label in labels
|
|
]
|
|
client_responses = await asyncio.gather(*tasks)
|
|
|
|
# Add the labels to the tokens data:
|
|
client_response = await mail_client.list_labels(tokens = google_tokens)
|
|
google_tokens.labels = client_response.data if client_response.success else None
|
|
|
|
# Now that we have passed the check,
|
|
# we save the tokens to the database:
|
|
auth_token.clientUserId = google_tokens.client_user_id
|
|
auth_token.token = google_tokens.model_dump()
|
|
auth_token.status = "active"
|
|
tokens_saved = await self.set_token(
|
|
sql_conn = sql_conn,
|
|
mongo_data_conn = mongo_data_conn,
|
|
token_key = inbound_data["state"],
|
|
auth_token = auth_token,
|
|
token_notes = {
|
|
"email": google_tokens.email,
|
|
"client": auth_token.client
|
|
},
|
|
display_name = google_tokens.displayName,
|
|
display_picture = google_tokens.displayPictureUrl,
|
|
session_token = session_token
|
|
)
|
|
|
|
# Note down the final result:
|
|
if tokens_saved:
|
|
response.success = True
|
|
response.message = "Authorization flow completed successfully."
|
|
else: response.message = "Failed to save the token(s)."
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
async def refresh_authorization(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
http_client: httpx.AsyncClient,
|
|
auth_token: CoreAuthTokenModel,
|
|
force_refresh: bool = False,
|
|
session_token: str = None
|
|
) -> CoreAuthTokenModel:
|
|
|
|
"""
|
|
To refresh the third-party client's access/authorization token(s) before use.
|
|
:param sql_conn: The database connection to use when storing the refreshed tokens.
|
|
:param mongo_data_conn: The database connection to use when storing the refreshed tokens.
|
|
:param mail_client: The connection of the third-party mail client.
|
|
:param http_client: The HTTP client to use to make the token refresh request.
|
|
:param auth_token: The auth-token model of the existing integration. This may get updated if a refresh is needed
|
|
(or forced).
|
|
:param force_refresh: Whether, or not, you would like to force a refresh even if the token hasn't expired yet.
|
|
:param session_token: The session token of the user. This will be null if this method is invoked by a cron
|
|
script in the background. Needed only to identify the user in case of a failure to send a timely alert.
|
|
:return: The same auth-token model instance, but maybe with updated tokens.
|
|
"""
|
|
|
|
# Extract the client's tokens from the full token payload given by the database:
|
|
google_tokens = GoogleAuthTokens(**auth_token.token)
|
|
|
|
# Refresh the tokens (if/as needed):
|
|
tokens_refreshed = await google_tokens.arefresh(
|
|
http_client = http_client,
|
|
client_id = mail_client.client_id,
|
|
client_secret = mail_client.client_secret,
|
|
force_refresh = force_refresh
|
|
)
|
|
|
|
# If the tokens were refreshed:
|
|
if tokens_refreshed:
|
|
|
|
# Try getting the user's profile from Gmail:
|
|
user_profile = await mail_client.get_user_profile(tokens = google_tokens)
|
|
if user_profile.success:
|
|
google_tokens.email = user_profile.data["emailAddress"]
|
|
google_tokens.displayName = user_profile.data["displayName"]
|
|
google_tokens.displayPictureUrl = user_profile.data["displayPictureUrl"]
|
|
|
|
# Update the existing auth-token model:
|
|
auth_token.token = google_tokens.model_dump()
|
|
auth_token.lastRefreshTs = date_time.get_current_utc_date_time(as_string = True)
|
|
|
|
# Try to update the record in the database:
|
|
await self.set_token(
|
|
sql_conn = sql_conn,
|
|
mongo_data_conn = mongo_data_conn,
|
|
token_key = auth_token.key,
|
|
auth_token = auth_token,
|
|
token_notes = {
|
|
"email": auth_token.clientUserId.get("email"),
|
|
"client": auth_token.client
|
|
},
|
|
display_name = google_tokens.email,
|
|
display_picture = google_tokens.displayPictureUrl,
|
|
session_token = session_token,
|
|
)
|
|
|
|
# Whether refreshed, or not, return the auth-token model:
|
|
return auth_token
|
|
|
|
# ┳┳┓ •┓ ┏┓ • •
|
|
# ┃┃┃┏┓┓┃ ┗┓┓┏┏┳┓┏┳┓┏┓┏┓┓┓┏┓╋┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗┻┛┗┗┛┗┗┗┻┛ ┗┗┗┻┗┗┗┛┛┗
|
|
|
|
pass
|
|
|
|
# ┳┳┓ •┓ ┏┓ ╹•
|
|
# ┃┃┃┏┓┓┃ ┗┓┓┏┏┓┏ ┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗┫┛┗┗ ┗┛┗┗┫
|
|
# ┛ ┛
|
|
|
|
# To synchronize the mails on the third-party client's server and your server. You are effectively making a copy of
|
|
# the mail on your database.
|
|
|
|
async def __sync_one_mail(
|
|
self,
|
|
mongo_data_conn: AsyncMongo,
|
|
user_info: CoreUserInfoModel,
|
|
auth_token: CoreAuthTokenModel,
|
|
mail_client: AsyncGmailClient,
|
|
google_tokens: GoogleAuthTokens,
|
|
message_id: str,
|
|
llm: CoreLLMController = None,
|
|
force_sync: bool = False
|
|
) -> MailSyncOneResult:
|
|
|
|
"""
|
|
To fetch one mail from the third-party client and store it in your database.
|
|
:param mongo_data_conn: The database connection to use to store the mail's payload.
|
|
:param user_info: The information about the user to whom this mail belongs.
|
|
:param auth_token: The credentials to use to get the mail from the third-party client.
|
|
:param mail_client: The third-party client's connection object.
|
|
:param google_tokens: The mail client's tokens the way they have to be used in their connection.
|
|
:param message_id: The way the third-party client recognizes the mail.
|
|
:param llm: To summarize the mail.
|
|
:param force_sync: If you'd like to forcefully re-sync the mail if its record already exists in your database.
|
|
:return: The structured response to express how the mail fetching went.
|
|
"""
|
|
|
|
# Start by assuming failure:
|
|
sync_result = MailSyncOneResult()
|
|
|
|
# If we've not been forced to re-sync the mail message,
|
|
# we first check if the mail already exists in our database:
|
|
if not force_sync:
|
|
mail_records = await self.get_message_previews(
|
|
mongo_data_conn = mongo_data_conn,
|
|
token_ids = [ObjectId(auth_token.authTokenId)],
|
|
limit = 1,
|
|
skip = 0,
|
|
additional_filter = {
|
|
"tokenId": ObjectId(auth_token.authTokenId),
|
|
"serviceType": auth_token.serviceType,
|
|
"client": auth_token.client,
|
|
"clientMessageId": message_id
|
|
}
|
|
)
|
|
if mail_records:
|
|
sync_result.success = True
|
|
sync_result.message = (
|
|
f"Gmail message '{message_id}' already "
|
|
f"sync'd on {mail_records[0].syncTs} (UTC)."
|
|
)
|
|
print(sync_result)
|
|
return sync_result
|
|
|
|
# Now that we know that we have to fetch the mail from GMail:
|
|
client_response = await mail_client.get_message(
|
|
tokens = google_tokens,
|
|
message_id = message_id,
|
|
return_raw = False
|
|
)
|
|
|
|
# If we didn't get the mail from Gmail;
|
|
if not client_response.success:
|
|
sync_result.message = f"Gmail (messageId: '{message_id}'): {client_response.message}"
|
|
return sync_result
|
|
|
|
# HANDLE ATTACHMENTS HERE:
|
|
client_response.data["payload"] = self.drop_attachments(client_response.data["payload"])
|
|
|
|
# Now we structure the message into the model:
|
|
all_recipients = []
|
|
for field in ["to", "cc", "bcc"]: all_recipients += [item["email"] for item in client_response.data[field]]
|
|
is_sent = False if google_tokens.email in all_recipients else True
|
|
mail_message = CoreMessageModel(
|
|
ts = client_response.data["ts"],
|
|
syncTs = date_time.get_current_utc_date_time(as_string = False),
|
|
tokenId = auth_token.authTokenId,
|
|
serviceType = auth_token.serviceType,
|
|
client = auth_token.client,
|
|
clientMessageId = message_id,
|
|
clientThreadId = client_response.data["threadId"],
|
|
isSent = True if is_sent else False,
|
|
isBroadcast = False,
|
|
sentSuccessfully = True if is_sent else False,
|
|
sender = [client_response.data["from"][0]["name"]],
|
|
recipient = all_recipients,
|
|
chat = None,
|
|
message = client_response.data,
|
|
snippet = client_response.data["subject"],
|
|
aiSnippet = None,
|
|
tags = ["Email", "Gmail"]
|
|
)
|
|
|
|
# Invoke the LLM:
|
|
try:
|
|
ai_snippet = await self.summarize_mail_with_ai(
|
|
mongo_data_conn = mongo_data_conn,
|
|
user_info = user_info,
|
|
llm = llm,
|
|
message = mail_message,
|
|
prompt_template = self.SENT_MAIL_SUMMARIZATION_PROMPT_TEMPLATE if is_sent else self.RECEIVED_MAIL_SUMMARIZATION_PROMPT_TEMPLATE
|
|
)
|
|
ai_json = ai_snippet.json
|
|
mail_message.aiSnippet = ai_snippet.summary
|
|
mail_message.aiSnippet["output"] = ai_json["summary"]
|
|
if ai_json["senderType"] is not None: mail_message.tags.append(ai_json["senderType"])
|
|
except Exception as exception:
|
|
self._printer(exception)
|
|
|
|
# Done here:
|
|
sync_result.success = True
|
|
sync_result.mailMessage = mail_message
|
|
return sync_result
|
|
|
|
async def sync_mails(
|
|
self,
|
|
sql_conn: AsyncMySQL,
|
|
mongo_data_conn: AsyncMongo,
|
|
mail_client: AsyncGmailClient,
|
|
auth_token: CoreAuthTokenModel,
|
|
user_info: CoreUserInfoModel | None,
|
|
llm: CoreLLMController = None,
|
|
force_sync: bool = False,
|
|
start_date: datetime.datetime = None,
|
|
end_date: datetime.datetime = None,
|
|
max_count: int = 100,
|
|
session_token: str = None
|
|
) -> MailSyncManyResults:
|
|
|
|
"""
|
|
To fetch mails from the third-party client and store them to your database.
|
|
:param sql_conn: The database connection to use to perform this task. Needed if the auth tokens need to be
|
|
refreshed or updated.
|
|
:param mongo_data_conn: The database connection to use to perform this task.
|
|
:param mail_client: The instance of the third-party mail client that will be used to get the URL.
|
|
:param auth_token: The credentials to the account with the third-party client.
|
|
:param user_info: The information about your user who is trying to use this system. Needed to note LLM token
|
|
usage in the process of mail summarization.
|
|
:param llm: The instance of the LLm that can be used to summarize the contents of the mail.
|
|
:param force_sync: To forcefully sync a mail even if it already exists in the database.
|
|
:param start_date: The starting date (inclusive) from which mails must be sync'd.
|
|
:param end_date: The ending date (inclusive) till which mails must be sync'd.
|
|
:param max_count: The max. no. of mails to sync.
|
|
:param session_token: TO identify a user session. This will be null if a cron script invokes this method, else
|
|
it will be received from the inputs of the API call.
|
|
:return: The structured response to express how the mail fetching went.
|
|
"""
|
|
|
|
# Start by assuming failure:
|
|
sync_results = MailSyncManyResults()
|
|
|
|
# Refresh the access token(s) if needed:
|
|
auth_token = await self.refresh_authorization(
|
|
sql_conn = sql_conn,
|
|
mongo_data_conn = mongo_data_conn,
|
|
mail_client = mail_client,
|
|
http_client = mail_client.http_client,
|
|
auth_token = auth_token,
|
|
force_refresh = False,
|
|
session_token = session_token
|
|
)
|
|
|
|
# If the user info was not given, take it from the token model:
|
|
if user_info is None: user_info = auth_token.user
|
|
|
|
# Extract the client's tokens from the full token model,
|
|
# and check if they are valid (not expired):
|
|
google_tokens = GoogleAuthTokens(**auth_token.token)
|
|
if google_tokens.expired:
|
|
sync_results.message = "Gmail token(s) have expired."
|
|
|
|
# Let's build the query to send to Google:
|
|
sub_queries = []
|
|
if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
|
|
if end_date: sub_queries.append((end_date + datetime.timedelta(days = 1)).strftime("before:%Y/%m/%d"))
|
|
query_string = " ".join(sub_queries)
|
|
|
|
# Let's enlist all the mails that fall in the date range:
|
|
client_response = await mail_client.list_messages(
|
|
tokens = google_tokens,
|
|
max_count = max_count,
|
|
query = query_string
|
|
)
|
|
|
|
# If the mail listing fails:
|
|
if not client_response.success:
|
|
self._printer(
|
|
client_response.success,
|
|
client_response.message,
|
|
client_response.data,
|
|
client_response.exception
|
|
)
|
|
sync_results.message = f"Gmail: {client_response.message}"
|
|
return sync_results
|
|
|
|
# Now, for every mail in the list, we fetch the mail and note the results:
|
|
messages_list = client_response.data["messages"]
|
|
tasks = [
|
|
self.__sync_one_mail(
|
|
mongo_data_conn = mongo_data_conn,
|
|
user_info = user_info,
|
|
auth_token = auth_token,
|
|
mail_client = mail_client,
|
|
google_tokens = google_tokens,
|
|
message_id = v["id"],
|
|
llm = llm,
|
|
force_sync = force_sync
|
|
) for v in messages_list.values()
|
|
]
|
|
individual_sync_results = await asyncio.gather(*tasks)
|
|
|
|
# Now we create operations for each mail,
|
|
# and maintain success/failure counters:
|
|
sync_results.totalCount = len(individual_sync_results)
|
|
mongo_operations = []
|
|
for result in individual_sync_results:
|
|
if result.success: sync_results.successCount += 1
|
|
else: sync_results.failureCount += 1
|
|
if result.mailMessage:
|
|
replacement_json = result.mailMessage.model_dump()
|
|
replacement_json.pop("_id", None)
|
|
mongo_operations.append(
|
|
ReplaceOne(
|
|
filter = {
|
|
"tokenId": ObjectId(auth_token.authTokenId),
|
|
"serviceType": auth_token.serviceType,
|
|
"client": auth_token.client,
|
|
"clientMessageId": result.mailMessage.clientMessageId
|
|
},
|
|
replacement = replacement_json,
|
|
upsert = True
|
|
)
|
|
)
|
|
|
|
# Make the bulk insert operation:
|
|
if mongo_operations:
|
|
sync_count = await self.bulk_operate_messages(
|
|
mongo_data_conn = mongo_data_conn,
|
|
mongo_operations = mongo_operations
|
|
)
|
|
|
|
# Apply the labels to the read messages:
|
|
try: client_response = await mail_client.modify_messages(
|
|
tokens = google_tokens,
|
|
message_ids = [v["id"] for v in messages_list.values()],
|
|
add_label_ids = [google_tokens.labels.get("TCAOFF", {}).get("id")]
|
|
)
|
|
except Exception as exception: pass
|
|
|
|
# Done here:
|
|
sync_results.message = f"{sync_results.successCount}/{sync_results.totalCount} mail(s) sync'd from Gmail."
|
|
return sync_results
|
|
|
|
# ┳┳┓ •┓ ┓ • •
|
|
# ┃┃┃┏┓┓┃ ┃ ┓┏╋┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗┛┗┗┛┗┗┫
|
|
# ┛
|
|
|
|
# Use these to show your users their mails once the mails are on your server. This would include activities like
|
|
# listing mails, showing full mails, showing mail trails, etc.
|
|
|
|
pass
|
|
|
|
# ┳┳┓ •┓ ┏┓ ┓•
|
|
# ┃┃┃┏┓┓┃ ┗┓┏┓┏┓┏┫┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┗ ┛┗┗┻┗┛┗┗┫
|
|
# ┛
|
|
|
|
pass
|
|
|
|
# ┳┳┓ •┓ ┳┳ ┓ •
|
|
# ┃┃┃┏┓┓┃ ┃┃┏┓┏┫┏┓╋┓┏┓┏┓
|
|
# ┛ ┗┗┻┗┗ ┗┛┣┛┗┻┗┻┗┗┛┗┗┫
|
|
# ┛ ┛
|
|
|
|
# We cannot modify the mails themselves, but we can set/unset tags on them for internal referencing and filtering.
|
|
# This will help the users organize their inboxes well.
|
|
|
|
pass
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MAIN PROGRAM ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
pass
|