(20241203) Mail sync'ing started. You can now hit a dedicated API endpoint to trigger a mail-sync process (in the background if you want) and save the mails to your database.
This commit is contained in:
+80
-71
@@ -42,10 +42,11 @@ sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For using Quart:
|
||||
from quart import Blueprint, current_app, request
|
||||
from quart import Blueprint, current_app, g, request
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.api.response import ResponseModel
|
||||
from utils_v2.api.async_quart import (
|
||||
@@ -70,10 +71,20 @@ from shared import constants
|
||||
|
||||
# Data Models:
|
||||
from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
|
||||
from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
# To work with LLMs:
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
@@ -114,7 +125,41 @@ def init(blueprint_setup_state):
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def sync_mails(
|
||||
mongo_conn: AsyncMongo,
|
||||
llm: ChatOpenAI,
|
||||
inbound_headers: dict,
|
||||
inbound_data: MailSyncRequestData
|
||||
) -> MailSyncManyResults:
|
||||
|
||||
"""
|
||||
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
|
||||
and running it in the background.
|
||||
:param mongo_conn: The instance of the database connector to use to sync the mails.
|
||||
:param llm: The instance of the LLM to use to summarize the mails.
|
||||
:param inbound_headers: The headers that came in with the request.
|
||||
:param inbound_data: The data that came in with the request.
|
||||
:return: The results of the mail-sync'ing attempt.
|
||||
"""
|
||||
|
||||
# Try to sync the mails:
|
||||
return await current_app.mail_sync_model.sync(
|
||||
session_token = inbound_headers["X-Session-Token"],
|
||||
mongo_conn = mongo_conn,
|
||||
account_identifier = inbound_data.accountId,
|
||||
llm = llm,
|
||||
force_sync = inbound_data.forceSync,
|
||||
start_date = inbound_data.startDate,
|
||||
end_date = inbound_data.endDate,
|
||||
max_count = inbound_data.maxCount
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@mail_sync_bp.route("/sync", methods = ["POST"])
|
||||
@mail_sync_bp.route("/sync/<mode>", methods = ["POST"])
|
||||
@set_api_version(api_version = "1.0.0")
|
||||
@read_input(sanitize_headers = False, sanitize_data = False)
|
||||
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
|
||||
@@ -125,7 +170,7 @@ def init(blueprint_setup_state):
|
||||
operation = "mailOAuthUrlReqApi",
|
||||
log_input = True,
|
||||
log_output = True,
|
||||
sensitive_keys = ["sessionToken"]
|
||||
sensitive_keys = ["sessionToken", "X-Session-Token"]
|
||||
)
|
||||
@log_chain_to_mongo(attr_name = "logs_mongo")
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@@ -135,6 +180,7 @@ def init(blueprint_setup_state):
|
||||
)
|
||||
@handle_cancelled_request()
|
||||
async def sync_mail(
|
||||
mode: Literal["background", "bg"] = None,
|
||||
inbound_headers: dict | MailSyncRequestHeaders = None,
|
||||
inbound_data: dict | MailSyncRequestData = None,
|
||||
inbound_files: dict = None,
|
||||
@@ -144,6 +190,7 @@ async def sync_mail(
|
||||
"""
|
||||
Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for
|
||||
ready access on the UI.
|
||||
:param mode: Set it to one of the specified options to make the sync'ing process go to the background.
|
||||
:param inbound_headers: auto-extracted by the decorators.
|
||||
:param inbound_data: auto-extracted by the decorators.
|
||||
:param inbound_files: auto-extracted by the decorators.
|
||||
@@ -151,11 +198,6 @@ async def sync_mail(
|
||||
:return: A standard response structure.
|
||||
"""
|
||||
|
||||
# ┏┓
|
||||
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┏┓┏┏
|
||||
# ┣┛┛ ┗ ┣┛┛ ┗┛┗┗ ┛┛
|
||||
# ┛
|
||||
|
||||
# If the session token is invalid/expired:
|
||||
if kwargs.get("session_info") is None:
|
||||
return ResponseModel(
|
||||
@@ -163,76 +205,43 @@ async def sync_mail(
|
||||
http_code = HttpCodes.UNAUTHORIZED
|
||||
)
|
||||
|
||||
# Start by assuming failure:
|
||||
mails_count = 0
|
||||
# Make the variables available in the scope of the current request:
|
||||
g.inbound_headers = inbound_headers
|
||||
g.inbound_data = inbound_data
|
||||
|
||||
# ┏┓ ┏┳┓ ┓
|
||||
# ┃┓┏┓╋ ┃ ┏┓┃┏┏┓┏┓
|
||||
# ┗┛┗ ┗ ┻ ┗┛┛┗┗ ┛┗
|
||||
# If we've been asked to sync the mails in the background:
|
||||
if mode in ["background", "bg"]:
|
||||
current_app.add_background_task(
|
||||
sync_mails,
|
||||
mongo_conn = current_app.data_mongo,
|
||||
llm = current_app.llm,
|
||||
inbound_headers = inbound_headers,
|
||||
inbound_data = inbound_data
|
||||
)
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.OK,
|
||||
http_code = HttpCodes.ACCEPTED,
|
||||
message = "your mails are being sync'd in the background"
|
||||
)
|
||||
|
||||
# Make a user identifier from the session info:
|
||||
user_auth = await current_app.mail_oauth_model.get_token(
|
||||
# Otherwise we process it right here:
|
||||
sync_results = await sync_mails(
|
||||
mongo_conn = current_app.data_mongo,
|
||||
serviceType = "email",
|
||||
user = kwargs["session_info"],
|
||||
llm = current_app.llm,
|
||||
inbound_headers = inbound_headers,
|
||||
inbound_data = inbound_data
|
||||
)
|
||||
|
||||
print("MAIL TOKEN(S):", json.to_string(user_auth, default = str))
|
||||
|
||||
if not user_auth: return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
http_code = HttpCodes.NOT_FOUND,
|
||||
message = "user's email not connected"
|
||||
)
|
||||
|
||||
# ┏┓┳┳┓ •┓
|
||||
# ┃┓┃┃┃┏┓┓┃
|
||||
# ┗┛┛ ┗┗┻┗┗
|
||||
|
||||
if user_auth["client"] == "gmail":
|
||||
|
||||
# Load the tokens into an object:
|
||||
user_tokens = GoogleAuthTokens(**user_auth["token"])
|
||||
|
||||
# Try refreshing the tokens:
|
||||
tokens_refreshed = await user_tokens.arefresh(
|
||||
http_client = current_app.http_client,
|
||||
client_id = current_app.gmail_client.client_id,
|
||||
client_secret = current_app.gmail_client.client_secret,
|
||||
force_refresh = False
|
||||
)
|
||||
|
||||
# Update the token in the database if needed:
|
||||
if tokens_refreshed: await current_app.mail_oauth_model.set_token(
|
||||
db_conn = current_app.sql_writer,
|
||||
mongo_conn = current_app.data_mongo,
|
||||
user_identifier = user_auth["_id"],
|
||||
token = user_tokens.model_dump(),
|
||||
session_token = inbound_headers["X-Session-Token"]
|
||||
)
|
||||
|
||||
# Now we try to sync the mails:
|
||||
mails_count = await current_app.mail_sync_model.sync(
|
||||
mongo_conn = current_app.data_mongo,
|
||||
user_info = kwargs["session_info"],
|
||||
mail_client = current_app.gmail_client,
|
||||
tokens = user_tokens,
|
||||
llm = None,
|
||||
force_sync = False,
|
||||
start_date = inbound_data.startDate,
|
||||
end_date = inbound_data.endDate,
|
||||
max_count = inbound_data.maxCount
|
||||
)
|
||||
|
||||
# ┳┓
|
||||
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
|
||||
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
|
||||
# ┛
|
||||
|
||||
# Done here:
|
||||
# Response:
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.OK if mails_count else StatusCodes.FAILED,
|
||||
message = f"{mails_count} mail(s) sync'd"
|
||||
status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK,
|
||||
http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS,
|
||||
message = sync_results.message,
|
||||
data = {
|
||||
"totalCount": sync_results.totalCount,
|
||||
"successCount": sync_results.successCount,
|
||||
"failureCount": sync_results.failureCount
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user