khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241203) Mail sync'ing started. You can now hit a dedicated API endpoint to trigger a mail-sync process (in the background if you want) and save the mails to your database.

Browse Source
This commit is contained in:
khushal
2024-12-03 13:48:23 +05:30
parent 3dda5043d2
commit 615d8c71a5
8 changed files with 976 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/blueprints/mail/list.py
+255
View File
@@ -0,0 +1,255 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 3rd Dec., 2024
OBJECTIVE:
To enlist multiple e-mails for a given user at a time.
REFERENCES:
N/A
DOWNLOADS:
N/A
NOTES:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For using Quart:
from quart import Blueprint, current_app, g, request
# My utils:
from utils_v2.string import json
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.api.codes import StatusCodes, HttpCodes
from utils_v2.api.response import ResponseModel
from utils_v2.api.async_quart import (
set_api_version,
read_input,
get_session_info,
log_request_to_mongo,
log_chain_to_mongo,
should_not_be_under_maintenance,
only_whitelisted_ips,
limit_rate,
validate_input,
handle_cancelled_request
)
# GMail-related utils:
from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
# Common:
from shared import constants
# Data Models:
from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults
# To work with datatypes:
from typing import Literal
# For asynchronous activities:
import asyncio
# To work with LLMs:
from langchain_openai import ChatOpenAI
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Related to Quart:
mail_sync_bp = Blueprint("mail_sync", __name__)
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
@mail_sync_bp.record_once
def init(blueprint_setup_state):
# This gets called when the blueprint is registered.
# Consider this to be a one-time setup for the whole blueprint:
pass
# ---------------------------------------------------------------------------------------------------------------------
async def sync_mails(
mongo_conn: AsyncMongo,
llm: ChatOpenAI,
inbound_headers: dict,
inbound_data: MailSyncRequestData
) -> MailSyncManyResults:
"""
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
and running it in the background.
:param mongo_conn: The instance of the database connector to use to sync the mails.
:param llm: The instance of the LLM to use to summarize the mails.
:param inbound_headers: The headers that came in with the request.
:param inbound_data: The data that came in with the request.
:return: The results of the mail-sync'ing attempt.
"""
# Try to sync the mails:
return await current_app.mail_sync_model.sync(
session_token = inbound_headers["X-Session-Token"],
mongo_conn = mongo_conn,
account_identifier = inbound_data.accountId,
llm = llm,
force_sync = inbound_data.forceSync,
start_date = inbound_data.startDate,
end_date = inbound_data.endDate,
max_count = inbound_data.maxCount
)
# ---------------------------------------------------------------------------------------------------------------------
@mail_sync_bp.route("/sync", methods = ["POST"])
@mail_sync_bp.route("/sync/<mode>", methods = ["POST"])
@set_api_version(api_version = "1.0.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
@log_request_to_mongo(
attr_name = "logs_mongo",
project = constants.PROJECT_NAME,
log_type = constants.MODULE_NAME,
operation = "mailOAuthUrlReqApi",
log_input = True,
log_output = True,
sensitive_keys = ["sessionToken", "X-Session-Token"]
)
@log_chain_to_mongo(attr_name = "logs_mongo")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@validate_input(
header_validator = lambda x: MailSyncRequestHeaders(**x).model_dump(),
data_validator = lambda x: MailSyncRequestData(**x)
)
@handle_cancelled_request()
async def sync_mail(
mode: Literal["background", "bg"] = None,
inbound_headers: dict | MailSyncRequestHeaders = None,
inbound_data: dict | MailSyncRequestData = None,
inbound_files: dict = None,
**kwargs
):
"""
Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for
ready access on the UI.
:param mode: Set it to one of the specified options to make the sync'ing process go to the background.
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
:param kwargs: Any number of extra inputs supplied by the decorators.
:return: A standard response structure.
"""
# If the session token is invalid/expired:
if kwargs.get("session_info") is None:
return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.UNAUTHORIZED
)
# Make the variables available in the scope of the current request:
g.inbound_headers = inbound_headers
g.inbound_data = inbound_data
# If we've been asked to sync the mails in the background:
if mode in ["background", "bg"]:
current_app.add_background_task(
sync_mails,
mongo_conn = current_app.data_mongo,
llm = current_app.llm,
inbound_headers = inbound_headers,
inbound_data = inbound_data
)
return ResponseModel(
status_code = StatusCodes.OK,
http_code = HttpCodes.ACCEPTED,
message = "your mails are being sync'd in the background"
)
# Otherwise we process it right here:
sync_results = await sync_mails(
mongo_conn = current_app.data_mongo,
llm = current_app.llm,
inbound_headers = inbound_headers,
inbound_data = inbound_data
)
# Response:
return ResponseModel(
status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK,
http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS,
message = sync_results.message,
data = {
"totalCount": sync_results.totalCount,
"successCount": sync_results.successCount,
"failureCount": sync_results.failureCount
}
)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
api/blueprints/mail/oauth_callback.py
+2 -2
View File
@@ -108,12 +108,12 @@ def init(blueprint_setup_state):
# ---------------------------------------------------------------------------------------------------------------------
async def handle_gmail_callback():
async def handle_gmail_callback() -> render_template:
"""
To handle the callbacks from GMail specifically. Refer to the individual comments to check what hap[pens at each
step of the process.
:return: A rendered template of the final status of the authorization.
:return: A rendered template (HTML) of the final status of the authorization.
"""
# Start by assuming failure:
api/blueprints/mail/oauth_request.py
+1 -1
View File
@@ -126,7 +126,7 @@ def init(blueprint_setup_state):
operation = "mailOAuthUrlReqApi",
log_input = True,
log_output = True,
sensitive_keys = ["sessionToken"]
sensitive_keys = ["sessionToken", "X-Session-Token"]
)
@log_chain_to_mongo(attr_name = "logs_mongo")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
api/blueprints/mail/sync.py
+80 -71
View File
@@ -42,10 +42,11 @@ sys.path.append(".")
sys.path.append("..")
# For using Quart:
from quart import Blueprint, current_app, request
from quart import Blueprint, current_app, g, request
# My utils:
from utils_v2.string import json
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.api.codes import StatusCodes, HttpCodes
from utils_v2.api.response import ResponseModel
from utils_v2.api.async_quart import (
@@ -70,10 +71,20 @@ from shared import constants
# Data Models:
from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults
# To work with datatypes:
from typing import Literal
# For asynchronous activities:
import asyncio
# To work with LLMs:
from langchain_openai import ChatOpenAI
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
@@ -114,7 +125,41 @@ def init(blueprint_setup_state):
# ---------------------------------------------------------------------------------------------------------------------
async def sync_mails(
mongo_conn: AsyncMongo,
llm: ChatOpenAI,
inbound_headers: dict,
inbound_data: MailSyncRequestData
) -> MailSyncManyResults:
"""
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
and running it in the background.
:param mongo_conn: The instance of the database connector to use to sync the mails.
:param llm: The instance of the LLM to use to summarize the mails.
:param inbound_headers: The headers that came in with the request.
:param inbound_data: The data that came in with the request.
:return: The results of the mail-sync'ing attempt.
"""
# Try to sync the mails:
return await current_app.mail_sync_model.sync(
session_token = inbound_headers["X-Session-Token"],
mongo_conn = mongo_conn,
account_identifier = inbound_data.accountId,
llm = llm,
force_sync = inbound_data.forceSync,
start_date = inbound_data.startDate,
end_date = inbound_data.endDate,
max_count = inbound_data.maxCount
)
# ---------------------------------------------------------------------------------------------------------------------
@mail_sync_bp.route("/sync", methods = ["POST"])
@mail_sync_bp.route("/sync/<mode>", methods = ["POST"])
@set_api_version(api_version = "1.0.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
@@ -125,7 +170,7 @@ def init(blueprint_setup_state):
operation = "mailOAuthUrlReqApi",
log_input = True,
log_output = True,
sensitive_keys = ["sessionToken"]
sensitive_keys = ["sessionToken", "X-Session-Token"]
)
@log_chain_to_mongo(attr_name = "logs_mongo")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@@ -135,6 +180,7 @@ def init(blueprint_setup_state):
)
@handle_cancelled_request()
async def sync_mail(
mode: Literal["background", "bg"] = None,
inbound_headers: dict | MailSyncRequestHeaders = None,
inbound_data: dict | MailSyncRequestData = None,
inbound_files: dict = None,
@@ -144,6 +190,7 @@ async def sync_mail(
"""
Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for
ready access on the UI.
:param mode: Set it to one of the specified options to make the sync'ing process go to the background.
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
@@ -151,11 +198,6 @@ async def sync_mail(
:return: A standard response structure.
"""
# ┏┓
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┏┓┏┏
# ┣┛┛ ┗ ┣┛┛ ┗┛┗┗ ┛┛
# ┛
# If the session token is invalid/expired:
if kwargs.get("session_info") is None:
return ResponseModel(
@@ -163,76 +205,43 @@ async def sync_mail(
http_code = HttpCodes.UNAUTHORIZED
)
# Start by assuming failure:
mails_count = 0
# Make the variables available in the scope of the current request:
g.inbound_headers = inbound_headers
g.inbound_data = inbound_data
# ┏┓ ┏┳┓ ┓
# ┃┓┏┓╋ ┃ ┏┓┃┏┏┓┏┓
# ┗┛┗ ┗ ┻ ┗┛┛┗┗ ┛┗
# If we've been asked to sync the mails in the background:
if mode in ["background", "bg"]:
current_app.add_background_task(
sync_mails,
mongo_conn = current_app.data_mongo,
llm = current_app.llm,
inbound_headers = inbound_headers,
inbound_data = inbound_data
)
return ResponseModel(
status_code = StatusCodes.OK,
http_code = HttpCodes.ACCEPTED,
message = "your mails are being sync'd in the background"
)
# Make a user identifier from the session info:
user_auth = await current_app.mail_oauth_model.get_token(
# Otherwise we process it right here:
sync_results = await sync_mails(
mongo_conn = current_app.data_mongo,
serviceType = "email",
user = kwargs["session_info"],
llm = current_app.llm,
inbound_headers = inbound_headers,
inbound_data = inbound_data
)
print("MAIL TOKEN(S):", json.to_string(user_auth, default = str))
if not user_auth: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "user's email not connected"
)
# ┏┓┳┳┓ •┓
# ┃┓┃┃┃┏┓┓┃
# ┗┛┛ ┗┗┻┗┗
if user_auth["client"] == "gmail":
# Load the tokens into an object:
user_tokens = GoogleAuthTokens(**user_auth["token"])
# Try refreshing the tokens:
tokens_refreshed = await user_tokens.arefresh(
http_client = current_app.http_client,
client_id = current_app.gmail_client.client_id,
client_secret = current_app.gmail_client.client_secret,
force_refresh = False
)
# Update the token in the database if needed:
if tokens_refreshed: await current_app.mail_oauth_model.set_token(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
user_identifier = user_auth["_id"],
token = user_tokens.model_dump(),
session_token = inbound_headers["X-Session-Token"]
)
# Now we try to sync the mails:
mails_count = await current_app.mail_sync_model.sync(
mongo_conn = current_app.data_mongo,
user_info = kwargs["session_info"],
mail_client = current_app.gmail_client,
tokens = user_tokens,
llm = None,
force_sync = False,
start_date = inbound_data.startDate,
end_date = inbound_data.endDate,
max_count = inbound_data.maxCount
)
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
# ┛
# Done here:
# Response:
return ResponseModel(
status_code = StatusCodes.OK if mails_count else StatusCodes.FAILED,
message = f"{mails_count} mail(s) sync'd"
status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK,
http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS,
message = sync_results.message,
data = {
"totalCount": sync_results.totalCount,
"successCount": sync_results.successCount,
"failureCount": sync_results.failureCount
}
)
api/main.py
+18 -2
View File
@@ -52,7 +52,7 @@ from shared import constants
from utils_v2.string import json
from utils_v2.api import async_quart
from utils_v2.date_time import date_time
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
from utils_v2.serialization.json_serializer import JSONSerializer
@@ -72,7 +72,7 @@ from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
# Behaviour Models:
from models.behaviour.mail.oauth_v2 import MailOAuthModel
from models.behaviour.mail.sync import MailSyncModel
from models.behaviour.mail.sync_v2 import MailSyncModel
# To make REST API calls:
import httpx
@@ -90,6 +90,9 @@ from api.blueprints.test.callback import test_callback_bp
# All the helpers:
from api.helpers.user import session
# To work with LLMs:
from langchain_openai import ChatOpenAI
# *****************************************************************************************************************
# ***** ****
@@ -278,6 +281,13 @@ async def app_startup(**kwargs):
debug = enable_debugging
)
await current_app.data_mongo.connect()
current_app.files_mongo = AsyncMongo(
connection_string = script_cred["mongoDb"]["files"]["connectionString"],
database_name = script_cred["mongoDb"]["files"]["dbName"],
max_connections = script_cred["mongoDb"]["files"]["poolSize"],
debug = enable_debugging
)
await current_app.data_mongo.connect()
# ┳ ┓ ┳┳┓ ┓ ┓
# ┃┏┓╋┏┓┏┓┏┓┏┓┃ ┃┃┃┏┓┏┫┏┓┃┏
@@ -315,6 +325,12 @@ async def app_startup(**kwargs):
debug_only_errors = False
)
# For LLMs:
current_app.llm = ChatOpenAI(
model = script_cred["openAi"]["model"],
openai_api_key = script_cred["openAi"]["openai_api_key"]
)
# Pick the important stuff:
current_app.whitelisted_ips = current_app.script_data["whitelistedIps"]