(20241203) Minor upgrades.
This commit is contained in:
+21
-86
@@ -10,7 +10,8 @@
|
|||||||
|
|
||||||
OBJECTIVE:
|
OBJECTIVE:
|
||||||
|
|
||||||
To enlist multiple e-mails for a given user at a time.
|
To list e-mails by their account identifier. Remember that the 'account identifier' is the '_id' of the document
|
||||||
|
in MongoDB that holds the tokens to authorize the e-mail id whose mails are being accessed.
|
||||||
|
|
||||||
REFERENCES:
|
REFERENCES:
|
||||||
|
|
||||||
@@ -48,6 +49,7 @@ from utils_v2.database.async_mongo_v2 import AsyncMongo
|
|||||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||||
from utils_v2.api.response import ResponseModel
|
from utils_v2.api.response import ResponseModel
|
||||||
from utils_v2.api.async_quart import (
|
from utils_v2.api.async_quart import (
|
||||||
|
make_ordered_json,
|
||||||
set_api_version,
|
set_api_version,
|
||||||
read_input,
|
read_input,
|
||||||
get_session_info,
|
get_session_info,
|
||||||
@@ -68,8 +70,7 @@ from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
|
|||||||
from shared import constants
|
from shared import constants
|
||||||
|
|
||||||
# Data Models:
|
# Data Models:
|
||||||
from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
|
from models.data.mail.list import MailListRequestHeaders, MailListByAccountIdRequestData
|
||||||
from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults
|
|
||||||
|
|
||||||
# To work with datatypes:
|
# To work with datatypes:
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
@@ -77,9 +78,6 @@ from typing import Literal
|
|||||||
# For asynchronous activities:
|
# For asynchronous activities:
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
# To work with LLMs:
|
|
||||||
from langchain_openai import ChatOpenAI
|
|
||||||
|
|
||||||
# To work with date and time:
|
# To work with date and time:
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
@@ -92,7 +90,7 @@ import datetime
|
|||||||
|
|
||||||
|
|
||||||
# Related to Quart:
|
# Related to Quart:
|
||||||
mail_sync_bp = Blueprint("mail_sync", __name__)
|
mail_retrieve_bp = Blueprint("mail_retrieve", __name__)
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
# *****************************************************************************************************************
|
||||||
@@ -112,7 +110,7 @@ mail_sync_bp = Blueprint("mail_sync", __name__)
|
|||||||
# *****************************************************************************************************************
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
@mail_sync_bp.record_once
|
@mail_retrieve_bp.record_once
|
||||||
def init(blueprint_setup_state):
|
def init(blueprint_setup_state):
|
||||||
|
|
||||||
# This gets called when the blueprint is registered.
|
# This gets called when the blueprint is registered.
|
||||||
@@ -123,41 +121,7 @@ def init(blueprint_setup_state):
|
|||||||
# ---------------------------------------------------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
async def sync_mails(
|
@mail_retrieve_bp.route("/list/account/id", methods = ["GET"])
|
||||||
mongo_conn: AsyncMongo,
|
|
||||||
llm: ChatOpenAI,
|
|
||||||
inbound_headers: dict,
|
|
||||||
inbound_data: MailSyncRequestData
|
|
||||||
) -> MailSyncManyResults:
|
|
||||||
|
|
||||||
"""
|
|
||||||
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
|
|
||||||
and running it in the background.
|
|
||||||
:param mongo_conn: The instance of the database connector to use to sync the mails.
|
|
||||||
:param llm: The instance of the LLM to use to summarize the mails.
|
|
||||||
:param inbound_headers: The headers that came in with the request.
|
|
||||||
:param inbound_data: The data that came in with the request.
|
|
||||||
:return: The results of the mail-sync'ing attempt.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Try to sync the mails:
|
|
||||||
return await current_app.mail_sync_model.sync(
|
|
||||||
session_token = inbound_headers["X-Session-Token"],
|
|
||||||
mongo_conn = mongo_conn,
|
|
||||||
account_identifier = inbound_data.accountId,
|
|
||||||
llm = llm,
|
|
||||||
force_sync = inbound_data.forceSync,
|
|
||||||
start_date = inbound_data.startDate,
|
|
||||||
end_date = inbound_data.endDate,
|
|
||||||
max_count = inbound_data.maxCount
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@mail_sync_bp.route("/sync", methods = ["POST"])
|
|
||||||
@mail_sync_bp.route("/sync/<mode>", methods = ["POST"])
|
|
||||||
@set_api_version(api_version = "1.0.0")
|
@set_api_version(api_version = "1.0.0")
|
||||||
@read_input(sanitize_headers = False, sanitize_data = False)
|
@read_input(sanitize_headers = False, sanitize_data = False)
|
||||||
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
|
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
|
||||||
@@ -165,7 +129,7 @@ async def sync_mails(
|
|||||||
attr_name = "logs_mongo",
|
attr_name = "logs_mongo",
|
||||||
project = constants.PROJECT_NAME,
|
project = constants.PROJECT_NAME,
|
||||||
log_type = constants.MODULE_NAME,
|
log_type = constants.MODULE_NAME,
|
||||||
operation = "mailOAuthUrlReqApi",
|
operation = "mailListByAccIdApi",
|
||||||
log_input = True,
|
log_input = True,
|
||||||
log_output = True,
|
log_output = True,
|
||||||
sensitive_keys = ["sessionToken", "X-Session-Token"]
|
sensitive_keys = ["sessionToken", "X-Session-Token"]
|
||||||
@@ -173,22 +137,19 @@ async def sync_mails(
|
|||||||
@log_chain_to_mongo(attr_name = "logs_mongo")
|
@log_chain_to_mongo(attr_name = "logs_mongo")
|
||||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||||
@validate_input(
|
@validate_input(
|
||||||
header_validator = lambda x: MailSyncRequestHeaders(**x).model_dump(),
|
header_validator = lambda x: MailGetRequestHeaders(**x).model_dump(),
|
||||||
data_validator = lambda x: MailSyncRequestData(**x)
|
data_validator = lambda x: MailGetRequestData(**x)
|
||||||
)
|
)
|
||||||
@handle_cancelled_request()
|
@handle_cancelled_request()
|
||||||
async def sync_mail(
|
async def get_one_mail(
|
||||||
mode: Literal["background", "bg"] = None,
|
inbound_headers: dict | MailGetRequestHeaders = None,
|
||||||
inbound_headers: dict | MailSyncRequestHeaders = None,
|
inbound_data: dict | MailGetRequestData = None,
|
||||||
inbound_data: dict | MailSyncRequestData = None,
|
|
||||||
inbound_files: dict = None,
|
inbound_files: dict = None,
|
||||||
**kwargs
|
**kwargs
|
||||||
):
|
):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for
|
Use this endpoint when the user wants to fetch one mail.
|
||||||
ready access on the UI.
|
|
||||||
:param mode: Set it to one of the specified options to make the sync'ing process go to the background.
|
|
||||||
:param inbound_headers: auto-extracted by the decorators.
|
:param inbound_headers: auto-extracted by the decorators.
|
||||||
:param inbound_data: auto-extracted by the decorators.
|
:param inbound_data: auto-extracted by the decorators.
|
||||||
:param inbound_files: auto-extracted by the decorators.
|
:param inbound_files: auto-extracted by the decorators.
|
||||||
@@ -203,43 +164,17 @@ async def sync_mail(
|
|||||||
http_code = HttpCodes.UNAUTHORIZED
|
http_code = HttpCodes.UNAUTHORIZED
|
||||||
)
|
)
|
||||||
|
|
||||||
# Make the variables available in the scope of the current request:
|
# Get the mail:
|
||||||
g.inbound_headers = inbound_headers
|
mail_data = await current_app.mail_retrieve_model.get_mail(
|
||||||
g.inbound_data = inbound_data
|
|
||||||
|
|
||||||
# If we've been asked to sync the mails in the background:
|
|
||||||
if mode in ["background", "bg"]:
|
|
||||||
current_app.add_background_task(
|
|
||||||
sync_mails,
|
|
||||||
mongo_conn = current_app.data_mongo,
|
|
||||||
llm = current_app.llm,
|
|
||||||
inbound_headers = inbound_headers,
|
|
||||||
inbound_data = inbound_data
|
|
||||||
)
|
|
||||||
return ResponseModel(
|
|
||||||
status_code = StatusCodes.OK,
|
|
||||||
http_code = HttpCodes.ACCEPTED,
|
|
||||||
message = "your mails are being sync'd in the background"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Otherwise we process it right here:
|
|
||||||
sync_results = await sync_mails(
|
|
||||||
mongo_conn = current_app.data_mongo,
|
mongo_conn = current_app.data_mongo,
|
||||||
llm = current_app.llm,
|
mail_identifier = inbound_data.mailId
|
||||||
inbound_headers = inbound_headers,
|
|
||||||
inbound_data = inbound_data
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Response:
|
# Done here:
|
||||||
return ResponseModel(
|
return ResponseModel(
|
||||||
status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK,
|
status_code = StatusCodes.OK if mail_data else StatusCodes.FAILED,
|
||||||
http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS,
|
http_code = HttpCodes.SUCCESS if mail_data else HttpCodes.NOT_FOUND,
|
||||||
message = sync_results.message,
|
data = mail_data
|
||||||
data = {
|
|
||||||
"totalCount": sync_results.totalCount,
|
|
||||||
"successCount": sync_results.successCount,
|
|
||||||
"failureCount": sync_results.failureCount
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,255 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
AUTHOR:
|
||||||
|
|
||||||
|
Khushal P Soonderji
|
||||||
|
|
||||||
|
DATE:
|
||||||
|
|
||||||
|
Tuesday, 3rd Dec., 2024
|
||||||
|
|
||||||
|
OBJECTIVE:
|
||||||
|
|
||||||
|
To enlist multiple e-mails for a given user at a time.
|
||||||
|
|
||||||
|
REFERENCES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
DOWNLOADS:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
NOTES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** IMPORT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# To make sibling directories accessible for imports:
|
||||||
|
import sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
# For using Quart:
|
||||||
|
from quart import Blueprint, current_app, g, request
|
||||||
|
|
||||||
|
# My utils:
|
||||||
|
from utils_v2.string import json
|
||||||
|
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||||
|
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||||
|
from utils_v2.api.response import ResponseModel
|
||||||
|
from utils_v2.api.async_quart import (
|
||||||
|
set_api_version,
|
||||||
|
read_input,
|
||||||
|
get_session_info,
|
||||||
|
log_request_to_mongo,
|
||||||
|
log_chain_to_mongo,
|
||||||
|
should_not_be_under_maintenance,
|
||||||
|
only_whitelisted_ips,
|
||||||
|
limit_rate,
|
||||||
|
validate_input,
|
||||||
|
handle_cancelled_request
|
||||||
|
)
|
||||||
|
|
||||||
|
# GMail-related utils:
|
||||||
|
from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
|
||||||
|
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
|
||||||
|
|
||||||
|
# Common:
|
||||||
|
from shared import constants
|
||||||
|
|
||||||
|
# Data Models:
|
||||||
|
from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
|
||||||
|
from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults
|
||||||
|
|
||||||
|
# To work with datatypes:
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
# For asynchronous activities:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
# To work with LLMs:
|
||||||
|
from langchain_openai import ChatOpenAI
|
||||||
|
|
||||||
|
# To work with date and time:
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MACROS / ONE-TIME INIT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# Related to Quart:
|
||||||
|
mail_sync_bp = Blueprint("mail_sync", __name__)
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** VARIABLES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** FUNCTIONS ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
@mail_sync_bp.record_once
|
||||||
|
def init(blueprint_setup_state):
|
||||||
|
|
||||||
|
# This gets called when the blueprint is registered.
|
||||||
|
# Consider this to be a one-time setup for the whole blueprint:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def sync_mails(
|
||||||
|
mongo_conn: AsyncMongo,
|
||||||
|
llm: ChatOpenAI,
|
||||||
|
inbound_headers: dict,
|
||||||
|
inbound_data: MailSyncRequestData
|
||||||
|
) -> MailSyncManyResults:
|
||||||
|
|
||||||
|
"""
|
||||||
|
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
|
||||||
|
and running it in the background.
|
||||||
|
:param mongo_conn: The instance of the database connector to use to sync the mails.
|
||||||
|
:param llm: The instance of the LLM to use to summarize the mails.
|
||||||
|
:param inbound_headers: The headers that came in with the request.
|
||||||
|
:param inbound_data: The data that came in with the request.
|
||||||
|
:return: The results of the mail-sync'ing attempt.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Try to sync the mails:
|
||||||
|
return await current_app.mail_sync_model.sync(
|
||||||
|
session_token = inbound_headers["X-Session-Token"],
|
||||||
|
mongo_conn = mongo_conn,
|
||||||
|
account_identifier = inbound_data.accountId,
|
||||||
|
llm = llm,
|
||||||
|
force_sync = inbound_data.forceSync,
|
||||||
|
start_date = inbound_data.startDate,
|
||||||
|
end_date = inbound_data.endDate,
|
||||||
|
max_count = inbound_data.maxCount
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@mail_sync_bp.route("/sync", methods = ["POST"])
|
||||||
|
@mail_sync_bp.route("/sync/<mode>", methods = ["POST"])
|
||||||
|
@set_api_version(api_version = "1.0.0")
|
||||||
|
@read_input(sanitize_headers = False, sanitize_data = False)
|
||||||
|
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
|
||||||
|
@log_request_to_mongo(
|
||||||
|
attr_name = "logs_mongo",
|
||||||
|
project = constants.PROJECT_NAME,
|
||||||
|
log_type = constants.MODULE_NAME,
|
||||||
|
operation = "mailOAuthUrlReqApi",
|
||||||
|
log_input = True,
|
||||||
|
log_output = True,
|
||||||
|
sensitive_keys = ["sessionToken", "X-Session-Token"]
|
||||||
|
)
|
||||||
|
@log_chain_to_mongo(attr_name = "logs_mongo")
|
||||||
|
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||||
|
@validate_input(
|
||||||
|
header_validator = lambda x: MailSyncRequestHeaders(**x).model_dump(),
|
||||||
|
data_validator = lambda x: MailSyncRequestData(**x)
|
||||||
|
)
|
||||||
|
@handle_cancelled_request()
|
||||||
|
async def sync_mail(
|
||||||
|
mode: Literal["background", "bg"] = None,
|
||||||
|
inbound_headers: dict | MailSyncRequestHeaders = None,
|
||||||
|
inbound_data: dict | MailSyncRequestData = None,
|
||||||
|
inbound_files: dict = None,
|
||||||
|
**kwargs
|
||||||
|
):
|
||||||
|
|
||||||
|
"""
|
||||||
|
Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for
|
||||||
|
ready access on the UI.
|
||||||
|
:param mode: Set it to one of the specified options to make the sync'ing process go to the background.
|
||||||
|
:param inbound_headers: auto-extracted by the decorators.
|
||||||
|
:param inbound_data: auto-extracted by the decorators.
|
||||||
|
:param inbound_files: auto-extracted by the decorators.
|
||||||
|
:param kwargs: Any number of extra inputs supplied by the decorators.
|
||||||
|
:return: A standard response structure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# If the session token is invalid/expired:
|
||||||
|
if kwargs.get("session_info") is None:
|
||||||
|
return ResponseModel(
|
||||||
|
status_code = StatusCodes.FAILED,
|
||||||
|
http_code = HttpCodes.UNAUTHORIZED
|
||||||
|
)
|
||||||
|
|
||||||
|
# Make the variables available in the scope of the current request:
|
||||||
|
g.inbound_headers = inbound_headers
|
||||||
|
g.inbound_data = inbound_data
|
||||||
|
|
||||||
|
# If we've been asked to sync the mails in the background:
|
||||||
|
if mode in ["background", "bg"]:
|
||||||
|
current_app.add_background_task(
|
||||||
|
sync_mails,
|
||||||
|
mongo_conn = current_app.data_mongo,
|
||||||
|
llm = current_app.llm,
|
||||||
|
inbound_headers = inbound_headers,
|
||||||
|
inbound_data = inbound_data
|
||||||
|
)
|
||||||
|
return ResponseModel(
|
||||||
|
status_code = StatusCodes.OK,
|
||||||
|
http_code = HttpCodes.ACCEPTED,
|
||||||
|
message = "your mails are being sync'd in the background"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Otherwise we process it right here:
|
||||||
|
sync_results = await sync_mails(
|
||||||
|
mongo_conn = current_app.data_mongo,
|
||||||
|
llm = current_app.llm,
|
||||||
|
inbound_headers = inbound_headers,
|
||||||
|
inbound_data = inbound_data
|
||||||
|
)
|
||||||
|
|
||||||
|
# Response:
|
||||||
|
return ResponseModel(
|
||||||
|
status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK,
|
||||||
|
http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS,
|
||||||
|
message = sync_results.message,
|
||||||
|
data = {
|
||||||
|
"totalCount": sync_results.totalCount,
|
||||||
|
"successCount": sync_results.successCount,
|
||||||
|
"failureCount": sync_results.failureCount
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MAIN PROGRAM ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
pass
|
||||||
@@ -1,312 +0,0 @@
|
|||||||
"""
|
|
||||||
|
|
||||||
AUTHOR:
|
|
||||||
|
|
||||||
Khushal P Soonderji
|
|
||||||
|
|
||||||
DATE:
|
|
||||||
|
|
||||||
Wednesday, 27th Nov., 2024
|
|
||||||
|
|
||||||
OBJECTIVE:
|
|
||||||
|
|
||||||
To define the interaction between the UI layer and the database connectivity in one place. Here we shall handle
|
|
||||||
all the activities for OAuth2.0 authorization requests for all the users of our service.
|
|
||||||
|
|
||||||
REFERENCES:
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
DOWNLOADS:
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** IMPORT ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# To make sibling directories accessible for imports:
|
|
||||||
import sys
|
|
||||||
sys.path.append(".")
|
|
||||||
sys.path.append("..")
|
|
||||||
|
|
||||||
# My async utils:
|
|
||||||
from utils_v2.string import json
|
|
||||||
from utils_v2.date_time import date_time
|
|
||||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
|
||||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
|
||||||
|
|
||||||
# Base model:
|
|
||||||
from models.behaviour.base import BaseModel
|
|
||||||
|
|
||||||
# To work with MongoDB:
|
|
||||||
from bson import ObjectId
|
|
||||||
|
|
||||||
# To work with datatypes:
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
# To make deep-copies:
|
|
||||||
import copy
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** MACROS / ONE-TIME INIT ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# --- Nothing Yet
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** VARIABLES ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# --- Nothing Yet
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** FUNCTIONS ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# --- Nothing Yet
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** CLASSES ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
class MailOAuthModel(BaseModel):
|
|
||||||
|
|
||||||
AUTH_COLLECTION = "_authTokens"
|
|
||||||
|
|
||||||
async def get_user_identifier(
|
|
||||||
self,
|
|
||||||
db_conn: AsyncMySQL,
|
|
||||||
mongo_conn: AsyncMongo,
|
|
||||||
user_info: dict,
|
|
||||||
service_client: Literal["gmail"],
|
|
||||||
auth_type: Literal["oauth"],
|
|
||||||
session_token: str = None
|
|
||||||
) -> ObjectId:
|
|
||||||
|
|
||||||
"""
|
|
||||||
Stores params from the session info and gives an identifier to use in the authorization URL. Use this when the
|
|
||||||
user requests an authorization URL to link your service to another service (like GMail).
|
|
||||||
:param db_conn: The database connection (MariaDB) to use to perform the action.
|
|
||||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
|
||||||
:param user_info: The dictionary that has the user's session information.
|
|
||||||
:param service_client: The name of the company or brand that is providing this service that is being integrated.
|
|
||||||
:param auth_type: To identify the type of authentication being done here. This could indicate simple password
|
|
||||||
authentication, more advance OAuth2.0 authentication, etc.
|
|
||||||
:param session_token: The session token of the user who requested this service.
|
|
||||||
:return: An ObjectId to later store the granted tokens.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Note down the timestamp at which this event occurred:
|
|
||||||
request_ts = date_time.get_current_utc_date_time(as_string = False)
|
|
||||||
|
|
||||||
# Get the identifier from the database:
|
|
||||||
inserted_id = await mongo_conn.insert_one(
|
|
||||||
collection = MailOAuthModel.AUTH_COLLECTION,
|
|
||||||
document = {
|
|
||||||
"version": "-1.0.1",
|
|
||||||
"serviceType": "email",
|
|
||||||
"client": service_client,
|
|
||||||
"authType": auth_type,
|
|
||||||
"user": user_info,
|
|
||||||
"token": None,
|
|
||||||
"firstRefreshTs": None,
|
|
||||||
"lastRefreshTs": None,
|
|
||||||
"lastRequestTs": request_ts
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return inserted_id
|
|
||||||
|
|
||||||
# # Get the identifier from the database:
|
|
||||||
# mongo_json = await mongo_conn.find_one_and_update(
|
|
||||||
# collection = MailOAuthModel.AUTH_COLLECTION,
|
|
||||||
# filter = {
|
|
||||||
# "serviceType": "email",
|
|
||||||
# "client": service_client,
|
|
||||||
# "authType": auth_type,
|
|
||||||
# "user": user_info,
|
|
||||||
# },
|
|
||||||
# update = {
|
|
||||||
# "$set": {
|
|
||||||
# "lastRequestTs": request_ts
|
|
||||||
# },
|
|
||||||
# "$setOnInsert": {
|
|
||||||
# "version": "1.0.0",
|
|
||||||
# "serviceType": "email",
|
|
||||||
# "client": service_client,
|
|
||||||
# "authType": auth_type,
|
|
||||||
# "user": user_info,
|
|
||||||
# "token": None,
|
|
||||||
# "firstRefreshTs": None,
|
|
||||||
# "lastRefreshTs": None,
|
|
||||||
# "firstRequestTs": request_ts,
|
|
||||||
# }
|
|
||||||
# },
|
|
||||||
# projection = {
|
|
||||||
# "_id": True
|
|
||||||
# },
|
|
||||||
# upsert = True,
|
|
||||||
# return_updated = True
|
|
||||||
# )
|
|
||||||
#
|
|
||||||
# # Tell MariaDB that an authorization request was initiated:
|
|
||||||
# db_json = {}
|
|
||||||
# if mongo_json is not None:
|
|
||||||
# db_json = await self.call_procedure(
|
|
||||||
# db_conn = db_conn,
|
|
||||||
# proc_name = "entity_integration_save",
|
|
||||||
# proc_args = (
|
|
||||||
# user_info["entityId"], # ............................................ 'p_entity_id'
|
|
||||||
# service_client, # ................................................... 'p_provider'
|
|
||||||
# "Auth Requested", # ................................................. 'p_current_status'
|
|
||||||
# "Auth URL Generated", # ............................................. 'p_last_action'
|
|
||||||
# None, # ............................................................. 'p_display_name'
|
|
||||||
# None, # ............................................................. 'p_display_picture'
|
|
||||||
# str(mongo_json["_id"]), # ........................................... 'p_token_id'
|
|
||||||
# json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
|
|
||||||
# user_info["userId"] # ............................................... 'p_created_by'
|
|
||||||
# ),
|
|
||||||
# session_token = session_token
|
|
||||||
# )
|
|
||||||
#
|
|
||||||
# # Done here:
|
|
||||||
# return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
|
|
||||||
|
|
||||||
async def set_token(
|
|
||||||
self,
|
|
||||||
db_conn: AsyncMySQL,
|
|
||||||
mongo_conn: AsyncMongo,
|
|
||||||
user_identifier: ObjectId | str,
|
|
||||||
token: dict,
|
|
||||||
session_token: str = None
|
|
||||||
) -> bool:
|
|
||||||
|
|
||||||
"""
|
|
||||||
This method is to be called when the end user authorizes your service to connect to his third-party account. For
|
|
||||||
example, when the end user allows you to access his GMail account. USE THIS FOR UPDATING (REFRESHING) TOKENS
|
|
||||||
ALSO.
|
|
||||||
:param db_conn: The database connection (MariaDB) to use to perform the action.
|
|
||||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
|
||||||
:param user_identifier: The identifier granted by the 'get_user_identifier' method.
|
|
||||||
:param token: The token granted by the third-party service.
|
|
||||||
:param session_token: The session token of the user who requested this service.
|
|
||||||
:return: True if saved, False if failed.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Start by assuming failure:
|
|
||||||
token_saved = False
|
|
||||||
|
|
||||||
# Note down the timestamp at which this event occurred:
|
|
||||||
request_ts = date_time.get_current_utc_date_time(as_string = False)
|
|
||||||
|
|
||||||
# Save the token to MongoDB:
|
|
||||||
mongo_json = await mongo_conn.find_one_and_update(
|
|
||||||
collection = MailOAuthModel.AUTH_COLLECTION,
|
|
||||||
filter = {"_id": ObjectId(user_identifier)},
|
|
||||||
update = {
|
|
||||||
"$set": {
|
|
||||||
"token": token,
|
|
||||||
"lastRefreshTs": request_ts,
|
|
||||||
},
|
|
||||||
"$setOnInsert": {
|
|
||||||
"firstRefreshTs": request_ts,
|
|
||||||
}
|
|
||||||
},
|
|
||||||
projection = {"token": False},
|
|
||||||
return_updated = True,
|
|
||||||
upsert = False
|
|
||||||
)
|
|
||||||
|
|
||||||
# Tell MariaDB that the token was saved:
|
|
||||||
if mongo_json is not None:
|
|
||||||
db_json = await self.call_procedure(
|
|
||||||
db_conn = db_conn,
|
|
||||||
proc_name = "entity_integration_save",
|
|
||||||
proc_args = (
|
|
||||||
mongo_json["user"]["entityId"], # ............................................. 'p_entity_id'
|
|
||||||
mongo_json["client"], # ....................................................... 'p_provider'
|
|
||||||
"Auth Granted", # ............................................................. 'p_current_status'
|
|
||||||
"Set Token", # ................................................................ 'p_last_action'
|
|
||||||
None, # ....................................................................... 'p_display_name'
|
|
||||||
None, # ....................................................................... 'p_display_picture'
|
|
||||||
user_identifier, # ............................................................ 'p_token_id'
|
|
||||||
json.to_string(python_data = {"email": token["email"]}, no_space = True), # ... 'p_notes'
|
|
||||||
mongo_json["user"]["userId"] # ................................................ 'p_created_by'
|
|
||||||
),
|
|
||||||
session_token = session_token
|
|
||||||
)
|
|
||||||
if db_json["status"] == 1: token_saved = True
|
|
||||||
|
|
||||||
# Done here:
|
|
||||||
return token_saved
|
|
||||||
|
|
||||||
async def get_token(
|
|
||||||
self,
|
|
||||||
mongo_conn: AsyncMongo,
|
|
||||||
user_identifier: ObjectId | str = None,
|
|
||||||
**kwargs
|
|
||||||
) -> dict | None:
|
|
||||||
|
|
||||||
"""
|
|
||||||
To retrieve stored tokens from the database.
|
|
||||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
|
||||||
:param user_identifier: The identifier granted by the 'get_user_identifier' method.
|
|
||||||
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
|
|
||||||
info, the client, the type of authentication used, or even the kind of service.
|
|
||||||
:return: The retrieved record that has the token, and information about the service and client if found, else
|
|
||||||
None when there is no matching record.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# Build the filter:
|
|
||||||
filter_json = {k: v for k, v in kwargs.items()}
|
|
||||||
if user_identifier: filter_json["_id"] = ObjectId(user_identifier)
|
|
||||||
|
|
||||||
# If there is no search criteria, we exit with failure:
|
|
||||||
if not filter_json: return None
|
|
||||||
|
|
||||||
# If there is some filtering possible,
|
|
||||||
# we fetch and return the token:
|
|
||||||
return await mongo_conn.find_one(
|
|
||||||
collection = self.AUTH_COLLECTION,
|
|
||||||
filter = filter_json,
|
|
||||||
projection = {
|
|
||||||
"_id": True,
|
|
||||||
"serviceType": True,
|
|
||||||
"authType": True,
|
|
||||||
"client": True,
|
|
||||||
"token": True
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** MAIN PROGRAM ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
|
|
||||||
pass
|
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
AUTHOR:
|
||||||
|
|
||||||
|
Khushal P Soonderji
|
||||||
|
|
||||||
|
DATE:
|
||||||
|
|
||||||
|
Tuesday, 3rd Dec., 2024
|
||||||
|
|
||||||
|
OBJECTIVE:
|
||||||
|
|
||||||
|
To enlist and retrieve mails for various filtering conditions.
|
||||||
|
|
||||||
|
REFERENCES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
DOWNLOADS:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** IMPORT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# To make sibling directories accessible for imports:
|
||||||
|
import sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
# My async utils:
|
||||||
|
from utils_v2.string import json
|
||||||
|
from utils_v2.date_time import date_time
|
||||||
|
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||||
|
|
||||||
|
# Base model:
|
||||||
|
from models.behaviour.base import BaseModel
|
||||||
|
|
||||||
|
# To work with MongoDB:
|
||||||
|
from bson import ObjectId
|
||||||
|
|
||||||
|
# To work with datatypes:
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
# For asynchronous activities:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MACROS / ONE-TIME INIT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** VARIABLES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** FUNCTIONS ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** CLASSES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
class MailRetrieveModel(BaseModel):
|
||||||
|
|
||||||
|
# For MongoDB:
|
||||||
|
AUTH_COLLECTION = "_authTokens"
|
||||||
|
MAIL_COLLECTION = "_messages"
|
||||||
|
|
||||||
|
async def get_mail(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongo,
|
||||||
|
mail_identifier: str | ObjectId
|
||||||
|
):
|
||||||
|
|
||||||
|
"""
|
||||||
|
Retrieves one full mail from the database.
|
||||||
|
:param mongo_conn: The instance of the database connector to use to get the mail's data.
|
||||||
|
:param mail_identifier: The '_id' of the document that holds the mail.
|
||||||
|
:return: Either the JSON that describes the mail or None if such a mail does not exist.
|
||||||
|
"""
|
||||||
|
|
||||||
|
mail_data = await mongo_conn.find_one(
|
||||||
|
collection = self.MAIL_COLLECTION,
|
||||||
|
filter = {"_id": ObjectId(mail_identifier)},
|
||||||
|
projection = {
|
||||||
|
"mailId": "_id",
|
||||||
|
"serviceType": True,
|
||||||
|
"client": True,
|
||||||
|
"payload.ts": True,
|
||||||
|
"payload.readTs": True,
|
||||||
|
"payload.from": True,
|
||||||
|
"payload.to": True,
|
||||||
|
"payload.cc": True,
|
||||||
|
"payload.bcc": True,
|
||||||
|
"payload.parts": True,
|
||||||
|
"payload.attachments": True,
|
||||||
|
"payload.labels": True,
|
||||||
|
"payload.snippet": True,
|
||||||
|
"payload.aiSnippet": "payload.aiSnippet.snippet",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if mail_data: mail_data["mailId"] = str(mail_data["mailId"])
|
||||||
|
return mail_data
|
||||||
|
|
||||||
|
async def list_by_account_identifier(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongo,
|
||||||
|
account_identifier: str | ObjectId,
|
||||||
|
limit: int = 25,
|
||||||
|
skip: int = 0
|
||||||
|
):
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MAIN PROGRAM ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
pass
|
||||||
@@ -1,327 +0,0 @@
|
|||||||
"""
|
|
||||||
|
|
||||||
AUTHOR:
|
|
||||||
|
|
||||||
Khushal P Soonderji
|
|
||||||
|
|
||||||
DATE:
|
|
||||||
|
|
||||||
Wednesday, 27th Nov., 2024
|
|
||||||
|
|
||||||
OBJECTIVE:
|
|
||||||
|
|
||||||
To define the interaction between the UI layer and the database connectivity in one place. Here we shall handle
|
|
||||||
all the activities for OAuth2.0 authorization requests for all the users of our service.
|
|
||||||
|
|
||||||
REFERENCES:
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
DOWNLOADS:
|
|
||||||
|
|
||||||
N/A
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** IMPORT ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# To make sibling directories accessible for imports:
|
|
||||||
import sys
|
|
||||||
|
|
||||||
from langchain.chains.summarize.stuff_prompt import prompt_template
|
|
||||||
from sqlalchemy.orm.collections import collection
|
|
||||||
|
|
||||||
sys.path.append(".")
|
|
||||||
sys.path.append("..")
|
|
||||||
|
|
||||||
# My async utils:
|
|
||||||
from utils_v2.string import json
|
|
||||||
from utils_v2.date_time import date_time
|
|
||||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
|
||||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
|
||||||
|
|
||||||
# Mail Clients:
|
|
||||||
from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
|
|
||||||
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
|
|
||||||
|
|
||||||
# Base model:
|
|
||||||
from models.behaviour.base import BaseModel
|
|
||||||
|
|
||||||
# To work with MongoDB:
|
|
||||||
from bson import ObjectId
|
|
||||||
from pymongo import InsertOne, UpdateOne
|
|
||||||
|
|
||||||
# To work with LLMs:
|
|
||||||
from langchain_openai import ChatOpenAI
|
|
||||||
from langchain_core.prompts import ChatPromptTemplate
|
|
||||||
|
|
||||||
# To work with datatypes:
|
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
# To make deep-copies:
|
|
||||||
import copy
|
|
||||||
|
|
||||||
# To work with date and time:
|
|
||||||
import datetime
|
|
||||||
|
|
||||||
# For asynchronous activities:
|
|
||||||
import asyncio
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** MACROS / ONE-TIME INIT ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# --- Nothing Yet
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** VARIABLES ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# --- Nothing Yet
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** FUNCTIONS ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
# --- Nothing Yet
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** CLASSES ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
class MailSyncModel(BaseModel):
|
|
||||||
|
|
||||||
# For MongoDB:
|
|
||||||
AUTH_COLLECTION = "_authTokens"
|
|
||||||
MAIL_COLLECTION = "_messages"
|
|
||||||
|
|
||||||
# For AI Magic through LLMs:
|
|
||||||
prompt_template = ChatPromptTemplate.from_messages([
|
|
||||||
(
|
|
||||||
"system",
|
|
||||||
"You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPs) FROM THE SUMMARY."
|
|
||||||
),
|
|
||||||
(
|
|
||||||
"user",
|
|
||||||
"Please summarize this mail: \"\"\"{mail}\"\"\""
|
|
||||||
)
|
|
||||||
])
|
|
||||||
|
|
||||||
async def __sync_one(
|
|
||||||
self,
|
|
||||||
mongo_conn: AsyncMongo,
|
|
||||||
user_info: dict,
|
|
||||||
mail_client: AsyncGMailClient,
|
|
||||||
tokens: GoogleAuthTokens,
|
|
||||||
message_id: str,
|
|
||||||
llm: ChatOpenAI = None,
|
|
||||||
force_sync: bool = False
|
|
||||||
) -> UpdateOne | None:
|
|
||||||
|
|
||||||
# ┏┓ ┓┏ • ┓ ┓
|
|
||||||
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
|
|
||||||
# ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
|
|
||||||
# ┛
|
|
||||||
|
|
||||||
mail_payload = None
|
|
||||||
mail_client_name = None
|
|
||||||
|
|
||||||
# ┏┓┓ ┓ ┏┓ • • ┳┓ ┓
|
|
||||||
# ┃ ┣┓┏┓┏┃┏ ┣ ┓┏┓┏╋┓┏┓┏┓ ┣┫┏┓┏┏┓┏┓┏┫┏
|
|
||||||
# ┗┛┛┗┗ ┗┛┗ ┗┛┛┗┗┛┗┗┛┗┗┫ ┛┗┗ ┗┗┛┛ ┗┻┛
|
|
||||||
# ┛
|
|
||||||
|
|
||||||
# Check if you already have that mail in your database:
|
|
||||||
existing_record = await mongo_conn.find_one(
|
|
||||||
collection = self.MAIL_COLLECTION,
|
|
||||||
filter = {
|
|
||||||
"messageType": "email",
|
|
||||||
"$or": [
|
|
||||||
{"payload.messageId": message_id}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
projection = {"_id": True}
|
|
||||||
)
|
|
||||||
|
|
||||||
# If there already exists such a record, and we haven't been forced to re-sync it:
|
|
||||||
if existing_record and not force_sync: return mail_payload
|
|
||||||
|
|
||||||
# ┏┓┳┳┓ •┓
|
|
||||||
# ┃┓┃┃┃┏┓┓┃
|
|
||||||
# ┗┛┛ ┗┗┻┗┗
|
|
||||||
|
|
||||||
if isinstance(mail_client, AsyncGMailClient):
|
|
||||||
|
|
||||||
# Note down the name of the mail client:
|
|
||||||
mail_client_name = "gmail"
|
|
||||||
|
|
||||||
# Fetch the formatted mail message:
|
|
||||||
client_response = await mail_client.get_message(
|
|
||||||
tokens = tokens,
|
|
||||||
message_id = message_id,
|
|
||||||
return_raw = False
|
|
||||||
)
|
|
||||||
|
|
||||||
# If the fetch was successful:
|
|
||||||
if client_response.success:
|
|
||||||
|
|
||||||
# Summarize the content:
|
|
||||||
if llm:
|
|
||||||
prompt = self.prompt_template.invoke({"mail": client_response.data["unformattedText"]})
|
|
||||||
llm_response = await llm.ainvoke(prompt)
|
|
||||||
client_response.data["aiSnippet"] = llm_response.content
|
|
||||||
|
|
||||||
# Note down the response:
|
|
||||||
mail_payload = client_response.data
|
|
||||||
|
|
||||||
# ┳┓
|
|
||||||
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
|
|
||||||
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
|
|
||||||
# ┛
|
|
||||||
|
|
||||||
if mail_payload:
|
|
||||||
return UpdateOne(
|
|
||||||
filter = {
|
|
||||||
"messageType": "email",
|
|
||||||
"$or": [
|
|
||||||
{"payload.messageId": message_id}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
update = {
|
|
||||||
"$set": {
|
|
||||||
"readTs": date_time.get_current_utc_date_time(),
|
|
||||||
"user": user_info,
|
|
||||||
"messageType": "email",
|
|
||||||
"connector": mail_client_name,
|
|
||||||
"payload": mail_payload
|
|
||||||
}
|
|
||||||
},
|
|
||||||
upsert = True
|
|
||||||
)
|
|
||||||
|
|
||||||
# Done here:
|
|
||||||
return mail_payload
|
|
||||||
|
|
||||||
async def sync(
|
|
||||||
self,
|
|
||||||
mongo_conn: AsyncMongo,
|
|
||||||
user_info: dict,
|
|
||||||
mail_client: AsyncGMailClient,
|
|
||||||
tokens: GoogleAuthTokens,
|
|
||||||
llm: ChatOpenAI = None,
|
|
||||||
force_sync: bool = False,
|
|
||||||
start_date: datetime.datetime = None,
|
|
||||||
end_date: datetime.datetime = None,
|
|
||||||
max_count: int = 100
|
|
||||||
) -> int:
|
|
||||||
|
|
||||||
# Start by assuming failure:
|
|
||||||
mails_count = 0
|
|
||||||
|
|
||||||
# ┏┓┳┳┓ •┓
|
|
||||||
# ┃┓┃┃┃┏┓┓┃
|
|
||||||
# ┗┛┛ ┗┗┻┗┗
|
|
||||||
|
|
||||||
if isinstance(mail_client, AsyncGMailClient):
|
|
||||||
|
|
||||||
# Enlist all the labels, we need to find the label that indicates that we've read the mail:
|
|
||||||
client_response = await mail_client.list_labels(tokens = tokens)
|
|
||||||
if not client_response.success: return mails_count
|
|
||||||
labels = client_response.data
|
|
||||||
custom_label = "Sync'd with TheCAOffice"
|
|
||||||
custom_label_id = labels.get(custom_label)
|
|
||||||
if custom_label_id is None:
|
|
||||||
client_response = await mail_client.create_label(
|
|
||||||
tokens = tokens,
|
|
||||||
label_name = custom_label,
|
|
||||||
label_visibility = "labelHide"
|
|
||||||
)
|
|
||||||
if not client_response.success: return mails_count
|
|
||||||
custom_label_id = client_response.data["id"]
|
|
||||||
|
|
||||||
# Build the query:
|
|
||||||
sub_queries = [f"-label:\"{custom_label}\""]
|
|
||||||
if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
|
|
||||||
if end_date: sub_queries.append(end_date.strftime("before:%Y/%m/%d"))
|
|
||||||
print("Q:", " ".join(sub_queries))
|
|
||||||
|
|
||||||
# Get a list of all the mails:
|
|
||||||
client_response = await mail_client.list_messages(
|
|
||||||
tokens = tokens,
|
|
||||||
max_count = max_count,
|
|
||||||
# query = " ".join(sub_queries)
|
|
||||||
)
|
|
||||||
if not client_response.success: return mails_count
|
|
||||||
messages_list = client_response.data["messages"]
|
|
||||||
print(messages_list)
|
|
||||||
|
|
||||||
# Create MongoDB operations for all the mails:
|
|
||||||
tasks = [
|
|
||||||
self.__sync_one(
|
|
||||||
mongo_conn = mongo_conn,
|
|
||||||
user_info = user_info,
|
|
||||||
mail_client = mail_client,
|
|
||||||
tokens = tokens,
|
|
||||||
message_id = v["id"],
|
|
||||||
llm = llm,
|
|
||||||
force_sync = force_sync
|
|
||||||
)
|
|
||||||
for k, v in messages_list.items()
|
|
||||||
]
|
|
||||||
mongo_operations = await asyncio.gather(*tasks)
|
|
||||||
mongo_operations = [mo for mo in mongo_operations if mo is not None]
|
|
||||||
|
|
||||||
# Write the mails to MongoDB:
|
|
||||||
mails_count = await mongo_conn.bulk_write(
|
|
||||||
collection = self.MAIL_COLLECTION,
|
|
||||||
requests = mongo_operations
|
|
||||||
)
|
|
||||||
print("MAILS COUNT:", mails_count)
|
|
||||||
|
|
||||||
# If all the mails were sync'd properly:
|
|
||||||
client_response = await mail_client.modify_messages(
|
|
||||||
tokens = tokens,
|
|
||||||
message_ids = [v["id"] for k, v in messages_list.items()],
|
|
||||||
add_label_ids = [custom_label_id]
|
|
||||||
)
|
|
||||||
|
|
||||||
# ┳┓
|
|
||||||
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
|
|
||||||
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
|
|
||||||
# ┛
|
|
||||||
|
|
||||||
# Done here:
|
|
||||||
return mails_count
|
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
# ***** ****
|
|
||||||
# *** MAIN PROGRAM ***
|
|
||||||
# ***** ****
|
|
||||||
# *****************************************************************************************************************
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
|
|
||||||
pass
|
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
AUTHOR:
|
||||||
|
|
||||||
|
Khushal P Soonderji
|
||||||
|
|
||||||
|
DATE:
|
||||||
|
|
||||||
|
Tuesday, 3rd Dec., 2024.
|
||||||
|
|
||||||
|
OBJECTIVE:
|
||||||
|
|
||||||
|
To provide a structure to query the full payload of an email.
|
||||||
|
|
||||||
|
REFERENCES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
DOWNLOADS:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** IMPORT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# To make sibling directories accessible for imports:
|
||||||
|
import sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
# For making data behaviour_models:
|
||||||
|
from pydantic import BaseModel, Field, field_validator, PastDatetime
|
||||||
|
from typing import Optional, Literal
|
||||||
|
|
||||||
|
# My utils:
|
||||||
|
from utils_v2.string import regex
|
||||||
|
from utils_v2.date_time import date_time
|
||||||
|
|
||||||
|
# To work with date and time:
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MACROS / ONE-TIME INIT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# RegEx Patterns:
|
||||||
|
REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** VARIABLES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** FUNCTIONS ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
class MailGetRequestHeaders(BaseModel):
|
||||||
|
|
||||||
|
sessionToken: str = Field(
|
||||||
|
description = "the session token of the user who is requesting the service",
|
||||||
|
pattern = REGEX_SESSION_TOKEN,
|
||||||
|
frozen = True,
|
||||||
|
alias = "X-Session-Token"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "allow"
|
||||||
|
|
||||||
|
def model_dump(self, *args, **kwargs):
|
||||||
|
return super().model_dump(*args, by_alias = True, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class MailGetRequestData(BaseModel):
|
||||||
|
|
||||||
|
mailId: str = Field(
|
||||||
|
description = "the mail identifier (Mongo ObjectId) of the document that holds the mail",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "forbid"
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MAIN PROGRAM ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
pass
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
AUTHOR:
|
||||||
|
|
||||||
|
Khushal P Soonderji
|
||||||
|
|
||||||
|
DATE:
|
||||||
|
|
||||||
|
Monday, 2nd Dec., 2024.
|
||||||
|
|
||||||
|
OBJECTIVE:
|
||||||
|
|
||||||
|
To provide the structure for the request that will come in to sync the mails of a particular user.
|
||||||
|
|
||||||
|
REFERENCES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
DOWNLOADS:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** IMPORT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# To make sibling directories accessible for imports:
|
||||||
|
import sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
# For making data behaviour_models:
|
||||||
|
from pydantic import BaseModel, Field, field_validator, PastDatetime
|
||||||
|
from typing import Optional, Literal
|
||||||
|
|
||||||
|
# My utils:
|
||||||
|
from utils_v2.string import regex
|
||||||
|
from utils_v2.date_time import date_time
|
||||||
|
|
||||||
|
# To work with date and time:
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MACROS / ONE-TIME INIT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# RegEx Patterns:
|
||||||
|
REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** VARIABLES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** FUNCTIONS ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
class MailSyncRequestHeaders(BaseModel):
|
||||||
|
|
||||||
|
sessionToken: str = Field(
|
||||||
|
description = "the session token of the user who is requesting the service",
|
||||||
|
pattern = REGEX_SESSION_TOKEN,
|
||||||
|
frozen = True,
|
||||||
|
alias = "X-Session-Token"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "allow"
|
||||||
|
|
||||||
|
def model_dump(self, *args, **kwargs):
|
||||||
|
return super().model_dump(*args, by_alias = True, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class MailSyncRequestData(BaseModel):
|
||||||
|
|
||||||
|
accountId: str = Field(
|
||||||
|
description = "the account identifier (Mongo ObjectId) granted by 'MailOAuthModel.get_account_identifier'",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
maxCount: int = Field(
|
||||||
|
description = "the max. no. of e-mails to sync at a given time",
|
||||||
|
default = 100,
|
||||||
|
ge = 1,
|
||||||
|
le = 100,
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
startDate: PastDatetime = Field(
|
||||||
|
description = "the starting date from which the user wants to sync their mail",
|
||||||
|
default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1),
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
endDate: PastDatetime = Field(
|
||||||
|
description = "the ending date till which the user wants to sync their mail",
|
||||||
|
default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1),
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
forceSync: bool = Field(
|
||||||
|
description = "use this to forcefully re-sync mails when you need to overwrite existing data in mongodb",
|
||||||
|
default = False
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "forbid"
|
||||||
|
|
||||||
|
# ┓┏ ┓• ┓ •
|
||||||
|
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
|
||||||
|
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
|
||||||
|
|
||||||
|
@field_validator("startDate", "endDate", mode = "before")
|
||||||
|
def to_datetime(cls, value):
|
||||||
|
if not isinstance(value, datetime.datetime):
|
||||||
|
value = date_time.parse_date_time(
|
||||||
|
input_value = value,
|
||||||
|
timezone = date_time.TIMEZONE_UTC
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class MailSyncOneResult(BaseModel):
|
||||||
|
|
||||||
|
success: bool = Field(
|
||||||
|
description = "whether, or not, the mail was successfully sync'd",
|
||||||
|
default = False
|
||||||
|
)
|
||||||
|
|
||||||
|
message: str | None = Field(
|
||||||
|
description = "a brief message to summarize the result of the process",
|
||||||
|
default = None
|
||||||
|
)
|
||||||
|
|
||||||
|
mailMessage: dict | None = Field(
|
||||||
|
description = "the actual data of the mail; can be null in a successful process if the mail is already sync'd",
|
||||||
|
default = None
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "forbid"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class MailSyncManyResults(BaseModel):
|
||||||
|
|
||||||
|
totalCount: int = Field(
|
||||||
|
description = "the total no. of mails that were to be sync'd",
|
||||||
|
default = 0
|
||||||
|
)
|
||||||
|
|
||||||
|
successCount: int = Field(
|
||||||
|
description = "the no. of mails that were successfully sync'd",
|
||||||
|
default = 0
|
||||||
|
)
|
||||||
|
|
||||||
|
failureCount: int = Field(
|
||||||
|
description = "the no. of mails that were successfully sync'd",
|
||||||
|
default = 0
|
||||||
|
)
|
||||||
|
|
||||||
|
message: str = Field(
|
||||||
|
description = "a brief message to summarize the results of the process",
|
||||||
|
default = None
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "forbid"
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MAIN PROGRAM ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
pass
|
||||||
@@ -136,17 +136,22 @@ class AuthDetailsIncompleteException(Exception):
|
|||||||
# *****************************************************************************************************************
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
async def make_ordered_json(json_data, http_code = 200):
|
async def make_ordered_json(
|
||||||
|
json_data,
|
||||||
|
no_space = True,
|
||||||
|
http_code = 200
|
||||||
|
):
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Quart sorts the fields of a dict when converting to a JSON response. Here we are manually making the response when
|
Quart sorts the fields of a dict when converting to a JSON response. Here we are manually making the response when
|
||||||
the sequence of the fields is sensitive.
|
the sequence of the fields is sensitive.
|
||||||
:param json_data: The data (dict, list, etc.) to be converted to a JSON string.
|
:param json_data: The data (dict, list, etc.) to be converted to a JSON string.
|
||||||
|
:param no_space: Set this to True to remove all excess white spaces from the JSON string. Saves bandwidth.
|
||||||
:param http_code: The HTTP status code you want to send with the response.
|
:param http_code: The HTTP status code you want to send with the response.
|
||||||
:return: The JSON-ified response such that the sequence of the fields is maintained.
|
:return: The JSON-ified response such that the sequence of the fields is maintained.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
response = await make_response(json.to_string(json_data, no_space = True), http_code)
|
response = await make_response(json.to_string(json_data, no_space = no_space), http_code)
|
||||||
response.headers["Content-Type"] = "application/json"
|
response.headers["Content-Type"] = "application/json"
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user