diff --git a/api/blueprints/mail/list.py b/api/blueprints/mail/list.py index e8d8712..8889864 100644 --- a/api/blueprints/mail/list.py +++ b/api/blueprints/mail/list.py @@ -10,7 +10,8 @@ OBJECTIVE: - To enlist multiple e-mails for a given user at a time. + To list e-mails by their account identifier. Remember that the 'account identifier' is the '_id' of the document + in MongoDB that holds the tokens to authorize the e-mail id whose mails are being accessed. REFERENCES: @@ -48,6 +49,7 @@ from utils_v2.database.async_mongo_v2 import AsyncMongo from utils_v2.api.codes import StatusCodes, HttpCodes from utils_v2.api.response import ResponseModel from utils_v2.api.async_quart import ( + make_ordered_json, set_api_version, read_input, get_session_info, @@ -68,8 +70,7 @@ from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens from shared import constants # Data Models: -from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData -from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults +from models.data.mail.list import MailListRequestHeaders, MailListByAccountIdRequestData # To work with datatypes: from typing import Literal @@ -77,9 +78,6 @@ from typing import Literal # For asynchronous activities: import asyncio -# To work with LLMs: -from langchain_openai import ChatOpenAI - # To work with date and time: import datetime @@ -92,7 +90,7 @@ import datetime # Related to Quart: -mail_sync_bp = Blueprint("mail_sync", __name__) +mail_retrieve_bp = Blueprint("mail_retrieve", __name__) # ***************************************************************************************************************** @@ -112,7 +110,7 @@ mail_sync_bp = Blueprint("mail_sync", __name__) # ***************************************************************************************************************** -@mail_sync_bp.record_once +@mail_retrieve_bp.record_once def init(blueprint_setup_state): # This gets called when the blueprint is registered. @@ -123,41 +121,7 @@ def init(blueprint_setup_state): # --------------------------------------------------------------------------------------------------------------------- -async def sync_mails( - mongo_conn: AsyncMongo, - llm: ChatOpenAI, - inbound_headers: dict, - inbound_data: MailSyncRequestData -) -> MailSyncManyResults: - - """ - A very simple function, but kept separate so that we get the option to switch between running it in the foreground - and running it in the background. - :param mongo_conn: The instance of the database connector to use to sync the mails. - :param llm: The instance of the LLM to use to summarize the mails. - :param inbound_headers: The headers that came in with the request. - :param inbound_data: The data that came in with the request. - :return: The results of the mail-sync'ing attempt. - """ - - # Try to sync the mails: - return await current_app.mail_sync_model.sync( - session_token = inbound_headers["X-Session-Token"], - mongo_conn = mongo_conn, - account_identifier = inbound_data.accountId, - llm = llm, - force_sync = inbound_data.forceSync, - start_date = inbound_data.startDate, - end_date = inbound_data.endDate, - max_count = inbound_data.maxCount - ) - - -# --------------------------------------------------------------------------------------------------------------------- - - -@mail_sync_bp.route("/sync", methods = ["POST"]) -@mail_sync_bp.route("/sync/", methods = ["POST"]) +@mail_retrieve_bp.route("/list/account/id", methods = ["GET"]) @set_api_version(api_version = "1.0.0") @read_input(sanitize_headers = False, sanitize_data = False) @get_session_info(key = "X-Session-Token", session_coro = "get_session") @@ -165,7 +129,7 @@ async def sync_mails( attr_name = "logs_mongo", project = constants.PROJECT_NAME, log_type = constants.MODULE_NAME, - operation = "mailOAuthUrlReqApi", + operation = "mailListByAccIdApi", log_input = True, log_output = True, sensitive_keys = ["sessionToken", "X-Session-Token"] @@ -173,22 +137,19 @@ async def sync_mails( @log_chain_to_mongo(attr_name = "logs_mongo") @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @validate_input( - header_validator = lambda x: MailSyncRequestHeaders(**x).model_dump(), - data_validator = lambda x: MailSyncRequestData(**x) + header_validator = lambda x: MailGetRequestHeaders(**x).model_dump(), + data_validator = lambda x: MailGetRequestData(**x) ) @handle_cancelled_request() -async def sync_mail( - mode: Literal["background", "bg"] = None, - inbound_headers: dict | MailSyncRequestHeaders = None, - inbound_data: dict | MailSyncRequestData = None, +async def get_one_mail( + inbound_headers: dict | MailGetRequestHeaders = None, + inbound_data: dict | MailGetRequestData = None, inbound_files: dict = None, **kwargs ): """ - Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for - ready access on the UI. - :param mode: Set it to one of the specified options to make the sync'ing process go to the background. + Use this endpoint when the user wants to fetch one mail. :param inbound_headers: auto-extracted by the decorators. :param inbound_data: auto-extracted by the decorators. :param inbound_files: auto-extracted by the decorators. @@ -203,43 +164,17 @@ async def sync_mail( http_code = HttpCodes.UNAUTHORIZED ) - # Make the variables available in the scope of the current request: - g.inbound_headers = inbound_headers - g.inbound_data = inbound_data - - # If we've been asked to sync the mails in the background: - if mode in ["background", "bg"]: - current_app.add_background_task( - sync_mails, - mongo_conn = current_app.data_mongo, - llm = current_app.llm, - inbound_headers = inbound_headers, - inbound_data = inbound_data - ) - return ResponseModel( - status_code = StatusCodes.OK, - http_code = HttpCodes.ACCEPTED, - message = "your mails are being sync'd in the background" - ) - - # Otherwise we process it right here: - sync_results = await sync_mails( + # Get the mail: + mail_data = await current_app.mail_retrieve_model.get_mail( mongo_conn = current_app.data_mongo, - llm = current_app.llm, - inbound_headers = inbound_headers, - inbound_data = inbound_data + mail_identifier = inbound_data.mailId ) - # Response: + # Done here: return ResponseModel( - status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK, - http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS, - message = sync_results.message, - data = { - "totalCount": sync_results.totalCount, - "successCount": sync_results.successCount, - "failureCount": sync_results.failureCount - } + status_code = StatusCodes.OK if mail_data else StatusCodes.FAILED, + http_code = HttpCodes.SUCCESS if mail_data else HttpCodes.NOT_FOUND, + data = mail_data ) diff --git a/api/blueprints/mail/retrieve.py b/api/blueprints/mail/retrieve.py new file mode 100644 index 0000000..e8d8712 --- /dev/null +++ b/api/blueprints/mail/retrieve.py @@ -0,0 +1,255 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 3rd Dec., 2024 + + OBJECTIVE: + + To enlist multiple e-mails for a given user at a time. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For using Quart: +from quart import Blueprint, current_app, g, request + +# My utils: +from utils_v2.string import json +from utils_v2.database.async_mongo_v2 import AsyncMongo +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.response import ResponseModel +from utils_v2.api.async_quart import ( + set_api_version, + read_input, + get_session_info, + log_request_to_mongo, + log_chain_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input, + handle_cancelled_request +) + +# GMail-related utils: +from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT +from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens + +# Common: +from shared import constants + +# Data Models: +from models.data.mail.sync import MailSyncRequestHeaders, MailSyncRequestData +from models.data.mail.sync import MailSyncOneResult, MailSyncManyResults + +# To work with datatypes: +from typing import Literal + +# For asynchronous activities: +import asyncio + +# To work with LLMs: +from langchain_openai import ChatOpenAI + +# To work with date and time: +import datetime + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Related to Quart: +mail_sync_bp = Blueprint("mail_sync", __name__) + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@mail_sync_bp.record_once +def init(blueprint_setup_state): + + # This gets called when the blueprint is registered. + # Consider this to be a one-time setup for the whole blueprint: + pass + + +# --------------------------------------------------------------------------------------------------------------------- + + +async def sync_mails( + mongo_conn: AsyncMongo, + llm: ChatOpenAI, + inbound_headers: dict, + inbound_data: MailSyncRequestData +) -> MailSyncManyResults: + + """ + A very simple function, but kept separate so that we get the option to switch between running it in the foreground + and running it in the background. + :param mongo_conn: The instance of the database connector to use to sync the mails. + :param llm: The instance of the LLM to use to summarize the mails. + :param inbound_headers: The headers that came in with the request. + :param inbound_data: The data that came in with the request. + :return: The results of the mail-sync'ing attempt. + """ + + # Try to sync the mails: + return await current_app.mail_sync_model.sync( + session_token = inbound_headers["X-Session-Token"], + mongo_conn = mongo_conn, + account_identifier = inbound_data.accountId, + llm = llm, + force_sync = inbound_data.forceSync, + start_date = inbound_data.startDate, + end_date = inbound_data.endDate, + max_count = inbound_data.maxCount + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@mail_sync_bp.route("/sync", methods = ["POST"]) +@mail_sync_bp.route("/sync/", methods = ["POST"]) +@set_api_version(api_version = "1.0.0") +@read_input(sanitize_headers = False, sanitize_data = False) +@get_session_info(key = "X-Session-Token", session_coro = "get_session") +@log_request_to_mongo( + attr_name = "logs_mongo", + project = constants.PROJECT_NAME, + log_type = constants.MODULE_NAME, + operation = "mailOAuthUrlReqApi", + log_input = True, + log_output = True, + sensitive_keys = ["sessionToken", "X-Session-Token"] +) +@log_chain_to_mongo(attr_name = "logs_mongo") +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@validate_input( + header_validator = lambda x: MailSyncRequestHeaders(**x).model_dump(), + data_validator = lambda x: MailSyncRequestData(**x) +) +@handle_cancelled_request() +async def sync_mail( + mode: Literal["background", "bg"] = None, + inbound_headers: dict | MailSyncRequestHeaders = None, + inbound_data: dict | MailSyncRequestData = None, + inbound_files: dict = None, + **kwargs +): + + """ + Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for + ready access on the UI. + :param mode: Set it to one of the specified options to make the sync'ing process go to the background. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If the session token is invalid/expired: + if kwargs.get("session_info") is None: + return ResponseModel( + status_code = StatusCodes.FAILED, + http_code = HttpCodes.UNAUTHORIZED + ) + + # Make the variables available in the scope of the current request: + g.inbound_headers = inbound_headers + g.inbound_data = inbound_data + + # If we've been asked to sync the mails in the background: + if mode in ["background", "bg"]: + current_app.add_background_task( + sync_mails, + mongo_conn = current_app.data_mongo, + llm = current_app.llm, + inbound_headers = inbound_headers, + inbound_data = inbound_data + ) + return ResponseModel( + status_code = StatusCodes.OK, + http_code = HttpCodes.ACCEPTED, + message = "your mails are being sync'd in the background" + ) + + # Otherwise we process it right here: + sync_results = await sync_mails( + mongo_conn = current_app.data_mongo, + llm = current_app.llm, + inbound_headers = inbound_headers, + inbound_data = inbound_data + ) + + # Response: + return ResponseModel( + status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK, + http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS, + message = sync_results.message, + data = { + "totalCount": sync_results.totalCount, + "successCount": sync_results.successCount, + "failureCount": sync_results.failureCount + } + ) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/models/behaviour/mail/oauth.py b/models/behaviour/mail/oauth.py deleted file mode 100644 index fecc8ef..0000000 --- a/models/behaviour/mail/oauth.py +++ /dev/null @@ -1,312 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Wednesday, 27th Nov., 2024 - - OBJECTIVE: - - To define the interaction between the UI layer and the database connectivity in one place. Here we shall handle - all the activities for OAuth2.0 authorization requests for all the users of our service. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# My async utils: -from utils_v2.string import json -from utils_v2.date_time import date_time -from utils_v2.database.async_mysql_v2 import AsyncMySQL -from utils_v2.database.async_mongo_v2 import AsyncMongo - -# Base model: -from models.behaviour.base import BaseModel - -# To work with MongoDB: -from bson import ObjectId - -# To work with datatypes: -from typing import Literal - -# To make deep-copies: -import copy - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class MailOAuthModel(BaseModel): - - AUTH_COLLECTION = "_authTokens" - - async def get_user_identifier( - self, - db_conn: AsyncMySQL, - mongo_conn: AsyncMongo, - user_info: dict, - service_client: Literal["gmail"], - auth_type: Literal["oauth"], - session_token: str = None - ) -> ObjectId: - - """ - Stores params from the session info and gives an identifier to use in the authorization URL. Use this when the - user requests an authorization URL to link your service to another service (like GMail). - :param db_conn: The database connection (MariaDB) to use to perform the action. - :param mongo_conn: The database connection (MongoDB) to use to perform the action. - :param user_info: The dictionary that has the user's session information. - :param service_client: The name of the company or brand that is providing this service that is being integrated. - :param auth_type: To identify the type of authentication being done here. This could indicate simple password - authentication, more advance OAuth2.0 authentication, etc. - :param session_token: The session token of the user who requested this service. - :return: An ObjectId to later store the granted tokens. - """ - - # Note down the timestamp at which this event occurred: - request_ts = date_time.get_current_utc_date_time(as_string = False) - - # Get the identifier from the database: - inserted_id = await mongo_conn.insert_one( - collection = MailOAuthModel.AUTH_COLLECTION, - document = { - "version": "-1.0.1", - "serviceType": "email", - "client": service_client, - "authType": auth_type, - "user": user_info, - "token": None, - "firstRefreshTs": None, - "lastRefreshTs": None, - "lastRequestTs": request_ts - } - ) - return inserted_id - - # # Get the identifier from the database: - # mongo_json = await mongo_conn.find_one_and_update( - # collection = MailOAuthModel.AUTH_COLLECTION, - # filter = { - # "serviceType": "email", - # "client": service_client, - # "authType": auth_type, - # "user": user_info, - # }, - # update = { - # "$set": { - # "lastRequestTs": request_ts - # }, - # "$setOnInsert": { - # "version": "1.0.0", - # "serviceType": "email", - # "client": service_client, - # "authType": auth_type, - # "user": user_info, - # "token": None, - # "firstRefreshTs": None, - # "lastRefreshTs": None, - # "firstRequestTs": request_ts, - # } - # }, - # projection = { - # "_id": True - # }, - # upsert = True, - # return_updated = True - # ) - # - # # Tell MariaDB that an authorization request was initiated: - # db_json = {} - # if mongo_json is not None: - # db_json = await self.call_procedure( - # db_conn = db_conn, - # proc_name = "entity_integration_save", - # proc_args = ( - # user_info["entityId"], # ............................................ 'p_entity_id' - # service_client, # ................................................... 'p_provider' - # "Auth Requested", # ................................................. 'p_current_status' - # "Auth URL Generated", # ............................................. 'p_last_action' - # None, # ............................................................. 'p_display_name' - # None, # ............................................................. 'p_display_picture' - # str(mongo_json["_id"]), # ........................................... 'p_token_id' - # json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes' - # user_info["userId"] # ............................................... 'p_created_by' - # ), - # session_token = session_token - # ) - # - # # Done here: - # return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None - - async def set_token( - self, - db_conn: AsyncMySQL, - mongo_conn: AsyncMongo, - user_identifier: ObjectId | str, - token: dict, - session_token: str = None - ) -> bool: - - """ - This method is to be called when the end user authorizes your service to connect to his third-party account. For - example, when the end user allows you to access his GMail account. USE THIS FOR UPDATING (REFRESHING) TOKENS - ALSO. - :param db_conn: The database connection (MariaDB) to use to perform the action. - :param mongo_conn: The database connection (MongoDB) to use to perform the action. - :param user_identifier: The identifier granted by the 'get_user_identifier' method. - :param token: The token granted by the third-party service. - :param session_token: The session token of the user who requested this service. - :return: True if saved, False if failed. - """ - - # Start by assuming failure: - token_saved = False - - # Note down the timestamp at which this event occurred: - request_ts = date_time.get_current_utc_date_time(as_string = False) - - # Save the token to MongoDB: - mongo_json = await mongo_conn.find_one_and_update( - collection = MailOAuthModel.AUTH_COLLECTION, - filter = {"_id": ObjectId(user_identifier)}, - update = { - "$set": { - "token": token, - "lastRefreshTs": request_ts, - }, - "$setOnInsert": { - "firstRefreshTs": request_ts, - } - }, - projection = {"token": False}, - return_updated = True, - upsert = False - ) - - # Tell MariaDB that the token was saved: - if mongo_json is not None: - db_json = await self.call_procedure( - db_conn = db_conn, - proc_name = "entity_integration_save", - proc_args = ( - mongo_json["user"]["entityId"], # ............................................. 'p_entity_id' - mongo_json["client"], # ....................................................... 'p_provider' - "Auth Granted", # ............................................................. 'p_current_status' - "Set Token", # ................................................................ 'p_last_action' - None, # ....................................................................... 'p_display_name' - None, # ....................................................................... 'p_display_picture' - user_identifier, # ............................................................ 'p_token_id' - json.to_string(python_data = {"email": token["email"]}, no_space = True), # ... 'p_notes' - mongo_json["user"]["userId"] # ................................................ 'p_created_by' - ), - session_token = session_token - ) - if db_json["status"] == 1: token_saved = True - - # Done here: - return token_saved - - async def get_token( - self, - mongo_conn: AsyncMongo, - user_identifier: ObjectId | str = None, - **kwargs - ) -> dict | None: - - """ - To retrieve stored tokens from the database. - :param mongo_conn: The database connection (MongoDB) to use to perform the action. - :param user_identifier: The identifier granted by the 'get_user_identifier' method. - :param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user - info, the client, the type of authentication used, or even the kind of service. - :return: The retrieved record that has the token, and information about the service and client if found, else - None when there is no matching record. - """ - - # Build the filter: - filter_json = {k: v for k, v in kwargs.items()} - if user_identifier: filter_json["_id"] = ObjectId(user_identifier) - - # If there is no search criteria, we exit with failure: - if not filter_json: return None - - # If there is some filtering possible, - # we fetch and return the token: - return await mongo_conn.find_one( - collection = self.AUTH_COLLECTION, - filter = filter_json, - projection = { - "_id": True, - "serviceType": True, - "authType": True, - "client": True, - "token": True - } - ) - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/models/behaviour/mail/retrieve.py b/models/behaviour/mail/retrieve.py new file mode 100644 index 0000000..abaf213 --- /dev/null +++ b/models/behaviour/mail/retrieve.py @@ -0,0 +1,154 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 3rd Dec., 2024 + + OBJECTIVE: + + To enlist and retrieve mails for various filtering conditions. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# My async utils: +from utils_v2.string import json +from utils_v2.date_time import date_time +from utils_v2.database.async_mongo_v2 import AsyncMongo + +# Base model: +from models.behaviour.base import BaseModel + +# To work with MongoDB: +from bson import ObjectId + +# To work with datatypes: +from typing import Literal + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class MailRetrieveModel(BaseModel): + + # For MongoDB: + AUTH_COLLECTION = "_authTokens" + MAIL_COLLECTION = "_messages" + + async def get_mail( + self, + mongo_conn: AsyncMongo, + mail_identifier: str | ObjectId + ): + + """ + Retrieves one full mail from the database. + :param mongo_conn: The instance of the database connector to use to get the mail's data. + :param mail_identifier: The '_id' of the document that holds the mail. + :return: Either the JSON that describes the mail or None if such a mail does not exist. + """ + + mail_data = await mongo_conn.find_one( + collection = self.MAIL_COLLECTION, + filter = {"_id": ObjectId(mail_identifier)}, + projection = { + "mailId": "_id", + "serviceType": True, + "client": True, + "payload.ts": True, + "payload.readTs": True, + "payload.from": True, + "payload.to": True, + "payload.cc": True, + "payload.bcc": True, + "payload.parts": True, + "payload.attachments": True, + "payload.labels": True, + "payload.snippet": True, + "payload.aiSnippet": "payload.aiSnippet.snippet", + } + ) + if mail_data: mail_data["mailId"] = str(mail_data["mailId"]) + return mail_data + + async def list_by_account_identifier( + self, + mongo_conn: AsyncMongo, + account_identifier: str | ObjectId, + limit: int = 25, + skip: int = 0 + ): + + pass + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/models/behaviour/mail/sync.py b/models/behaviour/mail/sync.py deleted file mode 100644 index 0e44ec5..0000000 --- a/models/behaviour/mail/sync.py +++ /dev/null @@ -1,327 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Wednesday, 27th Nov., 2024 - - OBJECTIVE: - - To define the interaction between the UI layer and the database connectivity in one place. Here we shall handle - all the activities for OAuth2.0 authorization requests for all the users of our service. - - REFERENCES: - - N/A - - DOWNLOADS: - - N/A - -""" - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys - -from langchain.chains.summarize.stuff_prompt import prompt_template -from sqlalchemy.orm.collections import collection - -sys.path.append(".") -sys.path.append("..") - -# My async utils: -from utils_v2.string import json -from utils_v2.date_time import date_time -from utils_v2.database.async_mysql_v2 import AsyncMySQL -from utils_v2.database.async_mongo_v2 import AsyncMongo - -# Mail Clients: -from utils_v2.goog.gmail.gmail_client import AsyncGMailClient -from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens - -# Base model: -from models.behaviour.base import BaseModel - -# To work with MongoDB: -from bson import ObjectId -from pymongo import InsertOne, UpdateOne - -# To work with LLMs: -from langchain_openai import ChatOpenAI -from langchain_core.prompts import ChatPromptTemplate - -# To work with datatypes: -from typing import Literal - -# To make deep-copies: -import copy - -# To work with date and time: -import datetime - -# For asynchronous activities: -import asyncio - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** CLASSES *** -# ***** **** -# ***************************************************************************************************************** - - -class MailSyncModel(BaseModel): - - # For MongoDB: - AUTH_COLLECTION = "_authTokens" - MAIL_COLLECTION = "_messages" - - # For AI Magic through LLMs: - prompt_template = ChatPromptTemplate.from_messages([ - ( - "system", - "You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPs) FROM THE SUMMARY." - ), - ( - "user", - "Please summarize this mail: \"\"\"{mail}\"\"\"" - ) - ]) - - async def __sync_one( - self, - mongo_conn: AsyncMongo, - user_info: dict, - mail_client: AsyncGMailClient, - tokens: GoogleAuthTokens, - message_id: str, - llm: ChatOpenAI = None, - force_sync: bool = False - ) -> UpdateOne | None: - - # ┏┓ ┓┏ • ┓ ┓ - # ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏ - # ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛ - # ┛ - - mail_payload = None - mail_client_name = None - - # ┏┓┓ ┓ ┏┓ • • ┳┓ ┓ - # ┃ ┣┓┏┓┏┃┏ ┣ ┓┏┓┏╋┓┏┓┏┓ ┣┫┏┓┏┏┓┏┓┏┫┏ - # ┗┛┛┗┗ ┗┛┗ ┗┛┛┗┗┛┗┗┛┗┗┫ ┛┗┗ ┗┗┛┛ ┗┻┛ - # ┛ - - # Check if you already have that mail in your database: - existing_record = await mongo_conn.find_one( - collection = self.MAIL_COLLECTION, - filter = { - "messageType": "email", - "$or": [ - {"payload.messageId": message_id} - ] - }, - projection = {"_id": True} - ) - - # If there already exists such a record, and we haven't been forced to re-sync it: - if existing_record and not force_sync: return mail_payload - - # ┏┓┳┳┓ •┓ - # ┃┓┃┃┃┏┓┓┃ - # ┗┛┛ ┗┗┻┗┗ - - if isinstance(mail_client, AsyncGMailClient): - - # Note down the name of the mail client: - mail_client_name = "gmail" - - # Fetch the formatted mail message: - client_response = await mail_client.get_message( - tokens = tokens, - message_id = message_id, - return_raw = False - ) - - # If the fetch was successful: - if client_response.success: - - # Summarize the content: - if llm: - prompt = self.prompt_template.invoke({"mail": client_response.data["unformattedText"]}) - llm_response = await llm.ainvoke(prompt) - client_response.data["aiSnippet"] = llm_response.content - - # Note down the response: - mail_payload = client_response.data - - # ┳┓ - # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓ - # ┛┗┗ ┛┣┛┗┛┛┗┛┗ - # ┛ - - if mail_payload: - return UpdateOne( - filter = { - "messageType": "email", - "$or": [ - {"payload.messageId": message_id} - ] - }, - update = { - "$set": { - "readTs": date_time.get_current_utc_date_time(), - "user": user_info, - "messageType": "email", - "connector": mail_client_name, - "payload": mail_payload - } - }, - upsert = True - ) - - # Done here: - return mail_payload - - async def sync( - self, - mongo_conn: AsyncMongo, - user_info: dict, - mail_client: AsyncGMailClient, - tokens: GoogleAuthTokens, - llm: ChatOpenAI = None, - force_sync: bool = False, - start_date: datetime.datetime = None, - end_date: datetime.datetime = None, - max_count: int = 100 - ) -> int: - - # Start by assuming failure: - mails_count = 0 - - # ┏┓┳┳┓ •┓ - # ┃┓┃┃┃┏┓┓┃ - # ┗┛┛ ┗┗┻┗┗ - - if isinstance(mail_client, AsyncGMailClient): - - # Enlist all the labels, we need to find the label that indicates that we've read the mail: - client_response = await mail_client.list_labels(tokens = tokens) - if not client_response.success: return mails_count - labels = client_response.data - custom_label = "Sync'd with TheCAOffice" - custom_label_id = labels.get(custom_label) - if custom_label_id is None: - client_response = await mail_client.create_label( - tokens = tokens, - label_name = custom_label, - label_visibility = "labelHide" - ) - if not client_response.success: return mails_count - custom_label_id = client_response.data["id"] - - # Build the query: - sub_queries = [f"-label:\"{custom_label}\""] - if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d")) - if end_date: sub_queries.append(end_date.strftime("before:%Y/%m/%d")) - print("Q:", " ".join(sub_queries)) - - # Get a list of all the mails: - client_response = await mail_client.list_messages( - tokens = tokens, - max_count = max_count, - # query = " ".join(sub_queries) - ) - if not client_response.success: return mails_count - messages_list = client_response.data["messages"] - print(messages_list) - - # Create MongoDB operations for all the mails: - tasks = [ - self.__sync_one( - mongo_conn = mongo_conn, - user_info = user_info, - mail_client = mail_client, - tokens = tokens, - message_id = v["id"], - llm = llm, - force_sync = force_sync - ) - for k, v in messages_list.items() - ] - mongo_operations = await asyncio.gather(*tasks) - mongo_operations = [mo for mo in mongo_operations if mo is not None] - - # Write the mails to MongoDB: - mails_count = await mongo_conn.bulk_write( - collection = self.MAIL_COLLECTION, - requests = mongo_operations - ) - print("MAILS COUNT:", mails_count) - - # If all the mails were sync'd properly: - client_response = await mail_client.modify_messages( - tokens = tokens, - message_ids = [v["id"] for k, v in messages_list.items()], - add_label_ids = [custom_label_id] - ) - - # ┳┓ - # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓ - # ┛┗┗ ┛┣┛┗┛┛┗┛┗ - # ┛ - - # Done here: - return mails_count - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - pass diff --git a/models/data/mail/list.py b/models/data/mail/list.py new file mode 100644 index 0000000..d4e6abb --- /dev/null +++ b/models/data/mail/list.py @@ -0,0 +1,127 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Tuesday, 3rd Dec., 2024. + + OBJECTIVE: + + To provide a structure to query the full payload of an email. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For making data behaviour_models: +from pydantic import BaseModel, Field, field_validator, PastDatetime +from typing import Optional, Literal + +# My utils: +from utils_v2.string import regex +from utils_v2.date_time import date_time + +# To work with date and time: +import datetime + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# RegEx Patterns: +REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class MailGetRequestHeaders(BaseModel): + + sessionToken: str = Field( + description = "the session token of the user who is requesting the service", + pattern = REGEX_SESSION_TOKEN, + frozen = True, + alias = "X-Session-Token" + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "allow" + + def model_dump(self, *args, **kwargs): + return super().model_dump(*args, by_alias = True, **kwargs) + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MailGetRequestData(BaseModel): + + mailId: str = Field( + description = "the mail identifier (Mongo ObjectId) of the document that holds the mail", + frozen = True + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "forbid" + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/models/data/mail/retrieve.py b/models/data/mail/retrieve.py new file mode 100644 index 0000000..ee3965c --- /dev/null +++ b/models/data/mail/retrieve.py @@ -0,0 +1,228 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Monday, 2nd Dec., 2024. + + OBJECTIVE: + + To provide the structure for the request that will come in to sync the mails of a particular user. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For making data behaviour_models: +from pydantic import BaseModel, Field, field_validator, PastDatetime +from typing import Optional, Literal + +# My utils: +from utils_v2.string import regex +from utils_v2.date_time import date_time + +# To work with date and time: +import datetime + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# RegEx Patterns: +REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class MailSyncRequestHeaders(BaseModel): + + sessionToken: str = Field( + description = "the session token of the user who is requesting the service", + pattern = REGEX_SESSION_TOKEN, + frozen = True, + alias = "X-Session-Token" + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "allow" + + def model_dump(self, *args, **kwargs): + return super().model_dump(*args, by_alias = True, **kwargs) + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MailSyncRequestData(BaseModel): + + accountId: str = Field( + description = "the account identifier (Mongo ObjectId) granted by 'MailOAuthModel.get_account_identifier'", + frozen = True + ) + + maxCount: int = Field( + description = "the max. no. of e-mails to sync at a given time", + default = 100, + ge = 1, + le = 100, + frozen = True + ) + + startDate: PastDatetime = Field( + description = "the starting date from which the user wants to sync their mail", + default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1), + frozen = True + ) + + endDate: PastDatetime = Field( + description = "the ending date till which the user wants to sync their mail", + default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1), + frozen = True + ) + + forceSync: bool = Field( + description = "use this to forcefully re-sync mails when you need to overwrite existing data in mongodb", + default = False + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "forbid" + + # ┓┏ ┓• ┓ • + # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓ + # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗ + + @field_validator("startDate", "endDate", mode = "before") + def to_datetime(cls, value): + if not isinstance(value, datetime.datetime): + value = date_time.parse_date_time( + input_value = value, + timezone = date_time.TIMEZONE_UTC + ) + return value + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MailSyncOneResult(BaseModel): + + success: bool = Field( + description = "whether, or not, the mail was successfully sync'd", + default = False + ) + + message: str | None = Field( + description = "a brief message to summarize the result of the process", + default = None + ) + + mailMessage: dict | None = Field( + description = "the actual data of the mail; can be null in a successful process if the mail is already sync'd", + default = None + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "forbid" + + +# --------------------------------------------------------------------------------------------------------------------- + + +class MailSyncManyResults(BaseModel): + + totalCount: int = Field( + description = "the total no. of mails that were to be sync'd", + default = 0 + ) + + successCount: int = Field( + description = "the no. of mails that were successfully sync'd", + default = 0 + ) + + failureCount: int = Field( + description = "the no. of mails that were successfully sync'd", + default = 0 + ) + + message: str = Field( + description = "a brief message to summarize the results of the process", + default = None + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "forbid" + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/utils_v2/api/async_quart.py b/utils_v2/api/async_quart.py index 7bd0abe..5331eda 100644 --- a/utils_v2/api/async_quart.py +++ b/utils_v2/api/async_quart.py @@ -136,17 +136,22 @@ class AuthDetailsIncompleteException(Exception): # ***************************************************************************************************************** -async def make_ordered_json(json_data, http_code = 200): +async def make_ordered_json( + json_data, + no_space = True, + http_code = 200 +): """ Quart sorts the fields of a dict when converting to a JSON response. Here we are manually making the response when the sequence of the fields is sensitive. :param json_data: The data (dict, list, etc.) to be converted to a JSON string. + :param no_space: Set this to True to remove all excess white spaces from the JSON string. Saves bandwidth. :param http_code: The HTTP status code you want to send with the response. :return: The JSON-ified response such that the sequence of the fields is maintained. """ - response = await make_response(json.to_string(json_data, no_space = True), http_code) + response = await make_response(json.to_string(json_data, no_space = no_space), http_code) response.headers["Content-Type"] = "application/json" return response