From bcbad8ecdbd763d6237c4aad18d34c2483e02f4c Mon Sep 17 00:00:00 2001 From: khushal Date: Mon, 2 Dec 2024 18:47:38 +0530 Subject: [PATCH] (20241202) Testing out the modified auth process. --- api/blueprints/mail/oauth_callback.py | 39 +++++- api/blueprints/mail/oauth_request.py | 5 +- api/blueprints/mail/sync.py | 62 ++++++++- api/main.py | 13 +- models/behaviour/mail/oauth.py | 112 +++++++++------- models/behaviour/mail/oauth_v2.py | 143 ++++++++++----------- models/behaviour/mail/sync.py | 169 +++++++++++++++++++++---- models/data/mail/oauth.py | 6 + models/data/mail/sync.py | 42 ++++-- views/mail/oauth/oauth_failure_v2.html | 1 + 10 files changed, 419 insertions(+), 173 deletions(-) diff --git a/api/blueprints/mail/oauth_callback.py b/api/blueprints/mail/oauth_callback.py index f78279c..ae64163 100644 --- a/api/blueprints/mail/oauth_callback.py +++ b/api/blueprints/mail/oauth_callback.py @@ -164,19 +164,35 @@ async def mail_callback( if tokens: - # Add information and: + # Get the e-mail id that granted authorization: user_profile = await current_app.gmail_client.get_user_profile(tokens = tokens) tokens.email = user_profile.data["emailAddress"] if user_profile.success else None + # The e-mail id that we requested access to and the one that granted us access should be the same: + placeholder_token = await current_app.mail_oauth_model.set_token( + mongo_conn = current_app.data_mongo, + account_identifier = inbound_data["state"] + ) + if ( + (not placeholder_token) or + placeholder_token["clientUserId"] != str(tokens.email) + ): return await render_template( + "/mail/oauth/oauth_failure_v2.html", + mail_client = mail_client.title(), + failure_hint = f"We were expecting authorization from '{placeholder_token['clientUserId']}' but got authorization from '{tokens.email}' instead." + ) + # Save the tokens to the database tokens_saved = await current_app.mail_oauth_model.set_token( db_conn = current_app.sql_writer, mongo_conn = current_app.data_mongo, session_token = inbound_headers.get("X-Session-Token"), - user_identifier = inbound_data["state"], + account_identifier = inbound_data["state"], token = tokens.model_dump() ) + # tokens_saved = False + # ┳┓ # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓ # ┛┗┗ ┛┣┛┗┛┛┗┛┗ @@ -192,11 +208,20 @@ async def mail_callback( # } # ) - # Return an HTML response: - return await render_template( - "/mail/oauth/oauth_success_v2.html" if tokens_saved else "/mail/oauth/oauth_failure_v2.html", - mail_client = mail_client.title() - ) + # Return an HTML response for success: + if tokens_saved: + return await render_template( + "/mail/oauth/oauth_success_v2.html", + mail_client = mail_client.title() + ) + + # Return an HTML response for failure: + else: + return await render_template( + "/mail/oauth/oauth_failure_v2.html", + mail_client = mail_client.title(), + failure_hint = f"Unknown error. Please use log-id '{kwargs['logId']}' to check with the support team." + ) # ***************************************************************************************************************** diff --git a/api/blueprints/mail/oauth_request.py b/api/blueprints/mail/oauth_request.py index f5eb902..1465061 100644 --- a/api/blueprints/mail/oauth_request.py +++ b/api/blueprints/mail/oauth_request.py @@ -175,11 +175,12 @@ async def request_oauth_authorization_url( # ┛ # Make a user identifier from the session info: - user_identifier = await current_app.mail_oauth_model.get_user_identifier( + user_identifier = await current_app.mail_oauth_model.get_account_identifier( db_conn = current_app.sql_writer, mongo_conn = current_app.data_mongo, session_token = inbound_headers["X-Session-Token"], user_info = kwargs["session_info"], + email_id = inbound_data.mailId, service_client = inbound_data.mailClient, auth_type = "oauth" ) @@ -204,7 +205,7 @@ async def request_oauth_authorization_url( access_type = "offline", approval_prompt = "force", include_granted_scopes = "true", - user_email = None + user_email = inbound_data.mailId ) # ┳┓ diff --git a/api/blueprints/mail/sync.py b/api/blueprints/mail/sync.py index 9f05c8a..f4321c2 100644 --- a/api/blueprints/mail/sync.py +++ b/api/blueprints/mail/sync.py @@ -63,6 +63,7 @@ from utils_v2.api.async_quart import ( # GMail-related utils: from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT +from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens # Common: from shared import constants @@ -113,7 +114,7 @@ def init(blueprint_setup_state): # --------------------------------------------------------------------------------------------------------------------- -@mail_sync_bp.route("/sync", methods = ["GET"]) +@mail_sync_bp.route("/sync", methods = ["POST"]) @set_api_version(api_version = "1.0.0") @read_input(sanitize_headers = False, sanitize_data = False) @get_session_info(key = "X-Session-Token", session_coro = "get_session") @@ -163,19 +164,65 @@ async def sync_mail( ) # Start by assuming failure: - auth_url = None + mails_count = 0 # ┏┓ ┏┳┓ ┓ # ┃┓┏┓╋ ┃ ┏┓┃┏┏┓┏┓ # ┗┛┗ ┗ ┻ ┗┛┛┗┗ ┛┗ # Make a user identifier from the session info: - user_tokens = await current_app.mail_oauth_model.get_token( + user_auth = await current_app.mail_oauth_model.get_token( mongo_conn = current_app.data_mongo, - user = kwargs["session_info"] + serviceType = "email", + user = kwargs["session_info"], ) - print("MAIL TOKEN(S):", json.to_string(user_tokens)) + print("MAIL TOKEN(S):", json.to_string(user_auth, default = str)) + + if not user_auth: return ResponseModel( + status_code = StatusCodes.FAILED, + http_code = HttpCodes.NOT_FOUND, + message = "user's email not connected" + ) + + # ┏┓┳┳┓ •┓ + # ┃┓┃┃┃┏┓┓┃ + # ┗┛┛ ┗┗┻┗┗ + + if user_auth["client"] == "gmail": + + # Load the tokens into an object: + user_tokens = GoogleAuthTokens(**user_auth["token"]) + + # Try refreshing the tokens: + tokens_refreshed = await user_tokens.arefresh( + http_client = current_app.http_client, + client_id = current_app.gmail_client.client_id, + client_secret = current_app.gmail_client.client_secret, + force_refresh = False + ) + + # Update the token in the database if needed: + if tokens_refreshed: await current_app.mail_oauth_model.set_token( + db_conn = current_app.sql_writer, + mongo_conn = current_app.data_mongo, + user_identifier = user_auth["_id"], + token = user_tokens.model_dump(), + session_token = inbound_headers["X-Session-Token"] + ) + + # Now we try to sync the mails: + mails_count = await current_app.mail_sync_model.sync( + mongo_conn = current_app.data_mongo, + user_info = kwargs["session_info"], + mail_client = current_app.gmail_client, + tokens = user_tokens, + llm = None, + force_sync = False, + start_date = inbound_data.startDate, + end_date = inbound_data.endDate, + max_count = inbound_data.maxCount + ) # ┳┓ # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓ @@ -183,7 +230,10 @@ async def sync_mail( # ┛ # Done here: - return ResponseModel(status_code = StatusCodes.OK) + return ResponseModel( + status_code = StatusCodes.OK if mails_count else StatusCodes.FAILED, + message = f"{mails_count} mail(s) sync'd" + ) # ***************************************************************************************************************** diff --git a/api/main.py b/api/main.py index c926df1..45c11a4 100644 --- a/api/main.py +++ b/api/main.py @@ -71,7 +71,8 @@ from utils_v2.api.async_quart import ( from utils_v2.goog.gmail.gmail_client import AsyncGMailClient # Behaviour Models: -from models.behaviour.mail.oauth import MailOAuthModel +from models.behaviour.mail.oauth_v2 import MailOAuthModel +from models.behaviour.mail.sync import MailSyncModel # To make REST API calls: import httpx @@ -82,6 +83,7 @@ from icecream import IceCreamDebugger # All the blueprints: from api.blueprints.mail.oauth_request import mail_oauth_bp from api.blueprints.mail.oauth_callback import mail_callback_bp +from api.blueprints.mail.sync import mail_sync_bp from api.blueprints.tech.chat_alerts import tech_chat_alert_bp from api.blueprints.test.callback import test_callback_bp @@ -113,6 +115,7 @@ app = Quart(__name__, template_folder = r"../views") app = cors(app) app.register_blueprint(mail_oauth_bp, url_prefix = f"/{MODULE_BASE}/mail") app.register_blueprint(mail_callback_bp, url_prefix = f"/{MODULE_BASE}/mail") +app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail") app.register_blueprint(tech_chat_alert_bp, url_prefix = f"/{MODULE_BASE}/tech/alert") app.register_blueprint(test_callback_bp, url_prefix = f"/{MODULE_BASE}/test") @@ -288,6 +291,14 @@ async def app_startup(**kwargs): debug_prefix = "Mail-OAuth | ", debug_only_errors = True ) + current_app.mail_sync_model = MailSyncModel( + cache = current_app.module_cache, + alert_url = current_app.script_data["alerts"]["url"], + http_client = current_app.http_client, + debug = enable_debugging, + debug_prefix = "Mail-Sync | ", + debug_only_errors = True + ) # ┏┓ # ┃ ┏┓┏┓┏┓┏┓┏╋┏┓┏┓┏ diff --git a/models/behaviour/mail/oauth.py b/models/behaviour/mail/oauth.py index 5d3a938..fecc8ef 100644 --- a/models/behaviour/mail/oauth.py +++ b/models/behaviour/mail/oauth.py @@ -122,59 +122,76 @@ class MailOAuthModel(BaseModel): request_ts = date_time.get_current_utc_date_time(as_string = False) # Get the identifier from the database: - mongo_json = await mongo_conn.find_one_and_update( + inserted_id = await mongo_conn.insert_one( collection = MailOAuthModel.AUTH_COLLECTION, - filter = { + document = { + "version": "-1.0.1", "serviceType": "email", "client": service_client, "authType": auth_type, "user": user_info, - }, - update = { - "$set": { - "lastRequestTs": request_ts - }, - "$setOnInsert": { - "version": "1.0.0", - "serviceType": "email", - "client": service_client, - "authType": auth_type, - "user": user_info, - "token": None, - "firstRefreshTs": None, - "lastRefreshTs": None, - "firstRequestTs": request_ts, - } - }, - projection = { - "_id": True - }, - upsert = True, - return_updated = True + "token": None, + "firstRefreshTs": None, + "lastRefreshTs": None, + "lastRequestTs": request_ts + } ) + return inserted_id - # Tell MariaDB that an authorization request was initiated: - db_json = {} - if mongo_json is not None: - db_json = await self.call_procedure( - db_conn = db_conn, - proc_name = "entity_integration_save", - proc_args = ( - user_info["entityId"], # ............................................ 'p_entity_id' - service_client, # ................................................... 'p_provider' - "Auth Requested", # ................................................. 'p_current_status' - "Auth URL Generated", # ............................................. 'p_last_action' - None, # ............................................................. 'p_display_name' - None, # ............................................................. 'p_display_picture' - str(mongo_json["_id"]), # ........................................... 'p_token_id' - json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes' - user_info["userId"] # ............................................... 'p_created_by' - ), - session_token = session_token - ) - - # Done here: - return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None + # # Get the identifier from the database: + # mongo_json = await mongo_conn.find_one_and_update( + # collection = MailOAuthModel.AUTH_COLLECTION, + # filter = { + # "serviceType": "email", + # "client": service_client, + # "authType": auth_type, + # "user": user_info, + # }, + # update = { + # "$set": { + # "lastRequestTs": request_ts + # }, + # "$setOnInsert": { + # "version": "1.0.0", + # "serviceType": "email", + # "client": service_client, + # "authType": auth_type, + # "user": user_info, + # "token": None, + # "firstRefreshTs": None, + # "lastRefreshTs": None, + # "firstRequestTs": request_ts, + # } + # }, + # projection = { + # "_id": True + # }, + # upsert = True, + # return_updated = True + # ) + # + # # Tell MariaDB that an authorization request was initiated: + # db_json = {} + # if mongo_json is not None: + # db_json = await self.call_procedure( + # db_conn = db_conn, + # proc_name = "entity_integration_save", + # proc_args = ( + # user_info["entityId"], # ............................................ 'p_entity_id' + # service_client, # ................................................... 'p_provider' + # "Auth Requested", # ................................................. 'p_current_status' + # "Auth URL Generated", # ............................................. 'p_last_action' + # None, # ............................................................. 'p_display_name' + # None, # ............................................................. 'p_display_picture' + # str(mongo_json["_id"]), # ........................................... 'p_token_id' + # json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes' + # user_info["userId"] # ............................................... 'p_created_by' + # ), + # session_token = session_token + # ) + # + # # Done here: + # return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None async def set_token( self, @@ -210,6 +227,9 @@ class MailOAuthModel(BaseModel): update = { "$set": { "token": token, + "lastRefreshTs": request_ts, + }, + "$setOnInsert": { "firstRefreshTs": request_ts, } }, diff --git a/models/behaviour/mail/oauth_v2.py b/models/behaviour/mail/oauth_v2.py index fecc8ef..5628776 100644 --- a/models/behaviour/mail/oauth_v2.py +++ b/models/behaviour/mail/oauth_v2.py @@ -6,7 +6,7 @@ DATE: - Wednesday, 27th Nov., 2024 + Monday, 2nd Dec., 2024 OBJECTIVE: @@ -95,11 +95,12 @@ class MailOAuthModel(BaseModel): AUTH_COLLECTION = "_authTokens" - async def get_user_identifier( + async def get_account_identifier( self, db_conn: AsyncMySQL, mongo_conn: AsyncMongo, user_info: dict, + email_id: str, service_client: Literal["gmail"], auth_type: Literal["oauth"], session_token: str = None @@ -111,6 +112,7 @@ class MailOAuthModel(BaseModel): :param db_conn: The database connection (MariaDB) to use to perform the action. :param mongo_conn: The database connection (MongoDB) to use to perform the action. :param user_info: The dictionary that has the user's session information. + :param email_id: The e-mail id that the user wants to connect to your service. :param service_client: The name of the company or brand that is providing this service that is being integrated. :param auth_type: To identify the type of authentication being done here. This could indicate simple password authentication, more advance OAuth2.0 authentication, etc. @@ -122,82 +124,68 @@ class MailOAuthModel(BaseModel): request_ts = date_time.get_current_utc_date_time(as_string = False) # Get the identifier from the database: - inserted_id = await mongo_conn.insert_one( + mongo_json = await mongo_conn.find_one_and_update( collection = MailOAuthModel.AUTH_COLLECTION, - document = { - "version": "-1.0.1", + filter = { "serviceType": "email", - "client": service_client, - "authType": auth_type, - "user": user_info, - "token": None, - "firstRefreshTs": None, - "lastRefreshTs": None, - "lastRequestTs": request_ts - } + "user": { + "entityId": user_info["entityId"], + "billingAccountId": user_info["billingAccountId"] + }, + "clientUserId": email_id + }, + update = { + "$set": { + "lastRequestTs": request_ts + }, + "$setOnInsert": { + "version": "1.1.0", + "serviceType": "email", + "client": service_client, + "authType": auth_type, + "user": user_info, + "token": None, + "firstRefreshTs": None, + "lastRefreshTs": None, + "firstRequestTs": request_ts, + } + }, + projection = { + "_id": True + }, + upsert = True, + return_updated = True ) - return inserted_id - # # Get the identifier from the database: - # mongo_json = await mongo_conn.find_one_and_update( - # collection = MailOAuthModel.AUTH_COLLECTION, - # filter = { - # "serviceType": "email", - # "client": service_client, - # "authType": auth_type, - # "user": user_info, - # }, - # update = { - # "$set": { - # "lastRequestTs": request_ts - # }, - # "$setOnInsert": { - # "version": "1.0.0", - # "serviceType": "email", - # "client": service_client, - # "authType": auth_type, - # "user": user_info, - # "token": None, - # "firstRefreshTs": None, - # "lastRefreshTs": None, - # "firstRequestTs": request_ts, - # } - # }, - # projection = { - # "_id": True - # }, - # upsert = True, - # return_updated = True - # ) - # - # # Tell MariaDB that an authorization request was initiated: - # db_json = {} - # if mongo_json is not None: - # db_json = await self.call_procedure( - # db_conn = db_conn, - # proc_name = "entity_integration_save", - # proc_args = ( - # user_info["entityId"], # ............................................ 'p_entity_id' - # service_client, # ................................................... 'p_provider' - # "Auth Requested", # ................................................. 'p_current_status' - # "Auth URL Generated", # ............................................. 'p_last_action' - # None, # ............................................................. 'p_display_name' - # None, # ............................................................. 'p_display_picture' - # str(mongo_json["_id"]), # ........................................... 'p_token_id' - # json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes' - # user_info["userId"] # ............................................... 'p_created_by' - # ), - # session_token = session_token - # ) - # - # # Done here: - # return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None + # Tell MariaDB that an authorization request was initiated: + db_json = {} + if mongo_json is not None: + db_json = await self.call_procedure( + db_conn = db_conn, + proc_name = "entity_integration_save", + proc_args = ( + user_info["entityId"], # ............................................ 'p_entity_id' + service_client, # ................................................... 'p_provider' + "Auth Requested", # ................................................. 'p_current_status' + "Auth URL Generated", # ............................................. 'p_last_action' + None, # ............................................................. 'p_display_name' + None, # ............................................................. 'p_display_picture' + str(mongo_json["_id"]), # ........................................... 'p_token_id' + json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes' + user_info["userId"] # ............................................... 'p_created_by' + ), + session_token = session_token + ) + + # Done here: + return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None async def set_token( self, db_conn: AsyncMySQL, mongo_conn: AsyncMongo, - user_identifier: ObjectId | str, + account_identifier: ObjectId | str, + email_id: str, token: dict, session_token: str = None ) -> bool: @@ -208,7 +196,9 @@ class MailOAuthModel(BaseModel): ALSO. :param db_conn: The database connection (MariaDB) to use to perform the action. :param mongo_conn: The database connection (MongoDB) to use to perform the action. - :param user_identifier: The identifier granted by the 'get_user_identifier' method. + :param account_identifier: The identifier granted by the 'get_account_identifier' method. + :param email_id: The e-mail id that the user tried to connect to your service. This should match the e-mail id + the user claimed he wants to connect when he used 'get_account_identifier'. :param token: The token granted by the third-party service. :param session_token: The session token of the user who requested this service. :return: True if saved, False if failed. @@ -223,7 +213,10 @@ class MailOAuthModel(BaseModel): # Save the token to MongoDB: mongo_json = await mongo_conn.find_one_and_update( collection = MailOAuthModel.AUTH_COLLECTION, - filter = {"_id": ObjectId(user_identifier)}, + filter = { + "_id": ObjectId(account_identifier), + "clientUserId": email_id + }, update = { "$set": { "token": token, @@ -250,7 +243,7 @@ class MailOAuthModel(BaseModel): "Set Token", # ................................................................ 'p_last_action' None, # ....................................................................... 'p_display_name' None, # ....................................................................... 'p_display_picture' - user_identifier, # ............................................................ 'p_token_id' + account_identifier, # ......................................................... 'p_token_id' json.to_string(python_data = {"email": token["email"]}, no_space = True), # ... 'p_notes' mongo_json["user"]["userId"] # ................................................ 'p_created_by' ), @@ -264,14 +257,14 @@ class MailOAuthModel(BaseModel): async def get_token( self, mongo_conn: AsyncMongo, - user_identifier: ObjectId | str = None, + account_identifier: ObjectId | str = None, **kwargs ) -> dict | None: """ To retrieve stored tokens from the database. :param mongo_conn: The database connection (MongoDB) to use to perform the action. - :param user_identifier: The identifier granted by the 'get_user_identifier' method. + :param account_identifier: The identifier granted by the 'account_identifier' method. :param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user info, the client, the type of authentication used, or even the kind of service. :return: The retrieved record that has the token, and information about the service and client if found, else @@ -280,7 +273,7 @@ class MailOAuthModel(BaseModel): # Build the filter: filter_json = {k: v for k, v in kwargs.items()} - if user_identifier: filter_json["_id"] = ObjectId(user_identifier) + if account_identifier: filter_json["_id"] = ObjectId(account_identifier) # If there is no search criteria, we exit with failure: if not filter_json: return None diff --git a/models/behaviour/mail/sync.py b/models/behaviour/mail/sync.py index da95512..0e44ec5 100644 --- a/models/behaviour/mail/sync.py +++ b/models/behaviour/mail/sync.py @@ -34,6 +34,7 @@ import sys from langchain.chains.summarize.stuff_prompt import prompt_template +from sqlalchemy.orm.collections import collection sys.path.append(".") sys.path.append("..") @@ -53,6 +54,7 @@ from models.behaviour.base import BaseModel # To work with MongoDB: from bson import ObjectId +from pymongo import InsertOne, UpdateOne # To work with LLMs: from langchain_openai import ChatOpenAI @@ -64,6 +66,12 @@ from typing import Literal # To make deep-copies: import copy +# To work with date and time: +import datetime + +# For asynchronous activities: +import asyncio + # ***************************************************************************************************************** # ***** **** @@ -112,7 +120,7 @@ class MailSyncModel(BaseModel): prompt_template = ChatPromptTemplate.from_messages([ ( "system", - "You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPS) FROM THE SUMMARY." + "You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPs) FROM THE SUMMARY." ), ( "user", @@ -120,17 +128,24 @@ class MailSyncModel(BaseModel): ) ]) - async def sync_one( + async def __sync_one( self, mongo_conn: AsyncMongo, - user_identifier: str | ObjectId, + user_info: dict, mail_client: AsyncGMailClient, tokens: GoogleAuthTokens, message_id: str, llm: ChatOpenAI = None, - session_token: str = None, force_sync: bool = False - ) -> ObjectId: + ) -> UpdateOne | None: + + # ┏┓ ┓┏ • ┓ ┓ + # ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏ + # ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛ + # ┛ + + mail_payload = None + mail_client_name = None # ┏┓┓ ┓ ┏┓ • • ┳┓ ┓ # ┃ ┣┓┏┓┏┃┏ ┣ ┓┏┓┏╋┓┏┓┏┓ ┣┫┏┓┏┏┓┏┓┏┫┏ @@ -150,15 +165,7 @@ class MailSyncModel(BaseModel): ) # If there already exists such a record, and we haven't been forced to re-sync it: - if existing_record and not force_sync: return existing_record["_id"] - - # ┏┓ ┓┏ • ┓ ┓ - # ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏ - # ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛ - # ┛ - - mail_payload = None - mail_id = existing_record["_id"] if existing_record else None + if existing_record and not force_sync: return mail_payload # ┏┓┳┳┓ •┓ # ┃┓┃┃┃┏┓┓┃ @@ -166,10 +173,10 @@ class MailSyncModel(BaseModel): if isinstance(mail_client, AsyncGMailClient): - # Refresh the tokens: - tokens_refreshed + # Note down the name of the mail client: + mail_client_name = "gmail" - # Fetch the mail formatted message: + # Fetch the formatted mail message: client_response = await mail_client.get_message( tokens = tokens, message_id = message_id, @@ -180,20 +187,132 @@ class MailSyncModel(BaseModel): if client_response.success: # Summarize the content: - prompt = self.prompt_template.invoke({"mail": client_response.data.pop["unformattedText"]}) - llm_response = await llm.ainvoke(prompt) - client_response.data["aiSnippet"] = llm_response.content + if llm: + prompt = self.prompt_template.invoke({"mail": client_response.data["unformattedText"]}) + llm_response = await llm.ainvoke(prompt) + client_response.data["aiSnippet"] = llm_response.content # Note down the response: mail_payload = client_response.data - # ┏┓ ┏┳┓┓ ┳┳┓ •┓ - # ┗┓┓┏┏┓┏ ┃ ┣┓┏┓ ┃┃┃┏┓┓┃ - # ┗┛┗┫┛┗┗ ┻ ┛┗┗ ┛ ┗┗┻┗┗ - # ┛ + # ┳┓ + # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓ + # ┛┗┗ ┛┣┛┗┛┛┗┛┗ + # ┛ if mail_payload: - pass + return UpdateOne( + filter = { + "messageType": "email", + "$or": [ + {"payload.messageId": message_id} + ] + }, + update = { + "$set": { + "readTs": date_time.get_current_utc_date_time(), + "user": user_info, + "messageType": "email", + "connector": mail_client_name, + "payload": mail_payload + } + }, + upsert = True + ) + + # Done here: + return mail_payload + + async def sync( + self, + mongo_conn: AsyncMongo, + user_info: dict, + mail_client: AsyncGMailClient, + tokens: GoogleAuthTokens, + llm: ChatOpenAI = None, + force_sync: bool = False, + start_date: datetime.datetime = None, + end_date: datetime.datetime = None, + max_count: int = 100 + ) -> int: + + # Start by assuming failure: + mails_count = 0 + + # ┏┓┳┳┓ •┓ + # ┃┓┃┃┃┏┓┓┃ + # ┗┛┛ ┗┗┻┗┗ + + if isinstance(mail_client, AsyncGMailClient): + + # Enlist all the labels, we need to find the label that indicates that we've read the mail: + client_response = await mail_client.list_labels(tokens = tokens) + if not client_response.success: return mails_count + labels = client_response.data + custom_label = "Sync'd with TheCAOffice" + custom_label_id = labels.get(custom_label) + if custom_label_id is None: + client_response = await mail_client.create_label( + tokens = tokens, + label_name = custom_label, + label_visibility = "labelHide" + ) + if not client_response.success: return mails_count + custom_label_id = client_response.data["id"] + + # Build the query: + sub_queries = [f"-label:\"{custom_label}\""] + if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d")) + if end_date: sub_queries.append(end_date.strftime("before:%Y/%m/%d")) + print("Q:", " ".join(sub_queries)) + + # Get a list of all the mails: + client_response = await mail_client.list_messages( + tokens = tokens, + max_count = max_count, + # query = " ".join(sub_queries) + ) + if not client_response.success: return mails_count + messages_list = client_response.data["messages"] + print(messages_list) + + # Create MongoDB operations for all the mails: + tasks = [ + self.__sync_one( + mongo_conn = mongo_conn, + user_info = user_info, + mail_client = mail_client, + tokens = tokens, + message_id = v["id"], + llm = llm, + force_sync = force_sync + ) + for k, v in messages_list.items() + ] + mongo_operations = await asyncio.gather(*tasks) + mongo_operations = [mo for mo in mongo_operations if mo is not None] + + # Write the mails to MongoDB: + mails_count = await mongo_conn.bulk_write( + collection = self.MAIL_COLLECTION, + requests = mongo_operations + ) + print("MAILS COUNT:", mails_count) + + # If all the mails were sync'd properly: + client_response = await mail_client.modify_messages( + tokens = tokens, + message_ids = [v["id"] for k, v in messages_list.items()], + add_label_ids = [custom_label_id] + ) + + # ┳┓ + # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓ + # ┛┗┗ ┛┣┛┗┛┛┗┛┗ + # ┛ + + # Done here: + return mails_count # ***************************************************************************************************************** diff --git a/models/data/mail/oauth.py b/models/data/mail/oauth.py index ce3d5a1..33efabd 100644 --- a/models/data/mail/oauth.py +++ b/models/data/mail/oauth.py @@ -103,6 +103,12 @@ class OAuthMailAuthorizationRequestData(BaseModel): frozen = True ) + mailId: str = Field( + description = "the e-mail id that the user intends to authorize", + pattern = regex.REGEX_EMAIL_ID, + frozen = True + ) + # ┏┓ ┏• # ┃ ┏┓┏┓╋┓┏┓ # ┗┛┗┛┛┗┛┗┗┫ diff --git a/models/data/mail/sync.py b/models/data/mail/sync.py index ce3d5a1..6bad524 100644 --- a/models/data/mail/sync.py +++ b/models/data/mail/sync.py @@ -6,12 +6,11 @@ DATE: - Wednesday, 27th Nov., 2024. + Monday, 2nd Dec., 2024. OBJECTIVE: - To provide the structure for the request and response of the APIs that will be used to request OAuth2.0 - authorization for mail services. + To provide the structure for the request that will come in to sync the mails of a particular user. REFERENCES: @@ -37,11 +36,15 @@ sys.path.append(".") sys.path.append("..") # For making data behaviour_models: -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, PastDatetime from typing import Optional, Literal # My utils: from utils_v2.string import regex +from utils_v2.date_time import date_time + +# To work with date and time: +import datetime # ***************************************************************************************************************** @@ -72,7 +75,7 @@ REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9] # ***************************************************************************************************************** -class OAuthMailAuthorizationRequestHeaders(BaseModel): +class MailSyncRequestHeaders(BaseModel): sessionToken: str = Field( description = "the session token of the user who is requesting the service", @@ -96,10 +99,23 @@ class OAuthMailAuthorizationRequestHeaders(BaseModel): # --------------------------------------------------------------------------------------------------------------------- -class OAuthMailAuthorizationRequestData(BaseModel): +class MailSyncRequestData(BaseModel): - mailClient: Literal["gmail"] = Field( - description = "the e-mail provider like 'gmail'", + maxCount: int = Field( + description = "the max. no. of e-mails to sync at a given time", + default = 100, + frozen = True + ) + + startDate: PastDatetime = Field( + description = "the starting date from which the user wants to sync their mail", + default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1), + frozen = True + ) + + endDate: PastDatetime = Field( + description = "the ending date till which the user wants to sync their mail", + default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1), frozen = True ) @@ -115,9 +131,13 @@ class OAuthMailAuthorizationRequestData(BaseModel): # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓ # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗ - @field_validator("mailClient", mode = "before") - def to_lowercase(cls, value): - if isinstance(value, str): value = value.strip().lower() + @field_validator("startDate", "endDate", mode = "before") + def to_datetime(cls, value): + if not isinstance(value, datetime.datetime): + value = date_time.parse_date_time( + input_value = value, + timezone = date_time.TIMEZONE_UTC + ) return value diff --git a/views/mail/oauth/oauth_failure_v2.html b/views/mail/oauth/oauth_failure_v2.html index 4443581..03d0b65 100644 --- a/views/mail/oauth/oauth_failure_v2.html +++ b/views/mail/oauth/oauth_failure_v2.html @@ -102,6 +102,7 @@
✘

Authorization Failed

Something went wrong in getting authorization from your {{ mail_client }} account. +

Hint: {{ failure_hint }}

Please feel free to try the same steps again. You can close this tab at any time.