(20241202) Testing out the modified auth process.

api/blueprints/mail/oauth_callback.py

@@ -164,19 +164,35 @@ async def mail_callback(
 
         if tokens:
 
-            # Add information and:
+            # Get the e-mail id that granted authorization:
             user_profile = await current_app.gmail_client.get_user_profile(tokens = tokens)
             tokens.email = user_profile.data["emailAddress"] if user_profile.success else None
 
+            # The e-mail id that we requested access to and the one that granted us access should be the same:
+            placeholder_token = await current_app.mail_oauth_model.set_token(
+                mongo_conn = current_app.data_mongo,
+                account_identifier = inbound_data["state"]
+            )
+            if (
+                    (not placeholder_token) or
+                    placeholder_token["clientUserId"] != str(tokens.email)
+            ): return await render_template(
+                "/mail/oauth/oauth_failure_v2.html",
+                mail_client = mail_client.title(),
+                failure_hint = f"We were expecting authorization from '{placeholder_token['clientUserId']}' but got authorization from '{tokens.email}' instead."
+            )
+
             # Save the tokens to the database
             tokens_saved = await current_app.mail_oauth_model.set_token(
                 db_conn = current_app.sql_writer,
                 mongo_conn = current_app.data_mongo,
                 session_token = inbound_headers.get("X-Session-Token"),
-                user_identifier = inbound_data["state"],
+                account_identifier = inbound_data["state"],
                 token = tokens.model_dump()
             )
 
+            # tokens_saved = False
+
     # ┳┓
     # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
     # ┛┗┗ ┛┣┛┗┛┛┗┛┗
@@ -192,12 +208,21 @@ async def mail_callback(
     #     }
     # )
 
-    # Return an HTML response:
+    # Return an HTML response for success:
+    if tokens_saved:
         return await render_template(
-        "/mail/oauth/oauth_success_v2.html" if tokens_saved else "/mail/oauth/oauth_failure_v2.html",
+            "/mail/oauth/oauth_success_v2.html",
             mail_client = mail_client.title()
         )
 
+    # Return an HTML response for failure:
+    else:
+        return await render_template(
+            "/mail/oauth/oauth_failure_v2.html",
+            mail_client = mail_client.title(),
+            failure_hint = f"Unknown error. Please use log-id '{kwargs['logId']}' to check with the support team."
+        )
+
 
 #   *****************************************************************************************************************
 #  *****                                                                                                          ****

api/blueprints/mail/oauth_request.py

@@ -175,11 +175,12 @@ async def request_oauth_authorization_url(
     #            ┛
 
     # Make a user identifier from the session info:
-    user_identifier = await current_app.mail_oauth_model.get_user_identifier(
+    user_identifier = await current_app.mail_oauth_model.get_account_identifier(
         db_conn = current_app.sql_writer,
         mongo_conn = current_app.data_mongo,
         session_token = inbound_headers["X-Session-Token"],
         user_info = kwargs["session_info"],
+        email_id = inbound_data.mailId,
         service_client = inbound_data.mailClient,
         auth_type = "oauth"
     )
@@ -204,7 +205,7 @@ async def request_oauth_authorization_url(
             access_type = "offline",
             approval_prompt = "force",
             include_granted_scopes = "true",
-            user_email = None
+            user_email = inbound_data.mailId
         )
 
     # ┳┓

api/blueprints/mail/sync.py

@@ -63,6 +63,7 @@ from utils_v2.api.async_quart import (
 
 # GMail-related utils:
 from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
+from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
 
 # Common:
 from shared import constants
@@ -113,7 +114,7 @@ def init(blueprint_setup_state):
 # ---------------------------------------------------------------------------------------------------------------------
 
 
-@mail_sync_bp.route("/sync", methods = ["GET"])
+@mail_sync_bp.route("/sync", methods = ["POST"])
 @set_api_version(api_version = "1.0.0")
 @read_input(sanitize_headers = False, sanitize_data = False)
 @get_session_info(key = "X-Session-Token", session_coro = "get_session")
@@ -163,19 +164,65 @@ async def sync_mail(
         )
 
     # Start by assuming failure:
-    auth_url = None
+    mails_count = 0
 
     # ┏┓     ┏┳┓  ┓
     # ┃┓┏┓╋   ┃ ┏┓┃┏┏┓┏┓
     # ┗┛┗ ┗   ┻ ┗┛┛┗┗ ┛┗
 
     # Make a user identifier from the session info:
-    user_tokens = await current_app.mail_oauth_model.get_token(
+    user_auth = await current_app.mail_oauth_model.get_token(
         mongo_conn = current_app.data_mongo,
-        user = kwargs["session_info"]
+        serviceType = "email",
+        user = kwargs["session_info"],
     )
 
-    print("MAIL TOKEN(S):", json.to_string(user_tokens))
+    print("MAIL TOKEN(S):", json.to_string(user_auth, default = str))
+
+    if not user_auth: return ResponseModel(
+        status_code = StatusCodes.FAILED,
+        http_code = HttpCodes.NOT_FOUND,
+        message = "user's email not connected"
+    )
+
+    # ┏┓┳┳┓  •┓
+    # ┃┓┃┃┃┏┓┓┃
+    # ┗┛┛ ┗┗┻┗┗
+
+    if user_auth["client"] == "gmail":
+
+        # Load the tokens into an object:
+        user_tokens = GoogleAuthTokens(**user_auth["token"])
+
+        # Try refreshing the tokens:
+        tokens_refreshed = await user_tokens.arefresh(
+            http_client = current_app.http_client,
+            client_id = current_app.gmail_client.client_id,
+            client_secret = current_app.gmail_client.client_secret,
+            force_refresh = False
+        )
+
+        # Update the token in the database if needed:
+        if tokens_refreshed: await current_app.mail_oauth_model.set_token(
+            db_conn = current_app.sql_writer,
+            mongo_conn = current_app.data_mongo,
+            user_identifier = user_auth["_id"],
+            token = user_tokens.model_dump(),
+            session_token = inbound_headers["X-Session-Token"]
+        )
+
+        # Now we try to sync the mails:
+        mails_count = await current_app.mail_sync_model.sync(
+            mongo_conn = current_app.data_mongo,
+            user_info = kwargs["session_info"],
+            mail_client = current_app.gmail_client,
+            tokens = user_tokens,
+            llm = None,
+            force_sync = False,
+            start_date = inbound_data.startDate,
+            end_date = inbound_data.endDate,
+            max_count = inbound_data.maxCount
+        )
 
     # ┳┓
     # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
@@ -183,7 +230,10 @@ async def sync_mail(
     #      ┛
 
     # Done here:
-    return ResponseModel(status_code = StatusCodes.OK)
+    return ResponseModel(
+        status_code = StatusCodes.OK if mails_count else StatusCodes.FAILED,
+        message = f"{mails_count} mail(s) sync'd"
+    )
 
 
 #   *****************************************************************************************************************

api/main.py

@@ -71,7 +71,8 @@ from utils_v2.api.async_quart import (
 from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
 
 # Behaviour Models:
-from models.behaviour.mail.oauth import MailOAuthModel
+from models.behaviour.mail.oauth_v2 import MailOAuthModel
+from models.behaviour.mail.sync import MailSyncModel
 
 # To make REST API calls:
 import httpx
@@ -82,6 +83,7 @@ from icecream import IceCreamDebugger
 # All the blueprints:
 from api.blueprints.mail.oauth_request import mail_oauth_bp
 from api.blueprints.mail.oauth_callback import mail_callback_bp
+from api.blueprints.mail.sync import mail_sync_bp
 from api.blueprints.tech.chat_alerts import tech_chat_alert_bp
 from api.blueprints.test.callback import test_callback_bp
 
@@ -113,6 +115,7 @@ app = Quart(__name__, template_folder = r"../views")
 app = cors(app)
 app.register_blueprint(mail_oauth_bp, url_prefix = f"/{MODULE_BASE}/mail")
 app.register_blueprint(mail_callback_bp, url_prefix = f"/{MODULE_BASE}/mail")
+app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
 app.register_blueprint(tech_chat_alert_bp, url_prefix = f"/{MODULE_BASE}/tech/alert")
 app.register_blueprint(test_callback_bp, url_prefix = f"/{MODULE_BASE}/test")
 
@@ -288,6 +291,14 @@ async def app_startup(**kwargs):
         debug_prefix = "Mail-OAuth | ",
         debug_only_errors = True
     )
+    current_app.mail_sync_model = MailSyncModel(
+        cache = current_app.module_cache,
+        alert_url = current_app.script_data["alerts"]["url"],
+        http_client = current_app.http_client,
+        debug = enable_debugging,
+        debug_prefix = "Mail-Sync | ",
+        debug_only_errors = True
+    )
 
     # ┏┓
     # ┃ ┏┓┏┓┏┓┏┓┏╋┏┓┏┓┏

models/behaviour/mail/oauth.py

@@ -122,20 +122,10 @@ class MailOAuthModel(BaseModel):
         request_ts = date_time.get_current_utc_date_time(as_string = False)
 
         # Get the identifier from the database:
-        mongo_json = await mongo_conn.find_one_and_update(
+        inserted_id = await mongo_conn.insert_one(
             collection = MailOAuthModel.AUTH_COLLECTION,
-            filter = {
+            document = {
-                "serviceType": "email",
+                "version": "-1.0.1",
-                "client": service_client,
-                "authType": auth_type,
-                "user": user_info,
-            },
-            update = {
-                "$set": {
-                    "lastRequestTs": request_ts
-                },
-                "$setOnInsert": {
-                    "version": "1.0.0",
                 "serviceType": "email",
                 "client": service_client,
                 "authType": auth_type,
@@ -143,38 +133,65 @@ class MailOAuthModel(BaseModel):
                 "token": None,
                 "firstRefreshTs": None,
                 "lastRefreshTs": None,
-                    "firstRequestTs": request_ts,
+                "lastRequestTs": request_ts
             }
-            },
-            projection = {
-                "_id": True
-            },
-            upsert = True,
-            return_updated = True
         )
+        return inserted_id
 
-        # Tell MariaDB that an authorization request was initiated:
+        # # Get the identifier from the database:
-        db_json = {}
+        # mongo_json = await mongo_conn.find_one_and_update(
-        if mongo_json is not None:
+        #     collection = MailOAuthModel.AUTH_COLLECTION,
-            db_json = await self.call_procedure(
+        #     filter = {
-                db_conn = db_conn,
+        #         "serviceType": "email",
-                proc_name = "entity_integration_save",
+        #         "client": service_client,
-                proc_args = (
+        #         "authType": auth_type,
-                    user_info["entityId"],  # ............................................ 'p_entity_id'
+        #         "user": user_info,
-                    service_client,  # ................................................... 'p_provider'
+        #     },
-                    "Auth Requested",  # ................................................. 'p_current_status'
+        #     update = {
-                    "Auth URL Generated",  # ............................................. 'p_last_action'
+        #         "$set": {
-                    None,  # ............................................................. 'p_display_name'
+        #             "lastRequestTs": request_ts
-                    None,  # ............................................................. 'p_display_picture'
+        #         },
-                    str(mongo_json["_id"]),  # ........................................... 'p_token_id'
+        #         "$setOnInsert": {
-                    json.to_string(python_data = {"email": None}, no_space = True),  # ... 'p_notes'
+        #             "version": "1.0.0",
-                    user_info["userId"]  # ............................................... 'p_created_by'
+        #             "serviceType": "email",
-                ),
+        #             "client": service_client,
-                session_token = session_token
+        #             "authType": auth_type,
-            )
+        #             "user": user_info,
-
+        #             "token": None,
-        # Done here:
+        #             "firstRefreshTs": None,
-        return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
+        #             "lastRefreshTs": None,
+        #             "firstRequestTs": request_ts,
+        #         }
+        #     },
+        #     projection = {
+        #         "_id": True
+        #     },
+        #     upsert = True,
+        #     return_updated = True
+        # )
+        #
+        # # Tell MariaDB that an authorization request was initiated:
+        # db_json = {}
+        # if mongo_json is not None:
+        #     db_json = await self.call_procedure(
+        #         db_conn = db_conn,
+        #         proc_name = "entity_integration_save",
+        #         proc_args = (
+        #             user_info["entityId"],  # ............................................ 'p_entity_id'
+        #             service_client,  # ................................................... 'p_provider'
+        #             "Auth Requested",  # ................................................. 'p_current_status'
+        #             "Auth URL Generated",  # ............................................. 'p_last_action'
+        #             None,  # ............................................................. 'p_display_name'
+        #             None,  # ............................................................. 'p_display_picture'
+        #             str(mongo_json["_id"]),  # ........................................... 'p_token_id'
+        #             json.to_string(python_data = {"email": None}, no_space = True),  # ... 'p_notes'
+        #             user_info["userId"]  # ............................................... 'p_created_by'
+        #         ),
+        #         session_token = session_token
+        #     )
+        #
+        # # Done here:
+        # return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
 
     async def set_token(
             self,
@@ -210,6 +227,9 @@ class MailOAuthModel(BaseModel):
             update = {
                 "$set": {
                     "token": token,
+                    "lastRefreshTs": request_ts,
+                },
+                "$setOnInsert": {
                     "firstRefreshTs": request_ts,
                 }
             },

models/behaviour/mail/oauth_v2.py

@@ -6,7 +6,7 @@
 
     DATE:
 
-        Wednesday, 27th Nov., 2024
+        Monday, 2nd Dec., 2024
 
     OBJECTIVE:
 
@@ -95,11 +95,12 @@ class MailOAuthModel(BaseModel):
 
     AUTH_COLLECTION = "_authTokens"
 
-    async def get_user_identifier(
+    async def get_account_identifier(
             self,
             db_conn: AsyncMySQL,
             mongo_conn: AsyncMongo,
             user_info: dict,
+            email_id: str,
             service_client: Literal["gmail"],
             auth_type: Literal["oauth"],
             session_token: str = None
@@ -111,6 +112,7 @@ class MailOAuthModel(BaseModel):
         :param db_conn: The database connection (MariaDB) to use to perform the action.
         :param mongo_conn: The database connection (MongoDB) to use to perform the action.
         :param user_info: The dictionary that has the user's session information.
+        :param email_id: The e-mail id that the user wants to connect to your service.
         :param service_client: The name of the company or brand that is providing this service that is being integrated.
         :param auth_type: To identify the type of authentication being done here. This could indicate simple password
             authentication, more advance OAuth2.0 authentication, etc.
@@ -122,10 +124,22 @@ class MailOAuthModel(BaseModel):
         request_ts = date_time.get_current_utc_date_time(as_string = False)
 
         # Get the identifier from the database:
-        inserted_id = await mongo_conn.insert_one(
+        mongo_json = await mongo_conn.find_one_and_update(
             collection = MailOAuthModel.AUTH_COLLECTION,
-            document = {
+            filter = {
-                "version": "-1.0.1",
+                "serviceType": "email",
+                "user": {
+                    "entityId": user_info["entityId"],
+                    "billingAccountId": user_info["billingAccountId"]
+                },
+                "clientUserId": email_id
+            },
+            update = {
+                "$set": {
+                    "lastRequestTs": request_ts
+                },
+                "$setOnInsert": {
+                    "version": "1.1.0",
                     "serviceType": "email",
                     "client": service_client,
                     "authType": auth_type,
@@ -133,71 +147,45 @@ class MailOAuthModel(BaseModel):
                     "token": None,
                     "firstRefreshTs": None,
                     "lastRefreshTs": None,
-                "lastRequestTs": request_ts
+                    "firstRequestTs": request_ts,
                 }
+            },
+            projection = {
+                "_id": True
+            },
+            upsert = True,
+            return_updated = True
         )
-        return inserted_id
 
-        # # Get the identifier from the database:
+        # Tell MariaDB that an authorization request was initiated:
-        # mongo_json = await mongo_conn.find_one_and_update(
+        db_json = {}
-        #     collection = MailOAuthModel.AUTH_COLLECTION,
+        if mongo_json is not None:
-        #     filter = {
+            db_json = await self.call_procedure(
-        #         "serviceType": "email",
+                db_conn = db_conn,
-        #         "client": service_client,
+                proc_name = "entity_integration_save",
-        #         "authType": auth_type,
+                proc_args = (
-        #         "user": user_info,
+                    user_info["entityId"],  # ............................................ 'p_entity_id'
-        #     },
+                    service_client,  # ................................................... 'p_provider'
-        #     update = {
+                    "Auth Requested",  # ................................................. 'p_current_status'
-        #         "$set": {
+                    "Auth URL Generated",  # ............................................. 'p_last_action'
-        #             "lastRequestTs": request_ts
+                    None,  # ............................................................. 'p_display_name'
-        #         },
+                    None,  # ............................................................. 'p_display_picture'
-        #         "$setOnInsert": {
+                    str(mongo_json["_id"]),  # ........................................... 'p_token_id'
-        #             "version": "1.0.0",
+                    json.to_string(python_data = {"email": None}, no_space = True),  # ... 'p_notes'
-        #             "serviceType": "email",
+                    user_info["userId"]  # ............................................... 'p_created_by'
-        #             "client": service_client,
+                ),
-        #             "authType": auth_type,
+                session_token = session_token
-        #             "user": user_info,
+            )
-        #             "token": None,
+
-        #             "firstRefreshTs": None,
+        # Done here:
-        #             "lastRefreshTs": None,
+        return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
-        #             "firstRequestTs": request_ts,
-        #         }
-        #     },
-        #     projection = {
-        #         "_id": True
-        #     },
-        #     upsert = True,
-        #     return_updated = True
-        # )
-        #
-        # # Tell MariaDB that an authorization request was initiated:
-        # db_json = {}
-        # if mongo_json is not None:
-        #     db_json = await self.call_procedure(
-        #         db_conn = db_conn,
-        #         proc_name = "entity_integration_save",
-        #         proc_args = (
-        #             user_info["entityId"],  # ............................................ 'p_entity_id'
-        #             service_client,  # ................................................... 'p_provider'
-        #             "Auth Requested",  # ................................................. 'p_current_status'
-        #             "Auth URL Generated",  # ............................................. 'p_last_action'
-        #             None,  # ............................................................. 'p_display_name'
-        #             None,  # ............................................................. 'p_display_picture'
-        #             str(mongo_json["_id"]),  # ........................................... 'p_token_id'
-        #             json.to_string(python_data = {"email": None}, no_space = True),  # ... 'p_notes'
-        #             user_info["userId"]  # ............................................... 'p_created_by'
-        #         ),
-        #         session_token = session_token
-        #     )
-        #
-        # # Done here:
-        # return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
 
     async def set_token(
             self,
             db_conn: AsyncMySQL,
             mongo_conn: AsyncMongo,
-            user_identifier: ObjectId | str,
+            account_identifier: ObjectId | str,
+            email_id: str,
             token: dict,
             session_token: str = None
     ) -> bool:
@@ -208,7 +196,9 @@ class MailOAuthModel(BaseModel):
         ALSO.
         :param db_conn: The database connection (MariaDB) to use to perform the action.
         :param mongo_conn: The database connection (MongoDB) to use to perform the action.
-        :param user_identifier: The identifier granted by the 'get_user_identifier' method.
+        :param account_identifier: The identifier granted by the 'get_account_identifier' method.
+        :param email_id: The e-mail id that the user tried to connect to your service. This should match the e-mail id
+            the user claimed he wants to connect when he used 'get_account_identifier'.
         :param token: The token granted by the third-party service.
         :param session_token: The session token of the user who requested this service.
         :return: True if saved, False if failed.
@@ -223,7 +213,10 @@ class MailOAuthModel(BaseModel):
         # Save the token to MongoDB:
         mongo_json = await mongo_conn.find_one_and_update(
             collection = MailOAuthModel.AUTH_COLLECTION,
-            filter = {"_id": ObjectId(user_identifier)},
+            filter = {
+                "_id": ObjectId(account_identifier),
+                "clientUserId": email_id
+            },
             update = {
                 "$set": {
                     "token": token,
@@ -250,7 +243,7 @@ class MailOAuthModel(BaseModel):
                     "Set Token",  # ................................................................ 'p_last_action'
                     None,  # ....................................................................... 'p_display_name'
                     None,  # ....................................................................... 'p_display_picture'
-                    user_identifier,  # ............................................................ 'p_token_id'
+                    account_identifier,  # ......................................................... 'p_token_id'
                     json.to_string(python_data = {"email": token["email"]}, no_space = True),  # ... 'p_notes'
                     mongo_json["user"]["userId"]  # ................................................ 'p_created_by'
                 ),
@@ -264,14 +257,14 @@ class MailOAuthModel(BaseModel):
     async def get_token(
             self,
             mongo_conn: AsyncMongo,
-            user_identifier: ObjectId | str = None,
+            account_identifier: ObjectId | str = None,
             **kwargs
     ) -> dict | None:
 
         """
         To retrieve stored tokens from the database.
         :param mongo_conn: The database connection (MongoDB) to use to perform the action.
-        :param user_identifier: The identifier granted by the 'get_user_identifier' method.
+        :param account_identifier: The identifier granted by the 'account_identifier' method.
         :param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
             info, the client, the type of authentication used, or even the kind of service.
         :return: The retrieved record that has the token, and information about the service and client if found, else
@@ -280,7 +273,7 @@ class MailOAuthModel(BaseModel):
 
         # Build the filter:
         filter_json = {k: v for k, v in kwargs.items()}
-        if user_identifier: filter_json["_id"] = ObjectId(user_identifier)
+        if account_identifier: filter_json["_id"] = ObjectId(account_identifier)
 
         # If there is no search criteria, we exit with failure:
         if not filter_json: return None

models/behaviour/mail/sync.py

@@ -34,6 +34,7 @@
 import sys
 
 from langchain.chains.summarize.stuff_prompt import prompt_template
+from sqlalchemy.orm.collections import collection
 
 sys.path.append(".")
 sys.path.append("..")
@@ -53,6 +54,7 @@ from models.behaviour.base import BaseModel
 
 # To work with MongoDB:
 from bson import ObjectId
+from pymongo import InsertOne, UpdateOne
 
 # To work with LLMs:
 from langchain_openai import ChatOpenAI
@@ -64,6 +66,12 @@ from typing import Literal
 # To make deep-copies:
 import copy
 
+# To work with date and time:
+import datetime
+
+# For asynchronous activities:
+import asyncio
+
 
 #   *****************************************************************************************************************
 #  *****                                                                                                          ****
@@ -112,7 +120,7 @@ class MailSyncModel(BaseModel):
     prompt_template = ChatPromptTemplate.from_messages([
         (
             "system",
-            "You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPS) FROM THE SUMMARY."
+            "You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPs) FROM THE SUMMARY."
         ),
         (
             "user",
@@ -120,17 +128,24 @@ class MailSyncModel(BaseModel):
         )
     ])
 
-    async def sync_one(
+    async def __sync_one(
             self,
             mongo_conn: AsyncMongo,
-            user_identifier: str | ObjectId,
+            user_info: dict,
             mail_client: AsyncGMailClient,
             tokens: GoogleAuthTokens,
             message_id: str,
             llm: ChatOpenAI = None,
-            session_token: str = None,
             force_sync: bool = False
-    ) -> ObjectId:
+    ) -> UpdateOne | None:
+
+        # ┏┓              ┓┏    •  ┓ ┓
+        # ┃┃┏┓┏┓┏┓┏┓┏┓┏┓  ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
+        # ┣┛┛ ┗ ┣┛┗┻┛ ┗   ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
+        #       ┛
+
+        mail_payload = None
+        mail_client_name = None
 
         # ┏┓┓    ┓   ┏┓  •  •      ┳┓        ┓
         # ┃ ┣┓┏┓┏┃┏  ┣ ┓┏┓┏╋┓┏┓┏┓  ┣┫┏┓┏┏┓┏┓┏┫┏
@@ -150,15 +165,7 @@ class MailSyncModel(BaseModel):
         )
 
         # If there already exists such a record, and we haven't been forced to re-sync it:
-        if existing_record and not force_sync: return existing_record["_id"]
+        if existing_record and not force_sync: return mail_payload
-
-        # ┏┓              ┓┏    •  ┓ ┓
-        # ┃┃┏┓┏┓┏┓┏┓┏┓┏┓  ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
-        # ┣┛┛ ┗ ┣┛┗┻┛ ┗   ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
-        #       ┛
-
-        mail_payload = None
-        mail_id = existing_record["_id"] if existing_record else None
 
         # ┏┓┳┳┓  •┓
         # ┃┓┃┃┃┏┓┓┃
@@ -166,10 +173,10 @@ class MailSyncModel(BaseModel):
 
         if isinstance(mail_client, AsyncGMailClient):
 
-            # Refresh the tokens:
+            # Note down the name of the mail client:
-            tokens_refreshed
+            mail_client_name = "gmail"
 
-            # Fetch the mail formatted message:
+            # Fetch the formatted mail message:
             client_response = await mail_client.get_message(
                 tokens = tokens,
                 message_id = message_id,
@@ -180,20 +187,132 @@ class MailSyncModel(BaseModel):
             if client_response.success:
 
                 # Summarize the content:
-                prompt = self.prompt_template.invoke({"mail": client_response.data.pop["unformattedText"]})
+                if llm:
+                    prompt = self.prompt_template.invoke({"mail": client_response.data["unformattedText"]})
                     llm_response = await llm.ainvoke(prompt)
                     client_response.data["aiSnippet"] = llm_response.content
 
                 # Note down the response:
                 mail_payload = client_response.data
 
-        # ┏┓       ┏┳┓┓     ┳┳┓  •┓
+        # ┳┓
-        # ┗┓┓┏┏┓┏   ┃ ┣┓┏┓  ┃┃┃┏┓┓┃
+        # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
-        # ┗┛┗┫┛┗┗   ┻ ┛┗┗   ┛ ┗┗┻┗┗
+        # ┛┗┗ ┛┣┛┗┛┛┗┛┗
         #      ┛
 
         if mail_payload:
-            pass
+            return UpdateOne(
+                filter = {
+                    "messageType": "email",
+                    "$or": [
+                        {"payload.messageId": message_id}
+                    ]
+                },
+                update = {
+                    "$set": {
+                        "readTs": date_time.get_current_utc_date_time(),
+                        "user": user_info,
+                        "messageType": "email",
+                        "connector": mail_client_name,
+                        "payload": mail_payload
+                    }
+                },
+                upsert = True
+            )
+
+        # Done here:
+        return mail_payload
+
+    async def sync(
+            self,
+            mongo_conn: AsyncMongo,
+            user_info: dict,
+            mail_client: AsyncGMailClient,
+            tokens: GoogleAuthTokens,
+            llm: ChatOpenAI = None,
+            force_sync: bool = False,
+            start_date: datetime.datetime = None,
+            end_date: datetime.datetime = None,
+            max_count: int = 100
+    ) -> int:
+
+        # Start by assuming failure:
+        mails_count = 0
+
+        # ┏┓┳┳┓  •┓
+        # ┃┓┃┃┃┏┓┓┃
+        # ┗┛┛ ┗┗┻┗┗
+
+        if isinstance(mail_client, AsyncGMailClient):
+
+            # Enlist all the labels, we need to find the label that indicates that we've read the mail:
+            client_response = await mail_client.list_labels(tokens = tokens)
+            if not client_response.success: return mails_count
+            labels = client_response.data
+            custom_label = "Sync'd with TheCAOffice"
+            custom_label_id = labels.get(custom_label)
+            if custom_label_id is None:
+                client_response = await mail_client.create_label(
+                    tokens = tokens,
+                    label_name = custom_label,
+                    label_visibility = "labelHide"
+                )
+                if not client_response.success: return mails_count
+                custom_label_id = client_response.data["id"]
+
+            # Build the query:
+            sub_queries = [f"-label:\"{custom_label}\""]
+            if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
+            if end_date: sub_queries.append(end_date.strftime("before:%Y/%m/%d"))
+            print("Q:", " ".join(sub_queries))
+
+            # Get a list of all the mails:
+            client_response = await mail_client.list_messages(
+                tokens = tokens,
+                max_count = max_count,
+                # query = " ".join(sub_queries)
+            )
+            if not client_response.success: return mails_count
+            messages_list = client_response.data["messages"]
+            print(messages_list)
+
+            # Create MongoDB operations for all the mails:
+            tasks = [
+                self.__sync_one(
+                    mongo_conn = mongo_conn,
+                    user_info = user_info,
+                    mail_client = mail_client,
+                    tokens = tokens,
+                    message_id = v["id"],
+                    llm = llm,
+                    force_sync = force_sync
+                )
+                for k, v in messages_list.items()
+            ]
+            mongo_operations = await asyncio.gather(*tasks)
+            mongo_operations = [mo for mo in mongo_operations if mo is not None]
+
+            # Write the mails to MongoDB:
+            mails_count = await mongo_conn.bulk_write(
+                collection = self.MAIL_COLLECTION,
+                requests = mongo_operations
+            )
+            print("MAILS COUNT:", mails_count)
+
+            # If all the mails were sync'd properly:
+            client_response = await mail_client.modify_messages(
+                tokens = tokens,
+                message_ids = [v["id"] for k, v in messages_list.items()],
+                add_label_ids = [custom_label_id]
+            )
+
+        # ┳┓
+        # ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
+        # ┛┗┗ ┛┣┛┗┛┛┗┛┗
+        #      ┛
+
+        # Done here:
+        return mails_count
 
 
 #   *****************************************************************************************************************

models/data/mail/oauth.py

@@ -103,6 +103,12 @@ class OAuthMailAuthorizationRequestData(BaseModel):
         frozen = True
     )
 
+    mailId: str = Field(
+        description = "the e-mail id that the user intends to authorize",
+        pattern = regex.REGEX_EMAIL_ID,
+        frozen = True
+    )
+
     # ┏┓    ┏•
     # ┃ ┏┓┏┓╋┓┏┓
     # ┗┛┗┛┛┗┛┗┗┫

models/data/mail/sync.py

@@ -6,12 +6,11 @@
 
     DATE:
 
-        Wednesday, 27th Nov., 2024.
+        Monday, 2nd Dec., 2024.
 
     OBJECTIVE:
 
-        To provide the structure for the request and response of the APIs that will be used to request OAuth2.0
+        To provide the structure for the request that will come in to sync the mails of a particular user.
-        authorization for mail services.
 
     REFERENCES:
 
@@ -37,11 +36,15 @@ sys.path.append(".")
 sys.path.append("..")
 
 # For making data behaviour_models:
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, field_validator, PastDatetime
 from typing import Optional, Literal
 
 # My utils:
 from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# To work with date and time:
+import datetime
 
 
 #   *****************************************************************************************************************
@@ -72,7 +75,7 @@ REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]
 #   *****************************************************************************************************************
 
 
-class OAuthMailAuthorizationRequestHeaders(BaseModel):
+class MailSyncRequestHeaders(BaseModel):
 
     sessionToken: str = Field(
         description = "the session token of the user who is requesting the service",
@@ -96,10 +99,23 @@ class OAuthMailAuthorizationRequestHeaders(BaseModel):
 # ---------------------------------------------------------------------------------------------------------------------
 
 
-class OAuthMailAuthorizationRequestData(BaseModel):
+class MailSyncRequestData(BaseModel):
 
-    mailClient: Literal["gmail"] = Field(
+    maxCount: int = Field(
-        description = "the e-mail provider like 'gmail'",
+        description = "the max. no. of e-mails to sync at a given time",
+        default = 100,
+        frozen = True
+    )
+
+    startDate: PastDatetime = Field(
+        description = "the starting date from which the user wants to sync their mail",
+        default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1),
+        frozen = True
+    )
+
+    endDate: PastDatetime = Field(
+        description = "the ending date till which the user wants to sync their mail",
+        default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1),
         frozen = True
     )
 
@@ -115,9 +131,13 @@ class OAuthMailAuthorizationRequestData(BaseModel):
     # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
     # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
 
-    @field_validator("mailClient", mode = "before")
+    @field_validator("startDate", "endDate", mode = "before")
-    def to_lowercase(cls, value):
+    def to_datetime(cls, value):
-        if isinstance(value, str): value = value.strip().lower()
+        if not isinstance(value, datetime.datetime):
+            value = date_time.parse_date_time(
+                input_value = value,
+                timezone = date_time.TIMEZONE_UTC
+            )
         return value
 
 

views/mail/oauth/oauth_failure_v2.html

@@ -102,6 +102,7 @@
         <div class="failure-circle">✘</div> <!-- Red circle with failure icon -->
         <h1><b>Authorization Failed</b></h1>
         <p>Something went wrong in getting authorization from your <b>{{ mail_client }}</b> account.
+            <br><br><b>Hint:</b> {{ failure_hint }}<br><br>
             Please feel free to try the same steps again. You can close this tab at any time.</p>
 
         <!-- Close button -->