khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241202) Testing out the modified auth process.

Browse Source
This commit is contained in:
khushal
2024-12-02 18:47:38 +05:30
parent aed746190d
commit bcbad8ecdb
10 changed files with 419 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/blueprints/mail/oauth_callback.py
+29 -4
View File
@@ -164,19 +164,35 @@ async def mail_callback(
if tokens:
# Add information and:
# Get the e-mail id that granted authorization:
user_profile = await current_app.gmail_client.get_user_profile(tokens = tokens)
tokens.email = user_profile.data["emailAddress"] if user_profile.success else None
# The e-mail id that we requested access to and the one that granted us access should be the same:
placeholder_token = await current_app.mail_oauth_model.set_token(
mongo_conn = current_app.data_mongo,
account_identifier = inbound_data["state"]
)
if (
(not placeholder_token) or
placeholder_token["clientUserId"] != str(tokens.email)
): return await render_template(
"/mail/oauth/oauth_failure_v2.html",
mail_client = mail_client.title(),
failure_hint = f"We were expecting authorization from '{placeholder_token['clientUserId']}' but got authorization from '{tokens.email}' instead."
)
# Save the tokens to the database
tokens_saved = await current_app.mail_oauth_model.set_token(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
session_token = inbound_headers.get("X-Session-Token"),
user_identifier = inbound_data["state"],
account_identifier = inbound_data["state"],
token = tokens.model_dump()
)
# tokens_saved = False
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
@@ -192,12 +208,21 @@ async def mail_callback(
# }
# )
# Return an HTML response:
# Return an HTML response for success:
if tokens_saved:
return await render_template(
"/mail/oauth/oauth_success_v2.html" if tokens_saved else "/mail/oauth/oauth_failure_v2.html",
"/mail/oauth/oauth_success_v2.html",
mail_client = mail_client.title()
)
# Return an HTML response for failure:
else:
return await render_template(
"/mail/oauth/oauth_failure_v2.html",
mail_client = mail_client.title(),
failure_hint = f"Unknown error. Please use log-id '{kwargs['logId']}' to check with the support team."
)
# *****************************************************************************************************************
# ***** ****
api/blueprints/mail/oauth_request.py
+3 -2
View File
@@ -175,11 +175,12 @@ async def request_oauth_authorization_url(
# ┛
# Make a user identifier from the session info:
user_identifier = await current_app.mail_oauth_model.get_user_identifier(
user_identifier = await current_app.mail_oauth_model.get_account_identifier(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
session_token = inbound_headers["X-Session-Token"],
user_info = kwargs["session_info"],
email_id = inbound_data.mailId,
service_client = inbound_data.mailClient,
auth_type = "oauth"
)
@@ -204,7 +205,7 @@ async def request_oauth_authorization_url(
access_type = "offline",
approval_prompt = "force",
include_granted_scopes = "true",
user_email = None
user_email = inbound_data.mailId
)
# ┳┓
api/blueprints/mail/sync.py
+56 -6
View File
@@ -63,6 +63,7 @@ from utils_v2.api.async_quart import (
# GMail-related utils:
from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
# Common:
from shared import constants
@@ -113,7 +114,7 @@ def init(blueprint_setup_state):
# ---------------------------------------------------------------------------------------------------------------------
@mail_sync_bp.route("/sync", methods = ["GET"])
@mail_sync_bp.route("/sync", methods = ["POST"])
@set_api_version(api_version = "1.0.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
@@ -163,19 +164,65 @@ async def sync_mail(
)
# Start by assuming failure:
auth_url = None
mails_count = 0
# ┏┓ ┏┳┓ ┓
# ┃┓┏┓╋ ┃ ┏┓┃┏┏┓┏┓
# ┗┛┗ ┗ ┻ ┗┛┛┗┗ ┛┗
# Make a user identifier from the session info:
user_tokens = await current_app.mail_oauth_model.get_token(
user_auth = await current_app.mail_oauth_model.get_token(
mongo_conn = current_app.data_mongo,
user = kwargs["session_info"]
serviceType = "email",
user = kwargs["session_info"],
)
print("MAIL TOKEN(S):", json.to_string(user_tokens))
print("MAIL TOKEN(S):", json.to_string(user_auth, default = str))
if not user_auth: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "user's email not connected"
)
# ┏┓┳┳┓ •┓
# ┃┓┃┃┃┏┓┓┃
# ┗┛┛ ┗┗┻┗┗
if user_auth["client"] == "gmail":
# Load the tokens into an object:
user_tokens = GoogleAuthTokens(**user_auth["token"])
# Try refreshing the tokens:
tokens_refreshed = await user_tokens.arefresh(
http_client = current_app.http_client,
client_id = current_app.gmail_client.client_id,
client_secret = current_app.gmail_client.client_secret,
force_refresh = False
)
# Update the token in the database if needed:
if tokens_refreshed: await current_app.mail_oauth_model.set_token(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
user_identifier = user_auth["_id"],
token = user_tokens.model_dump(),
session_token = inbound_headers["X-Session-Token"]
)
# Now we try to sync the mails:
mails_count = await current_app.mail_sync_model.sync(
mongo_conn = current_app.data_mongo,
user_info = kwargs["session_info"],
mail_client = current_app.gmail_client,
tokens = user_tokens,
llm = None,
force_sync = False,
start_date = inbound_data.startDate,
end_date = inbound_data.endDate,
max_count = inbound_data.maxCount
)
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
@@ -183,7 +230,10 @@ async def sync_mail(
# ┛
# Done here:
return ResponseModel(status_code = StatusCodes.OK)
return ResponseModel(
status_code = StatusCodes.OK if mails_count else StatusCodes.FAILED,
message = f"{mails_count} mail(s) sync'd"
)
# *****************************************************************************************************************
api/main.py
+12 -1
View File
@@ -71,7 +71,8 @@ from utils_v2.api.async_quart import (
from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
# Behaviour Models:
from models.behaviour.mail.oauth import MailOAuthModel
from models.behaviour.mail.oauth_v2 import MailOAuthModel
from models.behaviour.mail.sync import MailSyncModel
# To make REST API calls:
import httpx
@@ -82,6 +83,7 @@ from icecream import IceCreamDebugger
# All the blueprints:
from api.blueprints.mail.oauth_request import mail_oauth_bp
from api.blueprints.mail.oauth_callback import mail_callback_bp
from api.blueprints.mail.sync import mail_sync_bp
from api.blueprints.tech.chat_alerts import tech_chat_alert_bp
from api.blueprints.test.callback import test_callback_bp
@@ -113,6 +115,7 @@ app = Quart(__name__, template_folder = r"../views")
app = cors(app)
app.register_blueprint(mail_oauth_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_callback_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(tech_chat_alert_bp, url_prefix = f"/{MODULE_BASE}/tech/alert")
app.register_blueprint(test_callback_bp, url_prefix = f"/{MODULE_BASE}/test")
@@ -288,6 +291,14 @@ async def app_startup(**kwargs):
debug_prefix = "Mail-OAuth | ",
debug_only_errors = True
)
current_app.mail_sync_model = MailSyncModel(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "Mail-Sync | ",
debug_only_errors = True
)
# ┏┓
# ┃ ┏┓┏┓┏┓┏┓┏╋┏┓┏┓┏
models/behaviour/mail/oauth.py
+62 -42
View File
@@ -122,20 +122,10 @@ class MailOAuthModel(BaseModel):
request_ts = date_time.get_current_utc_date_time(as_string = False)
# Get the identifier from the database:
mongo_json = await mongo_conn.find_one_and_update(
inserted_id = await mongo_conn.insert_one(
collection = MailOAuthModel.AUTH_COLLECTION,
filter = {
"serviceType": "email",
"client": service_client,
"authType": auth_type,
"user": user_info,
},
update = {
"$set": {
"lastRequestTs": request_ts
},
"$setOnInsert": {
"version": "1.0.0",
document = {
"version": "-1.0.1",
"serviceType": "email",
"client": service_client,
"authType": auth_type,
@@ -143,38 +133,65 @@ class MailOAuthModel(BaseModel):
"token": None,
"firstRefreshTs": None,
"lastRefreshTs": None,
"firstRequestTs": request_ts,
"lastRequestTs": request_ts
}
},
projection = {
"_id": True
},
upsert = True,
return_updated = True
)
return inserted_id
# Tell MariaDB that an authorization request was initiated:
db_json = {}
if mongo_json is not None:
db_json = await self.call_procedure(
db_conn = db_conn,
proc_name = "entity_integration_save",
proc_args = (
user_info["entityId"], # ............................................ 'p_entity_id'
service_client, # ................................................... 'p_provider'
"Auth Requested", # ................................................. 'p_current_status'
"Auth URL Generated", # ............................................. 'p_last_action'
None, # ............................................................. 'p_display_name'
None, # ............................................................. 'p_display_picture'
str(mongo_json["_id"]), # ........................................... 'p_token_id'
json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
user_info["userId"] # ............................................... 'p_created_by'
),
session_token = session_token
)
# Done here:
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
# # Get the identifier from the database:
# mongo_json = await mongo_conn.find_one_and_update(
# collection = MailOAuthModel.AUTH_COLLECTION,
# filter = {
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# },
# update = {
# "$set": {
# "lastRequestTs": request_ts
# },
# "$setOnInsert": {
# "version": "1.0.0",
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# "token": None,
# "firstRefreshTs": None,
# "lastRefreshTs": None,
# "firstRequestTs": request_ts,
# }
# },
# projection = {
# "_id": True
# },
# upsert = True,
# return_updated = True
# )
#
# # Tell MariaDB that an authorization request was initiated:
# db_json = {}
# if mongo_json is not None:
# db_json = await self.call_procedure(
# db_conn = db_conn,
# proc_name = "entity_integration_save",
# proc_args = (
# user_info["entityId"], # ............................................ 'p_entity_id'
# service_client, # ................................................... 'p_provider'
# "Auth Requested", # ................................................. 'p_current_status'
# "Auth URL Generated", # ............................................. 'p_last_action'
# None, # ............................................................. 'p_display_name'
# None, # ............................................................. 'p_display_picture'
# str(mongo_json["_id"]), # ........................................... 'p_token_id'
# json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
# user_info["userId"] # ............................................... 'p_created_by'
# ),
# session_token = session_token
# )
#
# # Done here:
# return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
async def set_token(
self,
@@ -210,6 +227,9 @@ class MailOAuthModel(BaseModel):
update = {
"$set": {
"token": token,
"lastRefreshTs": request_ts,
},
"$setOnInsert": {
"firstRefreshTs": request_ts,
}
},
models/behaviour/mail/oauth_v2.py
+61 -68
View File
@@ -6,7 +6,7 @@
DATE:
Wednesday, 27th Nov., 2024
Monday, 2nd Dec., 2024
OBJECTIVE:
@@ -95,11 +95,12 @@ class MailOAuthModel(BaseModel):
AUTH_COLLECTION = "_authTokens"
async def get_user_identifier(
async def get_account_identifier(
self,
db_conn: AsyncMySQL,
mongo_conn: AsyncMongo,
user_info: dict,
email_id: str,
service_client: Literal["gmail"],
auth_type: Literal["oauth"],
session_token: str = None
@@ -111,6 +112,7 @@ class MailOAuthModel(BaseModel):
:param db_conn: The database connection (MariaDB) to use to perform the action.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param user_info: The dictionary that has the user's session information.
:param email_id: The e-mail id that the user wants to connect to your service.
:param service_client: The name of the company or brand that is providing this service that is being integrated.
:param auth_type: To identify the type of authentication being done here. This could indicate simple password
authentication, more advance OAuth2.0 authentication, etc.
@@ -122,10 +124,22 @@ class MailOAuthModel(BaseModel):
request_ts = date_time.get_current_utc_date_time(as_string = False)
# Get the identifier from the database:
inserted_id = await mongo_conn.insert_one(
mongo_json = await mongo_conn.find_one_and_update(
collection = MailOAuthModel.AUTH_COLLECTION,
document = {
"version": "-1.0.1",
filter = {
"serviceType": "email",
"user": {
"entityId": user_info["entityId"],
"billingAccountId": user_info["billingAccountId"]
},
"clientUserId": email_id
},
update = {
"$set": {
"lastRequestTs": request_ts
},
"$setOnInsert": {
"version": "1.1.0",
"serviceType": "email",
"client": service_client,
"authType": auth_type,
@@ -133,71 +147,45 @@ class MailOAuthModel(BaseModel):
"token": None,
"firstRefreshTs": None,
"lastRefreshTs": None,
"lastRequestTs": request_ts
"firstRequestTs": request_ts,
}
},
projection = {
"_id": True
},
upsert = True,
return_updated = True
)
return inserted_id
# # Get the identifier from the database:
# mongo_json = await mongo_conn.find_one_and_update(
# collection = MailOAuthModel.AUTH_COLLECTION,
# filter = {
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# },
# update = {
# "$set": {
# "lastRequestTs": request_ts
# },
# "$setOnInsert": {
# "version": "1.0.0",
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# "token": None,
# "firstRefreshTs": None,
# "lastRefreshTs": None,
# "firstRequestTs": request_ts,
# }
# },
# projection = {
# "_id": True
# },
# upsert = True,
# return_updated = True
# )
#
# # Tell MariaDB that an authorization request was initiated:
# db_json = {}
# if mongo_json is not None:
# db_json = await self.call_procedure(
# db_conn = db_conn,
# proc_name = "entity_integration_save",
# proc_args = (
# user_info["entityId"], # ............................................ 'p_entity_id'
# service_client, # ................................................... 'p_provider'
# "Auth Requested", # ................................................. 'p_current_status'
# "Auth URL Generated", # ............................................. 'p_last_action'
# None, # ............................................................. 'p_display_name'
# None, # ............................................................. 'p_display_picture'
# str(mongo_json["_id"]), # ........................................... 'p_token_id'
# json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
# user_info["userId"] # ............................................... 'p_created_by'
# ),
# session_token = session_token
# )
#
# # Done here:
# return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
# Tell MariaDB that an authorization request was initiated:
db_json = {}
if mongo_json is not None:
db_json = await self.call_procedure(
db_conn = db_conn,
proc_name = "entity_integration_save",
proc_args = (
user_info["entityId"], # ............................................ 'p_entity_id'
service_client, # ................................................... 'p_provider'
"Auth Requested", # ................................................. 'p_current_status'
"Auth URL Generated", # ............................................. 'p_last_action'
None, # ............................................................. 'p_display_name'
None, # ............................................................. 'p_display_picture'
str(mongo_json["_id"]), # ........................................... 'p_token_id'
json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
user_info["userId"] # ............................................... 'p_created_by'
),
session_token = session_token
)
# Done here:
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
async def set_token(
self,
db_conn: AsyncMySQL,
mongo_conn: AsyncMongo,
user_identifier: ObjectId | str,
account_identifier: ObjectId | str,
email_id: str,
token: dict,
session_token: str = None
) -> bool:
@@ -208,7 +196,9 @@ class MailOAuthModel(BaseModel):
ALSO.
:param db_conn: The database connection (MariaDB) to use to perform the action.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param user_identifier: The identifier granted by the 'get_user_identifier' method.
:param account_identifier: The identifier granted by the 'get_account_identifier' method.
:param email_id: The e-mail id that the user tried to connect to your service. This should match the e-mail id
the user claimed he wants to connect when he used 'get_account_identifier'.
:param token: The token granted by the third-party service.
:param session_token: The session token of the user who requested this service.
:return: True if saved, False if failed.
@@ -223,7 +213,10 @@ class MailOAuthModel(BaseModel):
# Save the token to MongoDB:
mongo_json = await mongo_conn.find_one_and_update(
collection = MailOAuthModel.AUTH_COLLECTION,
filter = {"_id": ObjectId(user_identifier)},
filter = {
"_id": ObjectId(account_identifier),
"clientUserId": email_id
},
update = {
"$set": {
"token": token,
@@ -250,7 +243,7 @@ class MailOAuthModel(BaseModel):
"Set Token", # ................................................................ 'p_last_action'
None, # ....................................................................... 'p_display_name'
None, # ....................................................................... 'p_display_picture'
user_identifier, # ............................................................ 'p_token_id'
account_identifier, # ......................................................... 'p_token_id'
json.to_string(python_data = {"email": token["email"]}, no_space = True), # ... 'p_notes'
mongo_json["user"]["userId"] # ................................................ 'p_created_by'
),
@@ -264,14 +257,14 @@ class MailOAuthModel(BaseModel):
async def get_token(
self,
mongo_conn: AsyncMongo,
user_identifier: ObjectId | str = None,
account_identifier: ObjectId | str = None,
**kwargs
) -> dict | None:
"""
To retrieve stored tokens from the database.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param user_identifier: The identifier granted by the 'get_user_identifier' method.
:param account_identifier: The identifier granted by the 'account_identifier' method.
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
info, the client, the type of authentication used, or even the kind of service.
:return: The retrieved record that has the token, and information about the service and client if found, else
@@ -280,7 +273,7 @@ class MailOAuthModel(BaseModel):
# Build the filter:
filter_json = {k: v for k, v in kwargs.items()}
if user_identifier: filter_json["_id"] = ObjectId(user_identifier)
if account_identifier: filter_json["_id"] = ObjectId(account_identifier)
# If there is no search criteria, we exit with failure:
if not filter_json: return None
models/behaviour/mail/sync.py
+141 -22
View File
@@ -34,6 +34,7 @@
import sys
from langchain.chains.summarize.stuff_prompt import prompt_template
from sqlalchemy.orm.collections import collection
sys.path.append(".")
sys.path.append("..")
@@ -53,6 +54,7 @@ from models.behaviour.base import BaseModel
# To work with MongoDB:
from bson import ObjectId
from pymongo import InsertOne, UpdateOne
# To work with LLMs:
from langchain_openai import ChatOpenAI
@@ -64,6 +66,12 @@ from typing import Literal
# To make deep-copies:
import copy
# To work with date and time:
import datetime
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
@@ -112,7 +120,7 @@ class MailSyncModel(BaseModel):
prompt_template = ChatPromptTemplate.from_messages([
(
"system",
"You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPS) FROM THE SUMMARY."
"You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPs) FROM THE SUMMARY."
),
(
"user",
@@ -120,17 +128,24 @@ class MailSyncModel(BaseModel):
)
])
async def sync_one(
async def __sync_one(
self,
mongo_conn: AsyncMongo,
user_identifier: str | ObjectId,
user_info: dict,
mail_client: AsyncGMailClient,
tokens: GoogleAuthTokens,
message_id: str,
llm: ChatOpenAI = None,
session_token: str = None,
force_sync: bool = False
) -> ObjectId:
) -> UpdateOne | None:
# ┏┓ ┓┏ • ┓ ┓
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
# ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
# ┛
mail_payload = None
mail_client_name = None
# ┏┓┓ ┓ ┏┓ • • ┳┓ ┓
# ┃ ┣┓┏┓┏┃┏ ┣ ┓┏┓┏╋┓┏┓┏┓ ┣┫┏┓┏┏┓┏┓┏┫┏
@@ -150,15 +165,7 @@ class MailSyncModel(BaseModel):
)
# If there already exists such a record, and we haven't been forced to re-sync it:
if existing_record and not force_sync: return existing_record["_id"]
# ┏┓ ┓┏ • ┓ ┓
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
# ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
# ┛
mail_payload = None
mail_id = existing_record["_id"] if existing_record else None
if existing_record and not force_sync: return mail_payload
# ┏┓┳┳┓ •┓
# ┃┓┃┃┃┏┓┓┃
@@ -166,10 +173,10 @@ class MailSyncModel(BaseModel):
if isinstance(mail_client, AsyncGMailClient):
# Refresh the tokens:
tokens_refreshed
# Note down the name of the mail client:
mail_client_name = "gmail"
# Fetch the mail formatted message:
# Fetch the formatted mail message:
client_response = await mail_client.get_message(
tokens = tokens,
message_id = message_id,
@@ -180,20 +187,132 @@ class MailSyncModel(BaseModel):
if client_response.success:
# Summarize the content:
prompt = self.prompt_template.invoke({"mail": client_response.data.pop["unformattedText"]})
if llm:
prompt = self.prompt_template.invoke({"mail": client_response.data["unformattedText"]})
llm_response = await llm.ainvoke(prompt)
client_response.data["aiSnippet"] = llm_response.content
# Note down the response:
mail_payload = client_response.data
# ┏┓ ┏┳┓┓ ┳┳┓ •┓
# ┗┓┓┏┏┓┏ ┃ ┣┓┏┓ ┃┃┃┏┓┓┃
# ┗┛┗┫┛┗┗ ┻ ┛┗┗ ┛ ┗┗┻┗┗
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
# ┛
if mail_payload:
pass
return UpdateOne(
filter = {
"messageType": "email",
"$or": [
{"payload.messageId": message_id}
]
},
update = {
"$set": {
"readTs": date_time.get_current_utc_date_time(),
"user": user_info,
"messageType": "email",
"connector": mail_client_name,
"payload": mail_payload
}
},
upsert = True
)
# Done here:
return mail_payload
async def sync(
self,
mongo_conn: AsyncMongo,
user_info: dict,
mail_client: AsyncGMailClient,
tokens: GoogleAuthTokens,
llm: ChatOpenAI = None,
force_sync: bool = False,
start_date: datetime.datetime = None,
end_date: datetime.datetime = None,
max_count: int = 100
) -> int:
# Start by assuming failure:
mails_count = 0
# ┏┓┳┳┓ •┓
# ┃┓┃┃┃┏┓┓┃
# ┗┛┛ ┗┗┻┗┗
if isinstance(mail_client, AsyncGMailClient):
# Enlist all the labels, we need to find the label that indicates that we've read the mail:
client_response = await mail_client.list_labels(tokens = tokens)
if not client_response.success: return mails_count
labels = client_response.data
custom_label = "Sync'd with TheCAOffice"
custom_label_id = labels.get(custom_label)
if custom_label_id is None:
client_response = await mail_client.create_label(
tokens = tokens,
label_name = custom_label,
label_visibility = "labelHide"
)
if not client_response.success: return mails_count
custom_label_id = client_response.data["id"]
# Build the query:
sub_queries = [f"-label:\"{custom_label}\""]
if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
if end_date: sub_queries.append(end_date.strftime("before:%Y/%m/%d"))
print("Q:", " ".join(sub_queries))
# Get a list of all the mails:
client_response = await mail_client.list_messages(
tokens = tokens,
max_count = max_count,
# query = " ".join(sub_queries)
)
if not client_response.success: return mails_count
messages_list = client_response.data["messages"]
print(messages_list)
# Create MongoDB operations for all the mails:
tasks = [
self.__sync_one(
mongo_conn = mongo_conn,
user_info = user_info,
mail_client = mail_client,
tokens = tokens,
message_id = v["id"],
llm = llm,
force_sync = force_sync
)
for k, v in messages_list.items()
]
mongo_operations = await asyncio.gather(*tasks)
mongo_operations = [mo for mo in mongo_operations if mo is not None]
# Write the mails to MongoDB:
mails_count = await mongo_conn.bulk_write(
collection = self.MAIL_COLLECTION,
requests = mongo_operations
)
print("MAILS COUNT:", mails_count)
# If all the mails were sync'd properly:
client_response = await mail_client.modify_messages(
tokens = tokens,
message_ids = [v["id"] for k, v in messages_list.items()],
add_label_ids = [custom_label_id]
)
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
# ┛
# Done here:
return mails_count
# *****************************************************************************************************************
models/data/mail/oauth.py
+6
View File
@@ -103,6 +103,12 @@ class OAuthMailAuthorizationRequestData(BaseModel):
frozen = True
)
mailId: str = Field(
description = "the e-mail id that the user intends to authorize",
pattern = regex.REGEX_EMAIL_ID,
frozen = True
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
models/data/mail/sync.py
+31 -11
View File
@@ -6,12 +6,11 @@
DATE:
Wednesday, 27th Nov., 2024.
Monday, 2nd Dec., 2024.
OBJECTIVE:
To provide the structure for the request and response of the APIs that will be used to request OAuth2.0
authorization for mail services.
To provide the structure for the request that will come in to sync the mails of a particular user.
REFERENCES:
@@ -37,11 +36,15 @@ sys.path.append(".")
sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator
from pydantic import BaseModel, Field, field_validator, PastDatetime
from typing import Optional, Literal
# My utils:
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with date and time:
import datetime
# *****************************************************************************************************************
@@ -72,7 +75,7 @@ REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]
# *****************************************************************************************************************
class OAuthMailAuthorizationRequestHeaders(BaseModel):
class MailSyncRequestHeaders(BaseModel):
sessionToken: str = Field(
description = "the session token of the user who is requesting the service",
@@ -96,10 +99,23 @@ class OAuthMailAuthorizationRequestHeaders(BaseModel):
# ---------------------------------------------------------------------------------------------------------------------
class OAuthMailAuthorizationRequestData(BaseModel):
class MailSyncRequestData(BaseModel):
mailClient: Literal["gmail"] = Field(
description = "the e-mail provider like 'gmail'",
maxCount: int = Field(
description = "the max. no. of e-mails to sync at a given time",
default = 100,
frozen = True
)
startDate: PastDatetime = Field(
description = "the starting date from which the user wants to sync their mail",
default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1),
frozen = True
)
endDate: PastDatetime = Field(
description = "the ending date till which the user wants to sync their mail",
default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1),
frozen = True
)
@@ -115,9 +131,13 @@ class OAuthMailAuthorizationRequestData(BaseModel):
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
@field_validator("mailClient", mode = "before")
def to_lowercase(cls, value):
if isinstance(value, str): value = value.strip().lower()
@field_validator("startDate", "endDate", mode = "before")
def to_datetime(cls, value):
if not isinstance(value, datetime.datetime):
value = date_time.parse_date_time(
input_value = value,
timezone = date_time.TIMEZONE_UTC
)
return value
views/mail/oauth/oauth_failure_v2.html
+1
View File
@@ -102,6 +102,7 @@
<div class="failure-circle">✘</div> <!-- Red circle with failure icon -->
<h1><b>Authorization Failed</b></h1>
<p>Something went wrong in getting authorization from your <b>{{ mail_client }}</b> account.
<br><br><b>Hint:</b> {{ failure_hint }}<br><br>
Please feel free to try the same steps again. You can close this tab at any time.</p>
<!-- Close button -->