khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241202) Testing out the modified auth process.

Browse Source
This commit is contained in:
khushal
2024-12-02 18:47:38 +05:30
parent aed746190d
commit bcbad8ecdb
10 changed files with 419 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/behaviour/mail/oauth.py
+66 -46
View File
@@ -122,59 +122,76 @@ class MailOAuthModel(BaseModel):
request_ts = date_time.get_current_utc_date_time(as_string = False)
# Get the identifier from the database:
mongo_json = await mongo_conn.find_one_and_update(
inserted_id = await mongo_conn.insert_one(
collection = MailOAuthModel.AUTH_COLLECTION,
filter = {
document = {
"version": "-1.0.1",
"serviceType": "email",
"client": service_client,
"authType": auth_type,
"user": user_info,
},
update = {
"$set": {
"lastRequestTs": request_ts
},
"$setOnInsert": {
"version": "1.0.0",
"serviceType": "email",
"client": service_client,
"authType": auth_type,
"user": user_info,
"token": None,
"firstRefreshTs": None,
"lastRefreshTs": None,
"firstRequestTs": request_ts,
}
},
projection = {
"_id": True
},
upsert = True,
return_updated = True
"token": None,
"firstRefreshTs": None,
"lastRefreshTs": None,
"lastRequestTs": request_ts
}
)
return inserted_id
# Tell MariaDB that an authorization request was initiated:
db_json = {}
if mongo_json is not None:
db_json = await self.call_procedure(
db_conn = db_conn,
proc_name = "entity_integration_save",
proc_args = (
user_info["entityId"], # ............................................ 'p_entity_id'
service_client, # ................................................... 'p_provider'
"Auth Requested", # ................................................. 'p_current_status'
"Auth URL Generated", # ............................................. 'p_last_action'
None, # ............................................................. 'p_display_name'
None, # ............................................................. 'p_display_picture'
str(mongo_json["_id"]), # ........................................... 'p_token_id'
json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
user_info["userId"] # ............................................... 'p_created_by'
),
session_token = session_token
)
# Done here:
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
# # Get the identifier from the database:
# mongo_json = await mongo_conn.find_one_and_update(
# collection = MailOAuthModel.AUTH_COLLECTION,
# filter = {
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# },
# update = {
# "$set": {
# "lastRequestTs": request_ts
# },
# "$setOnInsert": {
# "version": "1.0.0",
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# "token": None,
# "firstRefreshTs": None,
# "lastRefreshTs": None,
# "firstRequestTs": request_ts,
# }
# },
# projection = {
# "_id": True
# },
# upsert = True,
# return_updated = True
# )
#
# # Tell MariaDB that an authorization request was initiated:
# db_json = {}
# if mongo_json is not None:
# db_json = await self.call_procedure(
# db_conn = db_conn,
# proc_name = "entity_integration_save",
# proc_args = (
# user_info["entityId"], # ............................................ 'p_entity_id'
# service_client, # ................................................... 'p_provider'
# "Auth Requested", # ................................................. 'p_current_status'
# "Auth URL Generated", # ............................................. 'p_last_action'
# None, # ............................................................. 'p_display_name'
# None, # ............................................................. 'p_display_picture'
# str(mongo_json["_id"]), # ........................................... 'p_token_id'
# json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
# user_info["userId"] # ............................................... 'p_created_by'
# ),
# session_token = session_token
# )
#
# # Done here:
# return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
async def set_token(
self,
@@ -210,6 +227,9 @@ class MailOAuthModel(BaseModel):
update = {
"$set": {
"token": token,
"lastRefreshTs": request_ts,
},
"$setOnInsert": {
"firstRefreshTs": request_ts,
}
},
models/behaviour/mail/oauth_v2.py
+68 -75
View File
@@ -6,7 +6,7 @@
DATE:
Wednesday, 27th Nov., 2024
Monday, 2nd Dec., 2024
OBJECTIVE:
@@ -95,11 +95,12 @@ class MailOAuthModel(BaseModel):
AUTH_COLLECTION = "_authTokens"
async def get_user_identifier(
async def get_account_identifier(
self,
db_conn: AsyncMySQL,
mongo_conn: AsyncMongo,
user_info: dict,
email_id: str,
service_client: Literal["gmail"],
auth_type: Literal["oauth"],
session_token: str = None
@@ -111,6 +112,7 @@ class MailOAuthModel(BaseModel):
:param db_conn: The database connection (MariaDB) to use to perform the action.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param user_info: The dictionary that has the user's session information.
:param email_id: The e-mail id that the user wants to connect to your service.
:param service_client: The name of the company or brand that is providing this service that is being integrated.
:param auth_type: To identify the type of authentication being done here. This could indicate simple password
authentication, more advance OAuth2.0 authentication, etc.
@@ -122,82 +124,68 @@ class MailOAuthModel(BaseModel):
request_ts = date_time.get_current_utc_date_time(as_string = False)
# Get the identifier from the database:
inserted_id = await mongo_conn.insert_one(
mongo_json = await mongo_conn.find_one_and_update(
collection = MailOAuthModel.AUTH_COLLECTION,
document = {
"version": "-1.0.1",
filter = {
"serviceType": "email",
"client": service_client,
"authType": auth_type,
"user": user_info,
"token": None,
"firstRefreshTs": None,
"lastRefreshTs": None,
"lastRequestTs": request_ts
}
"user": {
"entityId": user_info["entityId"],
"billingAccountId": user_info["billingAccountId"]
},
"clientUserId": email_id
},
update = {
"$set": {
"lastRequestTs": request_ts
},
"$setOnInsert": {
"version": "1.1.0",
"serviceType": "email",
"client": service_client,
"authType": auth_type,
"user": user_info,
"token": None,
"firstRefreshTs": None,
"lastRefreshTs": None,
"firstRequestTs": request_ts,
}
},
projection = {
"_id": True
},
upsert = True,
return_updated = True
)
return inserted_id
# # Get the identifier from the database:
# mongo_json = await mongo_conn.find_one_and_update(
# collection = MailOAuthModel.AUTH_COLLECTION,
# filter = {
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# },
# update = {
# "$set": {
# "lastRequestTs": request_ts
# },
# "$setOnInsert": {
# "version": "1.0.0",
# "serviceType": "email",
# "client": service_client,
# "authType": auth_type,
# "user": user_info,
# "token": None,
# "firstRefreshTs": None,
# "lastRefreshTs": None,
# "firstRequestTs": request_ts,
# }
# },
# projection = {
# "_id": True
# },
# upsert = True,
# return_updated = True
# )
#
# # Tell MariaDB that an authorization request was initiated:
# db_json = {}
# if mongo_json is not None:
# db_json = await self.call_procedure(
# db_conn = db_conn,
# proc_name = "entity_integration_save",
# proc_args = (
# user_info["entityId"], # ............................................ 'p_entity_id'
# service_client, # ................................................... 'p_provider'
# "Auth Requested", # ................................................. 'p_current_status'
# "Auth URL Generated", # ............................................. 'p_last_action'
# None, # ............................................................. 'p_display_name'
# None, # ............................................................. 'p_display_picture'
# str(mongo_json["_id"]), # ........................................... 'p_token_id'
# json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
# user_info["userId"] # ............................................... 'p_created_by'
# ),
# session_token = session_token
# )
#
# # Done here:
# return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
# Tell MariaDB that an authorization request was initiated:
db_json = {}
if mongo_json is not None:
db_json = await self.call_procedure(
db_conn = db_conn,
proc_name = "entity_integration_save",
proc_args = (
user_info["entityId"], # ............................................ 'p_entity_id'
service_client, # ................................................... 'p_provider'
"Auth Requested", # ................................................. 'p_current_status'
"Auth URL Generated", # ............................................. 'p_last_action'
None, # ............................................................. 'p_display_name'
None, # ............................................................. 'p_display_picture'
str(mongo_json["_id"]), # ........................................... 'p_token_id'
json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
user_info["userId"] # ............................................... 'p_created_by'
),
session_token = session_token
)
# Done here:
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
async def set_token(
self,
db_conn: AsyncMySQL,
mongo_conn: AsyncMongo,
user_identifier: ObjectId | str,
account_identifier: ObjectId | str,
email_id: str,
token: dict,
session_token: str = None
) -> bool:
@@ -208,7 +196,9 @@ class MailOAuthModel(BaseModel):
ALSO.
:param db_conn: The database connection (MariaDB) to use to perform the action.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param user_identifier: The identifier granted by the 'get_user_identifier' method.
:param account_identifier: The identifier granted by the 'get_account_identifier' method.
:param email_id: The e-mail id that the user tried to connect to your service. This should match the e-mail id
the user claimed he wants to connect when he used 'get_account_identifier'.
:param token: The token granted by the third-party service.
:param session_token: The session token of the user who requested this service.
:return: True if saved, False if failed.
@@ -223,7 +213,10 @@ class MailOAuthModel(BaseModel):
# Save the token to MongoDB:
mongo_json = await mongo_conn.find_one_and_update(
collection = MailOAuthModel.AUTH_COLLECTION,
filter = {"_id": ObjectId(user_identifier)},
filter = {
"_id": ObjectId(account_identifier),
"clientUserId": email_id
},
update = {
"$set": {
"token": token,
@@ -250,7 +243,7 @@ class MailOAuthModel(BaseModel):
"Set Token", # ................................................................ 'p_last_action'
None, # ....................................................................... 'p_display_name'
None, # ....................................................................... 'p_display_picture'
user_identifier, # ............................................................ 'p_token_id'
account_identifier, # ......................................................... 'p_token_id'
json.to_string(python_data = {"email": token["email"]}, no_space = True), # ... 'p_notes'
mongo_json["user"]["userId"] # ................................................ 'p_created_by'
),
@@ -264,14 +257,14 @@ class MailOAuthModel(BaseModel):
async def get_token(
self,
mongo_conn: AsyncMongo,
user_identifier: ObjectId | str = None,
account_identifier: ObjectId | str = None,
**kwargs
) -> dict | None:
"""
To retrieve stored tokens from the database.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param user_identifier: The identifier granted by the 'get_user_identifier' method.
:param account_identifier: The identifier granted by the 'account_identifier' method.
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
info, the client, the type of authentication used, or even the kind of service.
:return: The retrieved record that has the token, and information about the service and client if found, else
@@ -280,7 +273,7 @@ class MailOAuthModel(BaseModel):
# Build the filter:
filter_json = {k: v for k, v in kwargs.items()}
if user_identifier: filter_json["_id"] = ObjectId(user_identifier)
if account_identifier: filter_json["_id"] = ObjectId(account_identifier)
# If there is no search criteria, we exit with failure:
if not filter_json: return None
models/behaviour/mail/sync.py
+144 -25
View File
@@ -34,6 +34,7 @@
import sys
from langchain.chains.summarize.stuff_prompt import prompt_template
from sqlalchemy.orm.collections import collection
sys.path.append(".")
sys.path.append("..")
@@ -53,6 +54,7 @@ from models.behaviour.base import BaseModel
# To work with MongoDB:
from bson import ObjectId
from pymongo import InsertOne, UpdateOne
# To work with LLMs:
from langchain_openai import ChatOpenAI
@@ -64,6 +66,12 @@ from typing import Literal
# To make deep-copies:
import copy
# To work with date and time:
import datetime
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
@@ -112,7 +120,7 @@ class MailSyncModel(BaseModel):
prompt_template = ChatPromptTemplate.from_messages([
(
"system",
"You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPS) FROM THE SUMMARY."
"You're a mail summary expert that summarizes mails in 150 chars or less. HIDE SENSITIVE INFO (LIKE OTPs) FROM THE SUMMARY."
),
(
"user",
@@ -120,17 +128,24 @@ class MailSyncModel(BaseModel):
)
])
async def sync_one(
async def __sync_one(
self,
mongo_conn: AsyncMongo,
user_identifier: str | ObjectId,
user_info: dict,
mail_client: AsyncGMailClient,
tokens: GoogleAuthTokens,
message_id: str,
llm: ChatOpenAI = None,
session_token: str = None,
force_sync: bool = False
) -> ObjectId:
) -> UpdateOne | None:
# ┏┓ ┓┏ • ┓ ┓
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
# ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
# ┛
mail_payload = None
mail_client_name = None
# ┏┓┓ ┓ ┏┓ • • ┳┓ ┓
# ┃ ┣┓┏┓┏┃┏ ┣ ┓┏┓┏╋┓┏┓┏┓ ┣┫┏┓┏┏┓┏┓┏┫┏
@@ -150,15 +165,7 @@ class MailSyncModel(BaseModel):
)
# If there already exists such a record, and we haven't been forced to re-sync it:
if existing_record and not force_sync: return existing_record["_id"]
# ┏┓ ┓┏ • ┓ ┓
# ┃┃┏┓┏┓┏┓┏┓┏┓┏┓ ┃┃┏┓┏┓┓┏┓┣┓┃┏┓┏
# ┣┛┛ ┗ ┣┛┗┻┛ ┗ ┗┛┗┻┛ ┗┗┻┗┛┗┗ ┛
# ┛
mail_payload = None
mail_id = existing_record["_id"] if existing_record else None
if existing_record and not force_sync: return mail_payload
# ┏┓┳┳┓ •┓
# ┃┓┃┃┃┏┓┓┃
@@ -166,10 +173,10 @@ class MailSyncModel(BaseModel):
if isinstance(mail_client, AsyncGMailClient):
# Refresh the tokens:
tokens_refreshed
# Note down the name of the mail client:
mail_client_name = "gmail"
# Fetch the mail formatted message:
# Fetch the formatted mail message:
client_response = await mail_client.get_message(
tokens = tokens,
message_id = message_id,
@@ -180,20 +187,132 @@ class MailSyncModel(BaseModel):
if client_response.success:
# Summarize the content:
prompt = self.prompt_template.invoke({"mail": client_response.data.pop["unformattedText"]})
llm_response = await llm.ainvoke(prompt)
client_response.data["aiSnippet"] = llm_response.content
if llm:
prompt = self.prompt_template.invoke({"mail": client_response.data["unformattedText"]})
llm_response = await llm.ainvoke(prompt)
client_response.data["aiSnippet"] = llm_response.content
# Note down the response:
mail_payload = client_response.data
# ┏┓ ┏┳┓┓ ┳┳┓ •┓
# ┗┓┓┏┏┓┏ ┃ ┣┓┏┓ ┃┃┃┏┓┓┃
# ┗┛┗┫┛┗┗ ┻ ┛┗┗ ┛ ┗┗┻┗┗
# ┛
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
# ┛
if mail_payload:
pass
return UpdateOne(
filter = {
"messageType": "email",
"$or": [
{"payload.messageId": message_id}
]
},
update = {
"$set": {
"readTs": date_time.get_current_utc_date_time(),
"user": user_info,
"messageType": "email",
"connector": mail_client_name,
"payload": mail_payload
}
},
upsert = True
)
# Done here:
return mail_payload
async def sync(
self,
mongo_conn: AsyncMongo,
user_info: dict,
mail_client: AsyncGMailClient,
tokens: GoogleAuthTokens,
llm: ChatOpenAI = None,
force_sync: bool = False,
start_date: datetime.datetime = None,
end_date: datetime.datetime = None,
max_count: int = 100
) -> int:
# Start by assuming failure:
mails_count = 0
# ┏┓┳┳┓ •┓
# ┃┓┃┃┃┏┓┓┃
# ┗┛┛ ┗┗┻┗┗
if isinstance(mail_client, AsyncGMailClient):
# Enlist all the labels, we need to find the label that indicates that we've read the mail:
client_response = await mail_client.list_labels(tokens = tokens)
if not client_response.success: return mails_count
labels = client_response.data
custom_label = "Sync'd with TheCAOffice"
custom_label_id = labels.get(custom_label)
if custom_label_id is None:
client_response = await mail_client.create_label(
tokens = tokens,
label_name = custom_label,
label_visibility = "labelHide"
)
if not client_response.success: return mails_count
custom_label_id = client_response.data["id"]
# Build the query:
sub_queries = [f"-label:\"{custom_label}\""]
if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
if end_date: sub_queries.append(end_date.strftime("before:%Y/%m/%d"))
print("Q:", " ".join(sub_queries))
# Get a list of all the mails:
client_response = await mail_client.list_messages(
tokens = tokens,
max_count = max_count,
# query = " ".join(sub_queries)
)
if not client_response.success: return mails_count
messages_list = client_response.data["messages"]
print(messages_list)
# Create MongoDB operations for all the mails:
tasks = [
self.__sync_one(
mongo_conn = mongo_conn,
user_info = user_info,
mail_client = mail_client,
tokens = tokens,
message_id = v["id"],
llm = llm,
force_sync = force_sync
)
for k, v in messages_list.items()
]
mongo_operations = await asyncio.gather(*tasks)
mongo_operations = [mo for mo in mongo_operations if mo is not None]
# Write the mails to MongoDB:
mails_count = await mongo_conn.bulk_write(
collection = self.MAIL_COLLECTION,
requests = mongo_operations
)
print("MAILS COUNT:", mails_count)
# If all the mails were sync'd properly:
client_response = await mail_client.modify_messages(
tokens = tokens,
message_ids = [v["id"] for k, v in messages_list.items()],
add_label_ids = [custom_label_id]
)
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
# ┛┗┗ ┛┣┛┗┛┛┗┛┗
# ┛
# Done here:
return mails_count
# *****************************************************************************************************************