khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241209) SMS auth and sending ready.

Browse Source
This commit is contained in:
khushal
2024-12-09 19:39:15 +05:30
parent d10b156c0b
commit 845827a6bc
18 changed files with 367 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/blueprints/ai/llm/invoke.py
+3 -1
View File
@@ -64,6 +64,7 @@ from shared import constants
# Data Models:
from models.data.api.ai.llm import LLMRequestHeaders, LLMInput
from models.data.core.user_info import CoreUserInfoModel
# For asynchronous activities:
import asyncio
@@ -163,7 +164,7 @@ async def invoke_llm(
# Call the LLM and see if its service worked or not:
llm_response = await current_app.llm.invoke(
mongo_conn = current_app.data_mongo,
user_info = kwargs["session_info"],
user_info = CoreUserInfoModel(**kwargs["session_info"]),
llm_input = inbound_data
)
success = False if llm_response.output is None else True
@@ -183,6 +184,7 @@ async def invoke_llm(
"model": llm_response.model,
"output": llm_response.output,
"tokens": llm_response.tokens.model_dump(),
"invocationId": llm_response.invocationId
} if success else None
)
api/blueprints/mail/sync.py
+5 -3
View File
@@ -72,6 +72,7 @@ from shared import constants
# Data Models:
from models.data.api.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
from models.data.api.mail.sync import MailSyncOneResult, MailSyncManyResults
from models.data.core.user_info import CoreUserInfoModel
# To work with datatypes:
from typing import Literal
@@ -126,7 +127,7 @@ def init(blueprint_setup_state):
async def sync_mails(
user_info: dict,
user_info: CoreUserInfoModel,
mongo_conn: AsyncMongo,
llm: ChatOpenAI,
inbound_headers: dict,
@@ -136,6 +137,7 @@ async def sync_mails(
"""
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
and running it in the background.
:param user_info: The information of the user as extracted from the session token.
:param mongo_conn: The instance of the database connector to use to sync the mails.
:param llm: The instance of the LLM to use to summarize the mails.
:param inbound_headers: The headers that came in with the request.
@@ -215,7 +217,7 @@ async def sync_mail(
if mode in ["background", "bg"]:
current_app.add_background_task(
sync_mails,
user_info = kwargs["session_info"],
user_info = CoreUserInfoModel(**kwargs["session_info"]),
mongo_conn = current_app.data_mongo,
llm = current_app.llm,
inbound_headers = inbound_headers,
@@ -229,7 +231,7 @@ async def sync_mail(
# Otherwise we process it right here:
sync_results = await sync_mails(
user_info = kwargs["session_info"],
user_info = CoreUserInfoModel(**kwargs["session_info"]),
mongo_conn = current_app.data_mongo,
llm = current_app.llm,
inbound_headers = inbound_headers,
api/blueprints/sms/auth.py
+53 -29
View File
@@ -59,15 +59,12 @@ from utils_v2.api.async_quart import (
handle_cancelled_request
)
# SMS-related utils:
from utils_v2.sms.nimbus.async_nimbus import AsyncNimbusSMS
from utils_v2.sms.savvy_bulk_sms.async_savvy_bulk_sms import AsyncSavvyBulkSMS
# Common:
from shared import constants
# Data Models:
from models.data.api.sms.auth import SMSAuthRequestHeaders, SMSAuthRequestData
from models.data.core.auth_token import CoreAuthTokenModel
# For asynchronous activities:
import asyncio
@@ -132,7 +129,7 @@ def init(blueprint_setup_state):
data_validator = lambda x: SMSAuthRequestData(**x)
)
@handle_cancelled_request()
async def request_oauth_authorization_url(
async def authorize_sms_client(
inbound_headers: dict | SMSAuthRequestHeaders = None,
inbound_data: dict | SMSAuthRequestData = None,
inbound_files: dict = None,
@@ -167,22 +164,36 @@ async def request_oauth_authorization_url(
# ┣ ┏┓┏┓ ┃┃┓┏┳┓┣┓┓┏┏ ┗┓┃┃┃┗┓ ┃┏┓┏┫┓┏┓
# ┻ ┗┛┛ ┛┗┗┛┗┗┗┛┗┻┛ ┗┛┛ ┗┗┛ ┻┛┗┗┻┗┗┻
if inbound_data.messageClient == "nimbusSmsIndia":
if inbound_data.smsClient == "nimbusSmsIndia":
token_id = await current_app.sms_auth_model.set(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
user_info = kwargs["session_info"],
client_user_id = {
"userId": inbound_data.auth.userId,
"senderId": inbound_data.auth.senderId,
"entityId": inbound_data.auth.entityId
},
auth = inbound_data.auth.model_dump(),
token = None,
service_client = inbound_data.smsClient,
auth_type = "auth",
sync_freq = 300,
auth_token = CoreAuthTokenModel(
serviceType = "sms",
client = inbound_data.smsClient,
authType = "auth",
auth = inbound_data.auth.model_dump(),
user = kwargs.get("session_info"),
clientUserId = {
"userId": inbound_data.auth.userId,
"senderId": inbound_data.auth.senderId,
"entityId": inbound_data.auth.entityId
},
status = "active",
syncFreq = 60
),
# user_info = kwargs["session_info"],
# client_user_id = {
# "userId": inbound_data.auth.userId,
# "senderId": inbound_data.auth.senderId,
# "entityId": inbound_data.auth.entityId
# },
# auth = inbound_data.auth.model_dump(),
# token = None,
# service_client = inbound_data.smsClient,
# auth_type = "auth",
# sync_freq = 300,
session_token = inbound_headers["X-Session-Token"]
)
@@ -191,21 +202,34 @@ async def request_oauth_authorization_url(
# ┻ ┗┛┛ ┗┛┗┻┗┛┗┛┗┫ ┻┛┗┻┗┛┗ ┗┛┛ ┗┗┛ ┛┗┛┗ ┛┗┗┫┗┻
# ┛ ┛
if inbound_data.messageClient == "savvyBulkSmsKenya":
elif inbound_data.smsClient == "savvyBulkSmsKenya":
token_id = await current_app.sms_auth_model.set(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
user_info = kwargs["session_info"],
client_user_id = {
"partnerId": inbound_data.auth.partnerId,
"shortCode": inbound_data.auth.shortCode
},
auth = inbound_data.auth.model_dump(),
token = None,
service_client = inbound_data.smsClient,
auth_type = "auth",
sync_freq = 300,
auth_token=CoreAuthTokenModel(
serviceType = "sms",
client = inbound_data.smsClient,
authType = "auth",
auth = inbound_data.auth.model_dump(),
user = kwargs.get("session_info"),
clientUserId = {
"partnerId": inbound_data.auth.partnerId,
"shortCode": inbound_data.auth.shortCode
},
status = "active",
syncFreq = 60
),
# user_info = kwargs["session_info"],
# client_user_id = {
# "partnerId": inbound_data.auth.partnerId,
# "shortCode": inbound_data.auth.shortCode
# },
# auth = inbound_data.auth.model_dump(),
# token = None,
# service_client = inbound_data.smsClient,
# auth_type = "auth",
# sync_freq = 300,
session_token = inbound_headers["X-Session-Token"]
)
@@ -219,7 +243,7 @@ async def request_oauth_authorization_url(
status_code = StatusCodes.OK if token_id else StatusCodes.FAILED,
http_code = HttpCodes.SUCCESS if token_id else HttpCodes.INTERNAL_SERVER_ERROR,
data = {
"smsClient": inbound_data.messageClient,
"smsClient": inbound_data.smsClient,
"authorized": True
}
)
api/blueprints/sms/send.py
+37 -95
View File
@@ -6,7 +6,7 @@
DATE:
Thursday, 5th Dec., 2024
Monday, 9th Dec., 2024
OBJECTIVE:
@@ -59,17 +59,13 @@ from utils_v2.api.async_quart import (
handle_cancelled_request
)
# SMS-related utils:
from utils_v2.sms.nimbus.async_nimbus import AsyncNimbusSMS
from utils_v2.sms.savvy_bulk_sms.async_savvy_bulk_sms import AsyncSavvyBulkSMS
# Data Models:
from models.data.api.sms.send import SMSSendRequestHeaders, SMSSendRequestData
from models.data.core.auth_token import CoreAuthTokenModel
# Common:
from shared import constants
# Data Models:
from models.data.api.sms.auth import SMSAuthRequestHeaders, SMSAuthRequestData
from models.data.core.auth_token import CoreAuthTokenModel
# For asynchronous activities:
import asyncio
@@ -82,7 +78,7 @@ import asyncio
# Related to Quart:
sms_auth_bp = Blueprint("sms_auth", __name__)
sms_send_bp = Blueprint("sms_send", __name__)
# *****************************************************************************************************************
@@ -102,7 +98,7 @@ sms_auth_bp = Blueprint("sms_auth", __name__)
# *****************************************************************************************************************
@sms_auth_bp.record_once
@sms_send_bp.record_once
def init(blueprint_setup_state):
# This gets called when the blueprint is registered.
@@ -113,7 +109,7 @@ def init(blueprint_setup_state):
# ---------------------------------------------------------------------------------------------------------------------
@sms_auth_bp.route("/auth", methods = ["POST"])
@sms_send_bp.route("/send", methods = ["POST"])
@set_api_version(api_version = "1.0.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
@@ -124,18 +120,18 @@ def init(blueprint_setup_state):
operation = "smsAuthApi",
log_input = True,
log_output = True,
sensitive_keys = ["sessionToken", "X-Session-Token"]
sensitive_keys = ["sessionToken", "X-Session-Token", "tokenId"]
)
@log_chain_to_mongo(attr_name = "logs_mongo")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@validate_input(
header_validator = lambda x: SMSAuthRequestHeaders(**x).model_dump(),
data_validator = lambda x: SMSAuthRequestData(**x)
header_validator = lambda x: SMSSendRequestHeaders(**x).model_dump(),
data_validator = lambda x: SMSSendRequestData(**x)
)
@handle_cancelled_request()
async def request_oauth_authorization_url(
inbound_headers: dict | SMSAuthRequestHeaders = None,
inbound_data: dict | SMSAuthRequestData = None,
async def send_sms(
inbound_headers: dict | SMSSendRequestHeaders = None,
inbound_data: dict | SMSSendRequestData = None,
inbound_files: dict = None,
**kwargs
):
@@ -161,81 +157,30 @@ async def request_oauth_authorization_url(
http_code = HttpCodes.UNAUTHORIZED
)
# Start by assuming failure:
token_id = None
# Fetch the auth-token to use to send this message:
auth_token = await current_app.sms_auth_model.get(
mongo_conn = current_app.data_mongo,
token_id = inbound_data.tokenId
)
# ┏┓ ┳┓• ┓ ┏┓┳┳┓┏┓ ┳ ┓•
# ┣ ┏┓┏┓ ┃┃┓┏┳┓┣┓┓┏┏ ┗┓┃┃┃┗┓ ┃┏┓┏┫┓┏┓
# ┻ ┗┛┛ ┛┗┗┛┗┗┗┛┗┻┛ ┗┛┛ ┗┗┛ ┻┛┗┗┻┗┗┻
# If no auth-token was found, we return with failure:
if not auth_token: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.BAD_REQUEST,
message = f"no such token id"
)
if inbound_data.smsClient == "nimbusSmsIndia":
# ┏┓ ┓ ┏┳┓┓ ┏┓┳┳┓┏┓
# ┗┓┏┓┏┓┏┫ ┃ ┣┓┏┓ ┗┓┃┃┃┗┓
# ┗┛┗ ┛┗┗┻ ┻ ┛┗┗ ┗┛┛ ┗┗┛
token_id = await current_app.sms_auth_model.set(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
auth_token = CoreAuthTokenModel(
serviceType = "sms",
client = inbound_data.smsClient,
authType = "auth",
auth = inbound_data.auth.model_dump(),
user = kwargs.get("session_info"),
clientUserId = {
"userId": inbound_data.auth.userId,
"senderId": inbound_data.auth.senderId,
"entityId": inbound_data.auth.entityId
},
status = "active",
syncFreq = 60
),
# user_info = kwargs["session_info"],
# client_user_id = {
# "userId": inbound_data.auth.userId,
# "senderId": inbound_data.auth.senderId,
# "entityId": inbound_data.auth.entityId
# },
# auth = inbound_data.auth.model_dump(),
# token = None,
# service_client = inbound_data.smsClient,
# auth_type = "auth",
# sync_freq = 300,
session_token = inbound_headers["X-Session-Token"]
)
# ┏┓ ┏┓ ┳┓ ┓┓ ┏┓┳┳┓┏┓ ┓┏┓
# ┣ ┏┓┏┓ ┗┓┏┓┓┏┓┏┓┏ ┣┫┓┏┃┃┏ ┗┓┃┃┃┗┓ ┃┫ ┏┓┏┓┓┏┏┓
# ┻ ┗┛┛ ┗┛┗┻┗┛┗┛┗┫ ┻┛┗┻┗┛┗ ┗┛┛ ┗┗┛ ┛┗┛┗ ┛┗┗┫┗┻
# ┛ ┛
elif inbound_data.smsClient == "savvyBulkSmsKenya":
token_id = await current_app.sms_auth_model.set(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
auth_token=CoreAuthTokenModel(
serviceType = "sms",
client = inbound_data.smsClient,
authType = "auth",
auth = inbound_data.auth.model_dump(),
user = kwargs.get("session_info"),
clientUserId = {
"partnerId": inbound_data.auth.partnerId,
"shortCode": inbound_data.auth.shortCode
},
status = "active",
syncFreq = 60
),
# user_info = kwargs["session_info"],
# client_user_id = {
# "partnerId": inbound_data.auth.partnerId,
# "shortCode": inbound_data.auth.shortCode
# },
# auth = inbound_data.auth.model_dump(),
# token = None,
# service_client = inbound_data.smsClient,
# auth_type = "auth",
# sync_freq = 300,
session_token = inbound_headers["X-Session-Token"]
)
client_response = await current_app.sms_send_model.send_sms(
mongo_conn = current_app.data_mongo,
token_id = inbound_data.tokenId,
auth_token = auth_token,
inbound_data = inbound_data,
session_token = inbound_headers["X-Session-Token"]
)
# ┳┓
# ┣┫┏┓┏┏┓┏┓┏┓┏┏┓
@@ -244,12 +189,9 @@ async def request_oauth_authorization_url(
# Done here:
return ResponseModel(
status_code = StatusCodes.OK if token_id else StatusCodes.FAILED,
http_code = HttpCodes.SUCCESS if token_id else HttpCodes.INTERNAL_SERVER_ERROR,
data = {
"smsClient": inbound_data.smsClient,
"authorized": True
}
status_code = StatusCodes.OK if client_response.success else StatusCodes.FAILED,
http_code = HttpCodes.SUCCESS if client_response.success else HttpCodes.INTERNAL_SERVER_ERROR,
message = None if client_response.success else f"SMS Client: {client_response.brief}"
)
api/helpers/user/session.py
+1 -1
View File
@@ -78,7 +78,7 @@ from models.data.core.user_info import CoreUserInfoModel
# *****************************************************************************************************************
async def get_session(session_token):
async def get_session(session_token) -> CoreUserInfoModel:
"""
Gets the session's info from the session token.
api/main.py
+13 -2
View File
@@ -72,9 +72,10 @@ from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
# Behaviour Models:
from models.behaviour.mail.oauth_v3 import MailOAuthModel
from models.behaviour.mail.sync_v2 import MailSyncModel
from models.behaviour.mail.sync_v3 import MailSyncModel
from models.behaviour.mail.retrieve import MailRetrieveModel
from models.behaviour.sms.auth import SMSAuthModel
from models.behaviour.sms.auth_v2 import SMSAuthModel
from models.behaviour.sms.send import SMSSendModel
from models.behaviour.ai.llm.open_ai import LLMOpenAI
# To make REST API calls:
@@ -90,6 +91,7 @@ from api.blueprints.mail.sync import mail_sync_bp
from api.blueprints.mail.list import mail_list_bp
from api.blueprints.mail.retrieve import mail_retrieve_bp
from api.blueprints.sms.auth import sms_auth_bp
from api.blueprints.sms.send import sms_send_bp
from api.blueprints.chat.auth import chat_auth_bp
from api.blueprints.chat.webhook import chat_webhook_bp
from api.blueprints.tech.chat_alerts import tech_chat_alert_bp
@@ -128,6 +130,7 @@ app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_list_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_retrieve_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(sms_auth_bp, url_prefix = f"/{MODULE_BASE}/sms")
app.register_blueprint(sms_send_bp, url_prefix = f"/{MODULE_BASE}/sms")
app.register_blueprint(chat_auth_bp, url_prefix = f"/{MODULE_BASE}/chat")
app.register_blueprint(chat_webhook_bp, url_prefix = f"/{MODULE_BASE}/chat")
app.register_blueprint(tech_chat_alert_bp, url_prefix = f"/{MODULE_BASE}/tech/alert")
@@ -352,6 +355,14 @@ async def app_startup(**kwargs):
debug_prefix = "SMS-Auth | ",
debug_only_errors = True
)
current_app.sms_send_model = SMSSendModel(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "SMS-Send | ",
debug_only_errors = True
)
current_app.printer("Internal models ready.")
models/behaviour/ai/llm/open_ai.py
+4 -2
View File
@@ -45,6 +45,7 @@ from models.behaviour.base import BaseModel
# Data Models:
from models.data.api.ai.llm import LLMInput, LLMOutput, LLMUsageTokens
from models.data.core.user_info import CoreUserInfoModel
# To work with LLMs:
from langchain_openai import ChatOpenAI
@@ -140,7 +141,7 @@ class LLMOpenAI(BaseModel):
async def invoke(
self,
mongo_conn: AsyncMongo,
user_info: dict,
user_info: CoreUserInfoModel,
llm_input: LLMInput
) -> LLMOutput:
@@ -167,12 +168,13 @@ class LLMOpenAI(BaseModel):
)
# Store this into MongoDB:
mongo_document = {"user": user_info}
mongo_document = {"user": user_info.model_dump()}
for k, v in llm_response.model_dump().items(): mongo_document[k] = v
inserted_id = await mongo_conn.insert_one(
collection = self.AI_USAGE_COLLECTION,
document = mongo_document
)
if inserted_id: llm_response.invocationId = str(inserted_id)
# Done here:
return llm_response
models/behaviour/mail/oauth_v3.py
+1 -1
View File
@@ -125,7 +125,7 @@ class MailOAuthModel(BaseModel):
mongo_json = await mongo_conn.find_one_and_update(
collection = MailOAuthModel.AUTH_COLLECTION,
filter = mongo_conn.dict_to_dot_notation({
"serviceType": "email",
"serviceType": auth_token.serviceType,
"user": {
"entityId": auth_token.user.entityId,
"billingAccountId": auth_token.user.billingAccountId
models/behaviour/mail/sync_v3.py
+90 -69
View File
@@ -6,11 +6,12 @@
DATE:
tuesday, 3rd Dec., 2024
ORIGINAL: Tuesday, 3rd Dec., 2024
UPGRADED: Monday, 9th Dec., 2024
OBJECTIVE:
From here we sync all mails between the mail client's server and TheCAOffice's database.
From here we sync all mails between the mail client's server and our internal database.
REFERENCES:
@@ -31,6 +32,10 @@
# To make sibling directories accessible for imports:
import sys
from models.data.core.auth_token import CoreAuthTokenModel
from models.data.core.message import CoreMessageModel
sys.path.append(".")
sys.path.append("..")
@@ -52,6 +57,7 @@ from models.behaviour.base import BaseModel
# Data models:
from models.data.api.mail.sync import MailSyncOneResult, MailSyncManyResults
from models.data.core.user_info import CoreUserInfoModel
# To work with MongoDB:
from bson import ObjectId
@@ -193,6 +199,10 @@ class MailSyncModel(BaseModel):
attachment_copy["url"] = api_data["url"]
break
# If the upload failed:
await asyncio.sleep(retry_delay)
retry_delay = retry_delay * backoff_multiplier
# Done here:
return attachment_copy
@@ -246,10 +256,12 @@ class MailSyncModel(BaseModel):
async def __sync_one_gmail(
self,
session_token: str,
user_info: dict,
user_info: CoreUserInfoModel,
mongo_conn: AsyncMongo,
token_id: ObjectId,
auth_token: CoreAuthTokenModel,
mail_client: AsyncGMailClient,
tokens: GoogleAuthTokens,
google_tokens: GoogleAuthTokens,
message_id: str,
llm: LLMOpenAI = None,
force_sync: bool = False
@@ -257,11 +269,9 @@ class MailSyncModel(BaseModel):
"""
Sync on mail from GMail.
:param session_token: The session token of the uer who is trying to upload this file.
:param user_info: The information of the user (derived from his session token).
:param mongo_conn: The instance of the connection to the database to use.
:param mail_client: The instance of the mail client to use to perform the action.
:param tokens: The tokens to use to fetch the mails.
:param google_tokens: The tokens to use to fetch the mails.
:param message_id: The id that Google uses to identify this mail. This will be received in the 'list_messages'
method.
:param llm: The instance of the LLM to use to summarize the mail's content.
@@ -278,19 +288,17 @@ class MailSyncModel(BaseModel):
if not force_sync:
mail_record = await mongo_conn.find_one(
collection = self.MAIL_COLLECTION,
filter = mongo_conn.dict_to_dot_notation({
"payload": {
"messageId": message_id
},
"user_info": {
"entityId": user_info["entityId"],
"billingAccountId": user_info["billingAccountId"]
}
}),
filter = {
"tokenId": ObjectId(token_id),
"serviceType": auth_token.serviceType,
"client": auth_token.client,
"clientMessageId": message_id
},
projection = {
"_id": False,
"readTs": "payload.readTs"
}
"readTs": True
},
raise_exception = True
)
if mail_record:
sync_result.success = True
@@ -299,7 +307,7 @@ class MailSyncModel(BaseModel):
# Now that we know that we have to fetch the mail from GMail:
client_response = await mail_client.get_message(
tokens = tokens,
tokens = google_tokens,
message_id = message_id,
return_raw = False
)
@@ -314,18 +322,18 @@ class MailSyncModel(BaseModel):
session_token = session_token,
attachments = client_response.data["attachments"],
attachment_tags = [
"email",
"gmail",
auth_token.serviceType,
auth_token.client,
client_response.data["from"][0]["name"],
client_response.data["from"][0]["email"],
tokens.email,
google_tokens.email,
],
attachment_metadata = {
"project": "tcaoff",
"serviceType": "email",
"client": "gmail",
"serviceType": auth_token.serviceType,
"client": auth_token.client,
"from": client_response.data["from"][0]["email"],
"to": tokens.email
"to": google_tokens.email
},
retry_count = 3
)
@@ -333,7 +341,7 @@ class MailSyncModel(BaseModel):
# Give a quick indicator of whether this mail is an inbox mail or sent mail:
all_recipients = []
for field in ["to", "cc", "bcc"]: all_recipients += [item["email"] for item in client_response.data[field]]
if tokens.email in all_recipients: client_response.data["isInbox"] = True
if google_tokens.email in all_recipients: client_response.data["isInbox"] = True
else: client_response.data["isInbox"] = False
# If an LLM is given,
@@ -342,14 +350,17 @@ class MailSyncModel(BaseModel):
if llm:
# Invoke the LLM:
llm_response = response = await llm.invoke(
llm_response = await llm.invoke(
mongo_conn = mongo_conn,
user_info = user_info,
llm_input = LLMInput(
messages = self.PROMPT_TEMPLATE + [
{
"role": "human",
"content": f"Please summarize this mail: \"\"\"{client_response.data['unformattedText']}\"\"\""
"content": (
"Please summarize this mail: "
f"\"\"\"{client_response.data['unformattedText']}\"\"\""
)
}
]
)
@@ -365,19 +376,30 @@ class MailSyncModel(BaseModel):
# Add the LLM's response to the main data:
client_response.data["aiSnippet"] = llm_json
# Fit the mail message into the model:
sync_result.mailMessage = CoreMessageModel(
ts = client_response.data["ts"],
readTs = date_time.get_current_utc_date_time(as_string = False),
tokenId = token_id,
serviceType = auth_token.serviceType,
client = auth_token.client,
clientMessageId = message_id,
clientThreadId = client_response.data["threadId"],
payload = client_response.data
)
# Done here:
sync_result.success = True
sync_result.mailMessage = client_response.data
return sync_result
async def __sync_many_gmail(
self,
session_token: str,
user_info: dict,
user_info: CoreUserInfoModel,
mongo_conn: AsyncMongo,
token_id: ObjectId,
auth_token: CoreAuthTokenModel,
mail_client: AsyncGMailClient,
tokens: GoogleAuthTokens,
llm: LLMOpenAI = None,
force_sync: bool = False,
start_date: datetime.datetime = None,
@@ -387,13 +409,10 @@ class MailSyncModel(BaseModel):
"""
Sync many mails from GMail in one shot.
:param session_token: The session token of the uer who is trying to upload this file.
:param user_info: The information of the user (derived from his session token).
:param mongo_conn: The instance of the connection to the database to use.
:param token_id: The id of the document in the database that holds the tokens to access the account.
Needed only for refreshing the tokens and saving them.
:param mail_client: The instance of the mail client to use to perform the action.
:param tokens: The tokens to use to fetch the mails.
:param llm: The instance of the LLM to use to summarize the mail's content.
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
database.
@@ -406,20 +425,24 @@ class MailSyncModel(BaseModel):
# Start by assuming failure:
sync_results = MailSyncManyResults()
# Extract the client's tokens from the full token payload given by the database:
google_tokens = GoogleAuthTokens(**auth_token.token)
# Refresh the tokens (if needed):
tokens_refreshed = await tokens.arefresh(
tokens_refreshed = await google_tokens.arefresh(
http_client = current_app.http_client,
client_id = mail_client.client_id,
client_secret = mail_client.client_secret
)
if tokens_refreshed: await current_app.mail_oauth_model.set_token(
db_conn = current_app.sql_writer,
mongo_conn = mongo_conn,
token_id = token_id,
client_user_id = tokens.client_user_id,
token = tokens,
session_token = session_token
)
if tokens_refreshed:
auth_token.token = google_tokens.model_dump()
auth_token.lastRefreshTs = date_time.get_current_utc_date_time(as_string = True)
await current_app.mail_oauth_model.set_token(
db_conn = current_app.sql_writer,
mongo_conn = mongo_conn,
token_id = token_id,
auth_token = auth_token
)
# Let's build the query:
sub_queries = []
@@ -429,7 +452,7 @@ class MailSyncModel(BaseModel):
# Let's enlist all the mails that fall in the date range:
client_response = await mail_client.list_messages(
tokens = tokens,
tokens = google_tokens,
max_count = max_count,
query = query_string
)
@@ -444,8 +467,10 @@ class MailSyncModel(BaseModel):
session_token = session_token,
user_info = user_info,
mongo_conn = mongo_conn,
token_id = token_id,
auth_token = auth_token,
mail_client = mail_client,
tokens = tokens,
google_tokens = google_tokens,
message_id = v["id"],
llm = llm,
force_sync = force_sync
@@ -462,21 +487,19 @@ class MailSyncModel(BaseModel):
else: sync_results.failureCount += 1
if result.mailMessage: mongo_operations.append(ReplaceOne(
filter = {
"serviceType": "email",
"$or": [
{
"client": "gmail",
"payload.messageId": result.mailMessage["messageId"]
}
]
},
replacement = {
"version": "1.0.0",
"tokenId": ObjectId(token_id),
"serviceType": "email",
"client": "gmail",
"payload": result.mailMessage
"tokenId": token_id,
"serviceType": auth_token.serviceType,
"client": auth_token.client,
"clientMessageId": result.mailMessage.clientMessageId
# "serviceType": auth_token.serviceType,
# "$or": [
# {
# "client": auth_token.client,
# "messageId": result.mailMessage.clientMessageId
# }
# ]
},
replacement = result.mailMessage.model_dump(),
upsert = True
))
@@ -490,9 +513,9 @@ class MailSyncModel(BaseModel):
# Apply the labels to the read messages:
try:
client_response = await mail_client.modify_messages(
tokens = tokens,
tokens = google_tokens,
message_ids = [v["id"] for v in messages_list.values()],
add_label_ids = [tokens.labels.get("TCAOFF", {}).get("id")]
add_label_ids = [google_tokens.labels.get("TCAOFF", {}).get("id")]
)
except Exception as exception:
self._printer(exception)
@@ -508,7 +531,7 @@ class MailSyncModel(BaseModel):
async def sync(
self,
session_token: str,
user_info: dict,
user_info: CoreUserInfoModel,
mongo_conn: AsyncMongo,
token_id: ObjectId,
llm: LLMOpenAI = None,
@@ -521,8 +544,6 @@ class MailSyncModel(BaseModel):
"""
Sync many mails at once from many types of clients. Use this as a common entry point after which you internally
route the request to the appropriate clients.
:param session_token: The session token of the uer who is trying to upload this file.
:param user_info: The information of the user (derived from his session token).
:param mongo_conn: The instance of the connection to the database to use.
:param token_id: The id of the document in the database that holds the tokens to access the account.
Needed only for refreshing the tokens and saving them.
@@ -543,13 +564,13 @@ class MailSyncModel(BaseModel):
# ┻ ┗ ┗┗┛┗ ┻ ┗┛┛┗┗ ┛┗┛
# We first load the authorization tokens:
auth_json = await current_app.mail_oauth_model.get_token(
auth_token = await current_app.mail_oauth_model.get_token(
mongo_conn = mongo_conn,
token_id = token_id,
)
# If we failed to load the authorization tokens:
if not auth_json:
if not auth_token:
sync_results.message = f"no such token id '{token_id}'"
return sync_results
@@ -557,14 +578,14 @@ class MailSyncModel(BaseModel):
# ┣ ┏┓┏┓ ┃┓┃┃┃┏┓┓┃
# ┻ ┗┛┛ ┗┛┛ ┗┗┻┗┗
if auth_json["client"] == "gmail":
if auth_token.client == "gmail":
return await self.__sync_many_gmail(
session_token = session_token,
user_info = user_info,
mongo_conn = mongo_conn,
token_id = token_id,
auth_token = auth_token,
mail_client = current_app.gmail_client,
tokens = GoogleAuthTokens(**auth_json["token"]),
llm = llm,
force_sync = force_sync,
start_date = start_date,
@@ -577,7 +598,7 @@ class MailSyncModel(BaseModel):
# ┻┛┗┗┛┗┻┗┗┗┻ ┗┛┗┗┗ ┛┗┗
# If we haven't been able to sync mail due to not entering any 'if' condition:
sync_results.message = f"no such mail client '{auth_json['client']}'"
sync_results.message = f"no such mail client '{auth_token.client}'"
return sync_results
models/behaviour/sms/auth_v2.py
+1 -1
View File
@@ -123,7 +123,7 @@ class SMSAuthModel(BaseModel):
mongo_json = await mongo_conn.find_one_and_update(
collection = self.AUTH_COLLECTION,
filter = mongo_conn.dict_to_dot_notation({
"serviceType": "email",
"serviceType": auth_token.serviceType,
"user": {
"entityId": auth_token.user.entityId,
"billingAccountId": auth_token.user.billingAccountId
models/behaviour/sms/send.py
+89 -102
View File
@@ -6,12 +6,11 @@
DATE:
ORIGINAL: Thursday, 5th Dec., 2024
UPGRADED: Monday, 9th Dec., 2024
Monday, 9th Dec., 2024
OBJECTIVE:
To work with auth details of SMS clients like Nimbus SMS (India) and Savvy Bulk SMS (Kenya).
To send SMS from clients like Nimbus SMS (India) and Savvy Bulk SMS (Kenya).
REFERENCES:
@@ -41,11 +40,23 @@ from utils_v2.date_time import date_time
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo
# SMS-related utils:
from utils_v2.sms.models.behaviour.nimbus.async_nimbus import AsyncNimbusSMS
from utils_v2.sms.models.behaviour.savvy_bulk_sms.async_savvy_bulk_sms import AsyncSavvyBulkSMS
# Base model:
from models.behaviour.base import BaseModel
# Data models:
from models.data.core.auth_token import CoreAuthTokenModel
from models.data.core.message import CoreMessageModel
from models.data.api.sms.send import (
SMSSendRequestHeaders,
SMSSendRequestData,
NimbusSMSIndiaMessage,
SavvyBulkSMSKenyaMessage
)
from utils_v2.sms.models.data.sms_message import SentSMSMessageModel
# To work with MongoDB:
from bson import ObjectId
@@ -94,125 +105,101 @@ import copy
# *****************************************************************************************************************
class SMSAuthModel(BaseModel):
class SMSSendModel(BaseModel):
AUTH_COLLECTION = "_authTokens"
MESSAGES_COLLECTION = "_messages"
async def set(
async def send_sms(
self,
db_conn: AsyncMySQL,
mongo_conn: AsyncMongo,
token_id: ObjectId | str,
auth_token: CoreAuthTokenModel,
inbound_data: SMSSendRequestData,
session_token: str = None
) -> ObjectId | None:
) -> SentSMSMessageModel:
"""
To store auth/tokens for a particular service to the database.
:param db_conn: The database connection (MariaDB) to use to perform the action.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param auth_token: An instance of the core auth-token model that holds data in the database.
:param session_token: The session token of the user who requested this service.
:return: An ObjectId to later store the granted tokens.
"""
# Note down the timestamp at which this event occurred:
request_ts = date_time.get_current_utc_date_time(as_string = False)
# Basic prep:
event_ts = date_time.get_current_utc_date_time(as_string = False)
client_response = None
message_id = None
sms_sent = None
# Get the identifier from the database:
# BE CAREFUL WITH THE KEYS HERE, THEY SHOULD MATCH THE FIELDS OF THE CORE AUTH-TOKEN MODEL:
mongo_json = await mongo_conn.find_one_and_update(
collection = self.AUTH_COLLECTION,
filter = mongo_conn.dict_to_dot_notation({
"serviceType": auth_token.serviceType,
"user": {
"entityId": auth_token.user.entityId,
"billingAccountId": auth_token.user.billingAccountId
},
"clientUserId": auth_token.clientUserId
}),
update = {
"$set": {
"lastRequestTs": auth_token.lastRequestTs,
"status": auth_token.status,
"syncFreq": auth_token.syncFreq
},
"$setOnInsert": {
"version": auth_token.version,
"serviceType": auth_token.serviceType,
"client": auth_token.client,
"authType": auth_token.authType,
"user": auth_token.user.model_dump(),
"clientUserId": auth_token.clientUserId,
"auth": auth_token.auth,
"token": auth_token.token,
"firstRefreshTs": auth_token.firstRefreshTs,
"lastRefreshTs": auth_token.lastRefreshTs,
"firstRequestTs": auth_token.firstRequestTs or request_ts
}
},
projection = {
"_id": True
},
upsert = True,
return_updated = True
)
# ┏┓ ┳┓• ┓ ┏┓┳┳┓┏┓ ┳ ┓•
# ┣ ┏┓┏┓ ┃┃┓┏┳┓┣┓┓┏┏ ┗┓┃┃┃┗┓ ┃┏┓┏┫┓┏┓
# ┻ ┗┛┛ ┛┗┗┛┗┗┗┛┗┻┛ ┗┛┛ ┗┗┛ ┻┛┗┗┻┗┗┻
# Tell MariaDB that an authorization request was initiated:
db_json = {}
if mongo_json is not None:
token_notes = auth_token.clientUserId
db_json = await self.call_procedure(
db_conn = db_conn,
proc_name = "entity_integration_save",
proc_args = (
auth_token.user.entityId, # ..................................... 'p_entity_id'
auth_token.client, # ............................................ 'p_provider'
auth_token.status, # ............................................ 'p_current_status'
"Auth Details Accepted", # ...................................... 'p_last_action'
None, # ......................................................... 'p_display_name'
None, # ......................................................... 'p_display_picture'
str(mongo_json["_id"]), # ....................................... 'p_token_id'
json.to_string(python_data = token_notes, no_space = True), # ... 'p_notes'
auth_token.user.userId # ........................................ 'p_created_by'
),
session_token = session_token
if isinstance(inbound_data.message, NimbusSMSIndiaMessage):
# Prepare the client:
sms_client = AsyncNimbusSMS(
entity_id = auth_token.auth.get("entityId"),
sender_id = auth_token.auth.get("senderId"),
user_id = auth_token.auth.get("userId"),
api_key = auth_token.auth.get("apiKey"),
http_client = self._http_client
)
# Send the SMS:
client_response = await sms_client.send_sms(
recipient_number = inbound_data.message.recipientNo,
message = inbound_data.message.text,
template_id = inbound_data.message.templateId
)
# ┏┓ ┏┓ ┳┓ ┓┓ ┏┓┳┳┓┏┓ ┓┏┓
# ┣ ┏┓┏┓ ┗┓┏┓┓┏┓┏┓┏ ┣┫┓┏┃┃┏ ┗┓┃┃┃┗┓ ┃┫ ┏┓┏┓┓┏┏┓
# ┻ ┗┛┛ ┗┛┗┻┗┛┗┛┗┫ ┻┛┗┻┗┛┗ ┗┛┛ ┗┗┛ ┛┗┛┗ ┛┗┗┫┗┻
# ┛ ┛
elif isinstance(inbound_data.message, SavvyBulkSMSKenyaMessage):
# Prepare the client:
sms_client = AsyncSavvyBulkSMS(
api_key = auth_token.auth.get("apiKey"),
partner_id = auth_token.auth.get("partnerId"),
short_code = auth_token.auth.get("shortCode"),
http_client = self._http_client
)
# Send the SMS:
client_response = await sms_client.send_sms(
recipient_number = inbound_data.message.recipientNo,
message = inbound_data.message.text
)
# ┏┓ ┏┳┓┓ ┳┳┓
# ┗┓┏┓┓┏┏┓ ┃ ┣┓┏┓ ┃┃┃┏┓┏┏┏┓┏┓┏┓
# ┗┛┗┻┗┛┗ ┻ ┛┗┗ ┛ ┗┗ ┛┛┗┻┗┫┗
# ┛
# Save the message:
if client_response:
message_id = await mongo_conn.insert_one(
collection = self.MESSAGES_COLLECTION,
document = CoreMessageModel(
ts = event_ts,
readTs = event_ts,
tokenId = ObjectId(token_id),
serviceType = auth_token.serviceType,
client = auth_token.client,
clientMessageId = client_response.messageId,
clientThreadId = None,
isInward = False,
sentSuccessfully = client_response.success,
payload = client_response.model_dump()
).model_dump()
)
# Done here:
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
async def get(
self,
mongo_conn: AsyncMongo,
token_id: ObjectId | str = None,
**kwargs
) -> dict | None:
"""
To retrieve stored auth/tokens from the database.
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
:param token_id: The identifier granted providing auth details for the first time in 'set_token'.
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
info, the client, the type of authentication used, or even the kind of service.
:return: The retrieved record that has the token, and information about the service and client if found, else
None when there is no matching record.
"""
# Build the filter:
filter_json = {k: v for k, v in kwargs.items()}
if token_id: filter_json["_id"] = ObjectId(token_id)
# If there is no search criteria, we exit with failure:
if not filter_json: return None
# If there is some filtering possible, we fetch the token:
token = await mongo_conn.find_one(
collection = self.AUTH_COLLECTION,
filter = filter_json,
)
# Done here:
return CoreAuthTokenModel(**token) if token else None
return client_response
# *****************************************************************************************************************
models/data/api/ai/llm.py
+7 -1
View File
@@ -37,7 +37,7 @@ sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
from typing import Optional, Literal, Union, List
from typing import Optional, Literal, Union, List, Any
# My utils:
from utils_v2.string import regex
@@ -206,6 +206,12 @@ class LLMOutput(BaseModel):
frozen = True
)
invocationId: Any | None = Field(
description = "the id of the document that notes this invocation; useful for reconciliation",
frozen = False,
default = None
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
models/data/api/mail/sync.py
+4 -1
View File
@@ -43,6 +43,9 @@ from typing import Optional, Literal
from utils_v2.string import regex
from utils_v2.date_time import date_time
# Data models:
from models.data.core.message import CoreMessageModel
# To work with date and time:
import datetime
@@ -168,7 +171,7 @@ class MailSyncOneResult(BaseModel):
default = None
)
mailMessage: dict | None = Field(
mailMessage: CoreMessageModel | None = Field(
description = "the actual data of the mail; can be null in a successful process if the mail is already sync'd",
default = None
)
models/data/api/sms/send.py
+32 -31
View File
@@ -6,11 +6,11 @@
DATE:
Thursday, 5th Dec., 2024.
Monday, 9th Dec., 2024.
OBJECTIVE:
To provide a structure to receive auth details of various SMS providers.
To provide a structure to receive API calls to send SMS messages from various third-party clients.
REFERENCES:
@@ -46,6 +46,9 @@ from utils_v2.date_time import date_time
# To work with date and time:
import datetime
# To work with MongoDB:
from bson.objectid import ObjectId
# *****************************************************************************************************************
# ***** ****
@@ -75,29 +78,22 @@ REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]
# *****************************************************************************************************************
class NimbusSMSIndiaAuth(BaseModel):
class NimbusSMSIndiaMessage(BaseModel):
entityId: str = Field(
description = "the entity id as registered with DLT",
recipientNo: str = Field(
description = "the phone no. of the target recipient",
min_length = 1,
frozen = True
)
senderId: str = Field(
description = "the 6-char code that you see in your SMS inbox",
min_length = 1,
frozen = True,
examples = ["HDFCBK", "NSESMS", "ZRODHA"]
)
userId: str = Field(
description = "the 6-digit id that Nimbus has assigned to you",
text: str = Field(
description = "the actual text that you want to send",
min_length = 1,
frozen = True
)
apiKey: str = Field(
description = "the key generated through Nimbus's portal",
templateId: str = Field(
description = "the id of the template that you are trying to use to send the message",
min_length = 1,
frozen = True
)
@@ -114,22 +110,16 @@ class NimbusSMSIndiaAuth(BaseModel):
# ---------------------------------------------------------------------------------------------------------------------
class SavvyBulkSMSKenyaAuth(BaseModel):
class SavvyBulkSMSKenyaMessage(BaseModel):
apiKey: str = Field(
description = "the key generated through Savvy's portal",
recipientNo: str = Field(
description = "the phone no. of the target recipient",
min_length = 1,
frozen = True
)
partnerId: str = Field(
description = "the key generated through Savvy's portal",
min_length = 1,
frozen = True
)
shortCode: str = Field(
description = "your short code with Savvy",
text: str = Field(
description = "the actual text that you want to send",
min_length = 1,
frozen = True
)
@@ -146,7 +136,7 @@ class SavvyBulkSMSKenyaAuth(BaseModel):
# ---------------------------------------------------------------------------------------------------------------------
class SMSAuthRequestHeaders(BaseModel):
class SMSSendRequestHeaders(BaseModel):
sessionToken: str = Field(
description = "the session token of the user who is requesting the service",
@@ -170,10 +160,10 @@ class SMSAuthRequestHeaders(BaseModel):
# ---------------------------------------------------------------------------------------------------------------------
class SMSAuthRequestData(BaseModel):
class SMSSendRequestData(BaseModel):
smsClient: Literal["nimbusSmsIndia", "savvyBulkSmsKenya"] = Field(alias = "client")
auth: Union[NimbusSMSIndiaAuth, SavvyBulkSMSKenyaAuth]
tokenId: ObjectId = Field(description = "the auth token to use to send this message")
message: Union[NimbusSMSIndiaMessage, SavvyBulkSMSKenyaMessage]
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
@@ -182,6 +172,17 @@ class SMSAuthRequestData(BaseModel):
class Config:
extra = "forbid"
arbitrary_types_allowed = True
# ┓┏ ┓• ┓ •
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
@field_validator("tokenId", mode = "before")
def parse_oid(cls, value):
try: value = ObjectId(value)
except: pass
return value
# *****************************************************************************************************************
models/data/core/auth_token.py
+5 -5
View File
@@ -86,7 +86,7 @@ class CoreAuthTokenModel(BaseModel):
description = "a hint about the version no. of this message",
min_length = 1,
frozen = True,
default = "1.0.0"
default = "2.0.0"
)
serviceType: Literal["email", "sms", "chat"] = Field(
@@ -109,25 +109,25 @@ class CoreAuthTokenModel(BaseModel):
frozen = True
)
firstRequestTs: AwareDatetime = Field(
firstRequestTs: AwareDatetime | None = Field(
description = "the time (utc) at which authorization was first requested",
frozen = True,
default = None
)
lastRequestTs: AwareDatetime = Field(
lastRequestTs: AwareDatetime | None = Field(
description = "the time (utc) at which authorization was last requested",
frozen = False,
default_factory = lambda: date_time.get_current_utc_date_time(as_string = False)
)
firstRefreshTs: AwareDatetime = Field(
firstRefreshTs: AwareDatetime | None = Field(
description = "the time (utc) at which the tokens were first refreshed",
frozen = False,
default = None
)
lastRefreshTs: AwareDatetime = Field(
lastRefreshTs: AwareDatetime | None = Field(
description = "the time (utc) at which the tokens were last refreshed",
frozen = False,
default = None
models/data/core/message.py
+20 -2
View File
@@ -83,7 +83,7 @@ class CoreMessageModel(BaseModel):
description = "a hint about the version no. of this message",
min_length = 1,
frozen = True,
default = "1.0.0"
default = "2.0.0"
)
ts: AwareDatetime = Field(
@@ -116,7 +116,7 @@ class CoreMessageModel(BaseModel):
frozen = True
)
clientMessageId: str | int = Field(
clientMessageId: str | int | None = Field(
description = "how the client identifies this message",
frozen = True
)
@@ -127,6 +127,24 @@ class CoreMessageModel(BaseModel):
default = None
)
isInward: bool = Field(
description = "to understand whether this message was an inward message or outward message",
frozen = True,
default = True
)
isBroadcast: bool = Field(
description = "to understand if this message was broadcasted or sent one-to-one",
frozen = True,
default = False
)
sentSuccessfully: bool | None = Field(
description = "when a message is an outgoing message, this indicates if the message was send successfully",
frozen = False,
default = False
)
payload: dict = Field(
description = "the actual contents of the message; will differ for each client",
frozen = True
models/data/core/payment.py
+1 -1
View File
@@ -130,7 +130,7 @@ class CorePaymentModel(BaseModel):
description = "a hint about the version no. of this message",
min_length = 1,
frozen = True,
default = "1.0.0"
default = "2.0.0"
)
paymentStatus: Literal[
models/data/core/user_info.py
+1 -1
View File
@@ -83,7 +83,7 @@ class CoreUserInfoModel(BaseModel):
description = "a hint about the version no. of this message",
min_length = 1,
frozen = True,
default = "1.0.0"
default = "2.0.0"
)
fullName: str | None = Field(