khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241212) Reorganizing code to perform core actions in one place.

Browse Source
This commit is contained in:
khushal
2024-12-12 18:34:51 +05:30
parent 70c817b3b8
commit fb95e0b38c
50 changed files with 1259 additions and 3166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/blueprints/ai/llm/invoke.py
+3 -2
View File
@@ -63,8 +63,9 @@ from utils_v2.api.async_quart import (
from shared import constants
# Data Models:
from models.data.api.ai.llm import LLMRequestHeaders, LLMInput
from models.data.core.user import CoreUserInfoModel
from models.api.ai.llm import LLMRequestHeaders
from models.core.ai.llm import LLMInput
from models.core.user import CoreUserInfoModel
# For asynchronous activities:
import asyncio
api/blueprints/mail/oauth_callback.py
+6 -3
View File
@@ -63,6 +63,9 @@ from utils_v2.api.async_quart import (
# GMail-related utils:
from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
# Data Models:
from models.core.auth_token import CoreAuthTokenModel
# Common:
from shared import constants
@@ -113,7 +116,7 @@ def init(blueprint_setup_state):
api_version = "1.0.0",
project = constants.PROJECT_NAME,
log_type = constants.MODULE_NAME,
operation = "gmailCllbk",
operation = "gmailClbk",
log_input = 2,
log_output = 1,
sensitive_keys = ["sessionToken", "X-Session-Token"]
@@ -156,7 +159,7 @@ async def handle_gmail_callback() -> render_template:
# Here's where we do the checking of the e-mails,
# if they don't match, we reject the authorization:
auth_token = await current_app.mail_oauth_model.get_token(
auth_token = await current_app.mail_controller.get_token(
mongo_conn = current_app.data_mongo,
token_id = g.inbound_data["state"]
)
@@ -211,7 +214,7 @@ async def handle_gmail_callback() -> render_template:
auth_token.clientUserId = google_tokens.client_user_id
auth_token.token = google_tokens.model_dump()
auth_token.status = "active"
tokens_saved = await current_app.mail_oauth_model.set_token(
tokens_saved = await current_app.mail_controller.set_token(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
session_token = g.inbound_headers.get("X-Session-Token"),
api/blueprints/mail/oauth_request.py
+3 -3
View File
@@ -67,11 +67,11 @@ from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
from shared import constants
# Data Models:
from models.data.api.mail.oauth import (
from models.api.mail.oauth import (
OAuthMailAuthorizationRequestHeaders,
OAuthMailAuthorizationRequestData
)
from models.data.core.auth_token import CoreAuthTokenModel
from models.core.auth_token import CoreAuthTokenModel
# For asynchronous activities:
import asyncio
@@ -175,7 +175,7 @@ async def request_oauth_authorization_url(
# ┗┛┗ ┛┗┗ ┛ ┗┻┗┗ ┻ ┗┛┛┗┗ ┛┗ ┻┗┻
# Make a user identifier from the session info:
token_id = await current_app.mail_oauth_model.get_token_id(
token_id = await current_app.mail_controller.get_token_id(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
auth_token = CoreAuthTokenModel(
api/blueprints/mail/sync.py → api/blueprints/mail/sync_old.py
View File
api/blueprints/mail/sync_v2.py
+256
View File
@@ -0,0 +1,256 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Monday, 2nd Dec., 2024
OBJECTIVE:
To receive requests for synchronising mails from various mail clients to the database. Sync'ing means we pull
the mail from the mail client (like GMail) and store it to our database. The mail is then ready for showing on
the UI at any time.
REFERENCES:
N/A
DOWNLOADS:
N/A
NOTES:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For using Quart:
from quart import Blueprint, current_app, g, request
# My utils:
from utils_v2.string import json
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.api.codes import StatusCodes, HttpCodes
from utils_v2.api.response import ResponseModel
from utils_v2.api.async_quart import (
set_api_version,
read_input,
get_session_info,
log_request_to_mongo,
log_chain_to_mongo,
should_not_be_under_maintenance,
only_whitelisted_ips,
limit_rate,
validate_input,
handle_cancelled_request
)
# GMail-related utils:
from utils_v2.goog.gmail.gmail_client import SCOPES_GMAIL_MAIL_MANAGEMENT
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
# Common:
from shared import constants
# Data Models:
from models.core.user import CoreUserInfoModel
from models.api.mail.sync import MailSyncRequestHeaders, MailSyncRequestData
from models.api.mail.sync import MailSyncOneResult, MailSyncManyResults
# To work with datatypes:
from typing import Literal
# For asynchronous activities:
import asyncio
# To work with LLMs:
from langchain_openai import ChatOpenAI
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Related to Quart:
mail_sync_bp = Blueprint("mail_sync", __name__)
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
@mail_sync_bp.record_once
def init(blueprint_setup_state):
# This gets called when the blueprint is registered.
# Consider this to be a one-time setup for the whole blueprint:
pass
# ---------------------------------------------------------------------------------------------------------------------
async def sync_mails(
user_info: CoreUserInfoModel,
inbound_headers: dict,
inbound_data: MailSyncRequestData
) -> MailSyncManyResults:
"""
A very simple function, but kept separate so that we get the option to switch between running it in the foreground
and running it in the background.
:param user_info: The information of the user as extracted from the session token.
:param inbound_headers: The headers that came in with the request.
:param inbound_data: The data that came in with the request.
:return: The results of the mail-sync'ing attempt.
"""
# Try to sync the mails:
return await current_app.mail_api_model.sync(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
user_info = user_info,
token_id = inbound_data.tokenId,
llm = current_app.llm,
force_sync = inbound_data.forceSync,
start_date = inbound_data.startDate,
end_date = inbound_data.endDate,
max_count = inbound_data.maxCount,
session_token = inbound_headers["X-Session-Token"],
)
# ---------------------------------------------------------------------------------------------------------------------
@mail_sync_bp.route("/sync", methods = ["POST"])
@mail_sync_bp.route("/sync/<mode>", methods = ["POST"])
@set_api_version(api_version = "1.0.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@get_session_info(key = "X-Session-Token", session_coro = "get_session")
@log_request_to_mongo(
attr_name = "logs_mongo",
project = constants.PROJECT_NAME,
log_type = constants.MODULE_NAME,
operation = "mailSyncApi",
log_input = True,
log_output = True,
sensitive_keys = ["sessionToken", "X-Session-Token"]
)
@log_chain_to_mongo(attr_name = "logs_mongo")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@validate_input(
header_validator = lambda x: MailSyncRequestHeaders(**x).model_dump(),
data_validator = lambda x: MailSyncRequestData(**x)
)
@handle_cancelled_request()
async def sync_mail(
mode: Literal["background", "bg"] = None,
inbound_headers: dict | MailSyncRequestHeaders = None,
inbound_data: dict | MailSyncRequestData = None,
inbound_files: dict = None,
**kwargs
):
"""
Use this when the user wants to pull old mails from some mail client (like GMail) and save it to the database for
ready access on the UI.
:param mode: Set it to one of the specified options to make the sync'ing process go to the background.
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
:param kwargs: Any number of extra inputs supplied by the decorators.
:return: A standard response structure.
"""
# If the session token is invalid/expired:
if kwargs.get("session_info") is None:
return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.UNAUTHORIZED
)
# Make the variables available in the scope of the current request:
g.inbound_headers = inbound_headers
g.inbound_data = inbound_data
# If we've been asked to sync the mails in the background:
if mode in ["background", "bg"]:
current_app.add_background_task(
sync_mails,
user_info = CoreUserInfoModel(**kwargs["session_info"]),
inbound_headers = inbound_headers,
inbound_data = inbound_data
)
return ResponseModel(
status_code = StatusCodes.OK,
http_code = HttpCodes.ACCEPTED,
message = "your mails are being sync'd in the background"
)
# Otherwise we process it right here:
sync_results = await sync_mails(
user_info = CoreUserInfoModel(**kwargs["session_info"]),
inbound_headers = inbound_headers,
inbound_data = inbound_data
)
# Response:
return ResponseModel(
status_code = StatusCodes.FAILED if sync_results.failureCount > 0 else StatusCodes.OK,
http_code = HttpCodes.INTERNAL_SERVER_ERROR if sync_results.failureCount > 0 else HttpCodes.SUCCESS,
message = sync_results.message,
data = {
"totalCount": sync_results.totalCount,
"successCount": sync_results.successCount,
"failureCount": sync_results.failureCount
}
)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
api/blueprints/tech/chat_alerts.py
+1 -1
View File
@@ -63,7 +63,7 @@ from utils_v2.api.async_quart import (
)
# Data models:
from models.data.api.tech.alerts import (
from models.api.tech.alerts import (
ChatAlertRequestHeaders,
ChatAlertRequestData
)
api/helpers/user/session.py
+1 -1
View File
@@ -38,7 +38,7 @@ sys.path.append("..")
from quart import current_app
# The data model:
from models.data.core.user import CoreUserInfoModel
from models.core.user import CoreUserInfoModel
# *****************************************************************************************************************
api/main.py
+40 -54
View File
@@ -70,13 +70,20 @@ from utils_v2.api.async_quart import (
# GMail-related utils:
from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
# Behaviour Models:
from models.behaviour.mail.oauth_v3 import MailOAuthModel
from models.behaviour.mail.sync_v3 import MailSyncModel
from models.behaviour.mail.retrieve import MailRetrieveModel
from models.behaviour.sms.auth_v2 import SMSAuthModel
from models.behaviour.sms.send import SMSSendModel
from models.behaviour.ai.llm.open_ai import LLMOpenAI
# Core Controller Models:
from controllers.core.message import MessageController
from controllers.core.auth_token import AuthTokenController
from controllers.core.ai.llm import LLMController
# API Controller Models:
from controllers.api.mail import MailController
# # Old Behaviour Models:
# from controllers.mail.oauth_v3 import MailOAuthModel
# from controllers.mail.sync_v3 import MailSyncModel
# from controllers.mail.retrieve import MailRetrieveModel
# from controllers.sms.auth_v2 import SMSAuthModel
# from controllers.sms.send import SMSSendModel
# To make REST API calls:
import httpx
@@ -87,13 +94,13 @@ from icecream import IceCreamDebugger
# All the blueprints:
from api.blueprints.mail.oauth_request import mail_oauth_bp
from api.blueprints.mail.oauth_callback import mail_callback_bp
from api.blueprints.mail.sync import mail_sync_bp
from api.blueprints.mail.list import mail_list_bp
from api.blueprints.mail.retrieve import mail_retrieve_bp
from api.blueprints.sms.auth import sms_auth_bp
from api.blueprints.sms.send import sms_send_bp
from api.blueprints.chat.auth import chat_auth_bp
from api.blueprints.chat.webhook import chat_webhook_bp
# from api.blueprints.mail.sync_v2 import mail_sync_bp
# from api.blueprints.mail.list import mail_list_bp
# from api.blueprints.mail.retrieve import mail_retrieve_bp
# from api.blueprints.sms.auth import sms_auth_bp
# from api.blueprints.sms.send import sms_send_bp
# from api.blueprints.chat.auth import chat_auth_bp
# from api.blueprints.chat.webhook import chat_webhook_bp
from api.blueprints.tech.chat_alerts import tech_chat_alert_bp
from api.blueprints.test.callback import test_callback_bp
from api.blueprints.ai.llm.invoke import llm_invoke_bp
@@ -126,13 +133,13 @@ app = Quart(__name__, template_folder = r"../views")
app = cors(app)
app.register_blueprint(mail_oauth_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_callback_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_list_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_retrieve_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(sms_auth_bp, url_prefix = f"/{MODULE_BASE}/sms")
app.register_blueprint(sms_send_bp, url_prefix = f"/{MODULE_BASE}/sms")
app.register_blueprint(chat_auth_bp, url_prefix = f"/{MODULE_BASE}/chat")
app.register_blueprint(chat_webhook_bp, url_prefix = f"/{MODULE_BASE}/chat")
# app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(mail_list_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(mail_retrieve_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(sms_auth_bp, url_prefix = f"/{MODULE_BASE}/sms")
# app.register_blueprint(sms_send_bp, url_prefix = f"/{MODULE_BASE}/sms")
# app.register_blueprint(chat_auth_bp, url_prefix = f"/{MODULE_BASE}/chat")
# app.register_blueprint(chat_webhook_bp, url_prefix = f"/{MODULE_BASE}/chat")
app.register_blueprint(tech_chat_alert_bp, url_prefix = f"/{MODULE_BASE}/tech/alert")
app.register_blueprint(test_callback_bp, url_prefix = f"/{MODULE_BASE}/test")
app.register_blueprint(llm_invoke_bp, url_prefix = f"/{MODULE_BASE}/ai")
@@ -318,53 +325,32 @@ async def app_startup(**kwargs):
current_app.printer("MongoDB ready.")
# ┳ ┓ ┳┳┓ ┓ ┓
# ┃┏┓╋┏┓┏┓┏┓┏┓┃ ┃┃┃┏┓┏┫┏┓┃┏
# ┻┛┗┗┗ ┛ ┛┗┗┻┗ ┛ ┗┗┛┗┻┗ ┗┛
# ┏┓ ┳┳┓ ┓ ┓
# ┃ ┏┓┏┓┏┓ ┃┃┃┏┓┏┫┏┓┃┏
# ┗┛┗┛┛ ┗ ┛ ┗┗┛┗┻┗ ┗┛
current_app.mail_oauth_model = MailOAuthModel(
current_app.core_message_controller = MessageController(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "Mail-OAuth | ",
debug_prefix = "Message (CM) | ",
debug_only_errors = True
)
current_app.mail_sync_model = MailSyncModel(
current_app.core_auth_token_controller = AuthTokenController(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "Mail-Sync | ",
debug_only_errors = True
)
current_app.mail_retrieve_model = MailRetrieveModel(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "Mail-Retr | ",
debug_prefix = "Message (CM) | ",
debug_only_errors = True
)
current_app.sms_auth_model = SMSAuthModel(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "SMS-Auth | ",
debug_only_errors = True
)
current_app.sms_send_model = SMSSendModel(
cache = current_app.module_cache,
alert_url = current_app.script_data["alerts"]["url"],
http_client = current_app.http_client,
debug = enable_debugging,
debug_prefix = "SMS-Send | ",
debug_only_errors = True
)
# ┏┓┏┓┳ ┏┓ ┓┓
# ┣┫┃┃┃ ┃ ┏┓┏┓╋┏┓┏┓┃┃┏┓┏┓┏
# ┛┗┣┛┻ ┗┛┗┛┛┗┗┛ ┗┛┗┗┗ ┛ ┛
current_app.printer("Internal models ready.")
current_app.mail_controller = MailController()
# ┏┓ ┓ ┏┓┓•
# ┃ ┏┓┏┓┏┓┏┓┏╋┏┓┏┓┏ ┏┓┏┓┏┫ ┃ ┃┓┏┓┏┓╋┏
@@ -389,7 +375,7 @@ async def app_startup(**kwargs):
# ┛
# For LLMs:
current_app.llm = LLMOpenAI(
current_app.llm = LLMController(
llm_creds = {
"model": script_cred["openAi"]["model"],
"openai_api_key": script_cred["openAi"]["openai_api_key"]