(20241212) Reorganizing code to perform core actions in one place.
This commit is contained in:
@@ -44,7 +44,7 @@ from utils_v2.string import regex
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# Data models:
|
||||
from models.data.core.message import CoreMessageModel
|
||||
from models.core.message import CoreMessageModel
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
@@ -1,223 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Thursday, 5th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To create an interface between OpenAI and our internal system to perform LLM-based activities.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data Models:
|
||||
from models.data.api.ai.llm import LLMInput, LLMOutput, LLMUsageTokens
|
||||
from models.data.core.user import CoreUserInfoModel
|
||||
|
||||
# To work with LLMs:
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class LLMOpenAI(BaseModel):
|
||||
|
||||
AI_USAGE_COLLECTION = "_aiUsage"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
llm_creds: dict,
|
||||
cache = None,
|
||||
alert_url = None,
|
||||
http_client = None,
|
||||
debug = True,
|
||||
debug_prefix = "Model | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
"""
|
||||
This is the model that works with OpenAi's LLM to perform tasks like text completion.
|
||||
:param llm_creds: The JSON that holds the credentials to access your OpenAI account. Should have the keys
|
||||
'model', and 'openai_api_key'.
|
||||
:param cache: The object to use for caching results from database calls.
|
||||
:param alert_url: Which URL to call when something goes wrong.
|
||||
:param http_client: The instance of an HTTP client to use when trying to send alerts and make other APIs.
|
||||
:param debug: Whether, or not, you would like to print debugging messages:
|
||||
:param debug_prefix: The prefix to print with the debugging messages.
|
||||
:param debug_only_errors: Whether you would like to print only error messages or all messages.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# Initialize the parent:
|
||||
super().__init__(
|
||||
cache = cache,
|
||||
alert_url = alert_url,
|
||||
http_client = http_client,
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix,
|
||||
debug_only_errors = debug_only_errors
|
||||
)
|
||||
|
||||
# Create the interface to the LLM:
|
||||
self.__llm = ChatOpenAI(**llm_creds)
|
||||
|
||||
async def invoke(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
user_info: CoreUserInfoModel,
|
||||
llm_input: LLMInput
|
||||
) -> LLMOutput:
|
||||
|
||||
# Format the message as per the format of OpenAI:
|
||||
prompt = [
|
||||
{
|
||||
"role": {"system": "system", "ai": "assistant", "human": "user"}[message.role],
|
||||
"content": message.content
|
||||
} for message in llm_input.messages
|
||||
]
|
||||
|
||||
# Invoke the AI, and format the response:
|
||||
llm_response = await self.__llm.ainvoke(prompt)
|
||||
llm_response = LLMOutput(
|
||||
messages = llm_input.messages,
|
||||
output = llm_response.content,
|
||||
client = "openai",
|
||||
model = llm_response.response_metadata["model_name"],
|
||||
tokens = LLMUsageTokens(
|
||||
input = llm_response.usage_metadata["input_tokens"],
|
||||
output = llm_response.usage_metadata["output_tokens"],
|
||||
total = llm_response.usage_metadata["total_tokens"],
|
||||
)
|
||||
)
|
||||
|
||||
# Store this into MongoDB:
|
||||
mongo_document = {"user": user_info.model_dump()}
|
||||
for k, v in llm_response.model_dump().items(): mongo_document[k] = v
|
||||
inserted_id = await mongo_conn.insert_one(
|
||||
collection = self.AI_USAGE_COLLECTION,
|
||||
document = mongo_document
|
||||
)
|
||||
if inserted_id: llm_response.invocationId = str(inserted_id)
|
||||
|
||||
# Done here:
|
||||
return llm_response
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
|
||||
# import asyncio
|
||||
#
|
||||
# llm_messages = [
|
||||
# {
|
||||
# "role": "system",
|
||||
# "content": "You are an office assistant."
|
||||
# },
|
||||
# {
|
||||
# "role": "ai",
|
||||
# "content": "Hello, sir. How may I help you today?"
|
||||
# },
|
||||
# {
|
||||
# "role": "human",
|
||||
# "content": "Please summarize this mail for me..."
|
||||
# }
|
||||
# ]
|
||||
#
|
||||
# my_llm = LLMOpenAI(
|
||||
# llm_creds = {
|
||||
# "model": "gpt-4o-mini",
|
||||
# "openai_api_key": "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"
|
||||
# }
|
||||
# )
|
||||
#
|
||||
# async def main():
|
||||
#
|
||||
# llm_response = await my_llm.invoke(llm_input = LLMInput(messages = llm_messages))
|
||||
# print("LLM RESPONSE:", llm_response.model_dump_json(indent = 4))
|
||||
#
|
||||
# asyncio.run(main())
|
||||
@@ -1,388 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Tuesday, 22nd Oct., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To provide an easy way to create models to handle documents for Bicree.
|
||||
This is the base model for this microservice. It will define the structure for all other models that will be
|
||||
used in this particular microservice.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My utils:
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
|
||||
from utils_v2.api.codes import StatusCodes
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import List
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class BaseModel:
|
||||
|
||||
PREVIEW_LENGTH = 250
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cache = None,
|
||||
alert_url = None,
|
||||
http_client = None,
|
||||
debug = True,
|
||||
debug_prefix = "Model | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
"""
|
||||
This is the base model.
|
||||
:param cache: The object to use for caching results from database calls.
|
||||
:param debug: Whether, or not, you would like to print debugging messages:
|
||||
:param debug_prefix: The prefix to print with the debugging messages.
|
||||
:param debug_only_errors: Whether you would like to print only error messages or all messages.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# Prepare the caching utility:
|
||||
self._cache = cache
|
||||
|
||||
# For sending alerts:
|
||||
self._alert_url = alert_url
|
||||
self._http_client = http_client
|
||||
|
||||
# Prepare the debugging utility:
|
||||
self._debug_prefix = debug_prefix
|
||||
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
|
||||
if not debug: self._printer.disable()
|
||||
self._debug_only_errors = debug_only_errors
|
||||
|
||||
# A semaphore for activities that must absolutely be done one at a time:
|
||||
self.__exclusive_semaphore = asyncio.Semaphore(1)
|
||||
|
||||
# A simple debugging output:
|
||||
self._printer("Model initialized.")
|
||||
|
||||
def enable_terminal_print(self):
|
||||
self._printer.enable()
|
||||
|
||||
def disable_terminal_print(self):
|
||||
self._printer.disable()
|
||||
|
||||
def debug_only_errors(self):
|
||||
self._debug_only_errors = True
|
||||
|
||||
def debug_everything(self):
|
||||
self._debug_only_errors = False
|
||||
|
||||
async def send_alert(
|
||||
self,
|
||||
message: str,
|
||||
session_token = None,
|
||||
alert_type = "error"
|
||||
):
|
||||
|
||||
"""
|
||||
Sends out an alert (ideally through the tech module). This is meant to be used when some exception occurs, and
|
||||
you want to be informed before the client complains.
|
||||
:param message: The message to send out to the admins.
|
||||
:param session_token: The session token of the user (optional) so that the alert message can display the name of
|
||||
the user who faced the trouble.
|
||||
:param alert_type: The type of alert to throw ("error", "warning", or "info").
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
if self._http_client is not None and self._alert_url is not None:
|
||||
response = await self._http_client.post(
|
||||
url = self._alert_url,
|
||||
headers = {"X-Session-Token": session_token} if session_token else None,
|
||||
json = {
|
||||
"message": message,
|
||||
"type": alert_type
|
||||
}
|
||||
)
|
||||
|
||||
async def call_cached_procedure(
|
||||
self,
|
||||
cache: AsyncRedisCache,
|
||||
cache_key: str,
|
||||
cache_expiry: int,
|
||||
db_conn: AsyncMySQL,
|
||||
proc_name: str,
|
||||
proc_args: tuple,
|
||||
retry_count: int = 1,
|
||||
backoff_seconds: float = 0.5,
|
||||
backoff_multiplier: float = 1.1,
|
||||
session_token: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
Calls a stored procedure and returns the response as a JSON-like object (dict or list).
|
||||
:param cache: The caching object to use to set the session in cache memory.
|
||||
:param cache_key: The string to use as the key when caching the response.
|
||||
:param cache_expiry: The no. of seconds after which this information will be deleted from the cache.
|
||||
:param db_conn: The connection instance to use to call the procedure.
|
||||
:param proc_name: The name of the stored procedure that must be called.
|
||||
:param proc_args: The args to be sent to the stored procedure.
|
||||
:param retry_count: The max. number of times to try in case one or more attempts fail.
|
||||
:param backoff_seconds: The time to wait before making the next attempt if the retry count is more than 1.
|
||||
:param backoff_multiplier: The factor that dictates how much to modify the time delay by when waiting to retry.
|
||||
:param session_token: A session token to share with the tech module when alerts need to be sent out for any
|
||||
occurrence of exceptions. If this is passed, the tech module will be able to tell you which user faced the
|
||||
issue.
|
||||
:return: The response from the stored procedure.
|
||||
"""
|
||||
|
||||
# check for the data in cache:
|
||||
data = await cache.get(cache_key)
|
||||
|
||||
# If the data isn't in the cache, call the procedure:
|
||||
if data is None:
|
||||
|
||||
# Make the database call:
|
||||
data = await self.call_procedure(
|
||||
db_conn = db_conn,
|
||||
proc_name = proc_name,
|
||||
proc_args = proc_args,
|
||||
retry_count = retry_count,
|
||||
backoff_seconds = backoff_seconds,
|
||||
backoff_multiplier = backoff_multiplier,
|
||||
session_token = session_token
|
||||
)
|
||||
|
||||
# If the database call succeeded, cache the response:
|
||||
if isinstance(data, dict) and data["status"] == 1:
|
||||
await cache.set(key = cache_key, value = data, expiry = cache_expiry)
|
||||
|
||||
# Done here:
|
||||
return data
|
||||
|
||||
async def call_procedure(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
proc_name: str,
|
||||
proc_args: tuple,
|
||||
retry_count: int = 1,
|
||||
backoff_seconds: float = 0.5,
|
||||
backoff_multiplier: float = 1.1,
|
||||
session_token: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
Calls a stored procedure and returns the response as a JSON-like object (dict or list).
|
||||
:param db_conn: The connection instance to use to call the procedure.
|
||||
:param proc_name: The name of the stored procedure that must be called.
|
||||
:param proc_args: The args to be sent to the stored procedure.
|
||||
:param retry_count: The max. number of times to try in case one or more attempts fail.
|
||||
:param backoff_seconds: The time to wait before making the next attempt if the retry count is more than 1.
|
||||
:param backoff_multiplier: The factor that dictates how much to modify the time delay by when waiting to retry.
|
||||
:param session_token: A session token to share with the tech module when alerts need to be sent out for any
|
||||
occurrence of exceptions. If this is passed, the tech module will be able to tell you which user faced the
|
||||
issue.
|
||||
:return: The response from the stored procedure.
|
||||
"""
|
||||
|
||||
# Call the stored procedure:
|
||||
db_json, exception = await db_conn.call_procedure_and_get_json(
|
||||
proc_name,
|
||||
proc_args,
|
||||
retry_count = retry_count,
|
||||
backoff_seconds = backoff_seconds,
|
||||
backoff_multiplier = backoff_multiplier,
|
||||
return_exception = True
|
||||
)
|
||||
|
||||
# Understand the response:
|
||||
success = True if db_json["status"] == 1 else False
|
||||
message = db_json.get("message")
|
||||
|
||||
# Debugging print:
|
||||
if not success or not self._debug_only_errors:
|
||||
self._printer(proc_name, proc_args, success, exception, message)
|
||||
|
||||
# Send an alert out on exceptions:
|
||||
if exception is not None:
|
||||
|
||||
# Format the message in Markdown format:
|
||||
exception_string = str(exception).replace("`", "'")
|
||||
formatted_message = f"*Module:*\n`{self._debug_prefix}`\n\n"
|
||||
formatted_message += f"*Proc:*\n`{proc_name}`\n\n"
|
||||
formatted_message += f"*Args:*\n`({', '.join([str(_) for _ in proc_args])})`\n\n"
|
||||
formatted_message += f"*Arg-Types:*\n`({', '.join([type(_).__name__ for _ in proc_args])})`\n\n"
|
||||
formatted_message += f"*Message:*\n`{message}`\n\n"
|
||||
formatted_message += f"*Success:*\n`{success}`\n\n"
|
||||
formatted_message += f"*Exception:*\n`{exception_string}`\n\n"
|
||||
|
||||
# Send the alert:
|
||||
await self.send_alert(formatted_message, session_token = session_token)
|
||||
|
||||
# Return the response:
|
||||
db_json["status_code"] = StatusCodes.OK if success else StatusCodes.FAILED
|
||||
return db_json
|
||||
|
||||
async def execute_one(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
query: str,
|
||||
session_token: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
Runs one query and sends an alert if that fails.
|
||||
:param db_conn: The connection to use to run the query.
|
||||
:param query: The query to run.
|
||||
:param session_token: A session token to share with the tech module when alerts need to be sent out for any
|
||||
occurrence of exceptions. If this is passed, the tech module will be able to tell you which user faced the
|
||||
issue.
|
||||
:return: The response from the database.
|
||||
"""
|
||||
|
||||
# Run the query:
|
||||
rows_affected, db_response, exception = await db_conn.execute_one(query = query, return_exception = True)
|
||||
|
||||
# Send an alert out on exceptions:
|
||||
if exception is not None:
|
||||
|
||||
# Created needed previews:
|
||||
query_preview = query if len(query) <= self.PREVIEW_LENGTH else query[:self.PREVIEW_LENGTH] + "..."
|
||||
|
||||
# Format the message in Markdown format:
|
||||
formatted_message = f"*Module:*\n`{self._debug_prefix}`\n\n"
|
||||
formatted_message += f"*Query:*\n`{query_preview}`\n\n"
|
||||
formatted_message += f"*Rows Affected:*\n`{rows_affected}`\n\n"
|
||||
formatted_message += f"*DB Response:*\n`{db_response}`\n\n"
|
||||
formatted_message += f"*Exception:*\n`{exception}`\n\n"
|
||||
|
||||
# Send the alert:
|
||||
await self.send_alert(formatted_message, session_token = session_token)
|
||||
|
||||
# Return the response:
|
||||
return rows_affected, db_response
|
||||
|
||||
async def execute_many(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
query: str,
|
||||
data: List[tuple],
|
||||
session_token: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
Runs many queries and sends an alert if that fails.
|
||||
:param db_conn: The connection to use to run the query.
|
||||
:param query: The query to run.
|
||||
:param data: The data to feed into the query.
|
||||
:param session_token: A session token to share with the tech module when alerts need to be sent out for any
|
||||
occurrence of exceptions. If this is passed, the tech module will be able to tell you which user faced the
|
||||
issue.
|
||||
:return: The response from the database.
|
||||
"""
|
||||
|
||||
# Run the query:
|
||||
rows_affected, db_response, exception = await db_conn.execute_many(
|
||||
query = query,
|
||||
data = data,
|
||||
return_exception = True
|
||||
)
|
||||
|
||||
# Send an alert out on exceptions:
|
||||
if exception is not None:
|
||||
|
||||
# Created needed previews:
|
||||
query_preview = query if len(query) <= self.PREVIEW_LENGTH else query[:self.PREVIEW_LENGTH] + "..."
|
||||
data_preview = str(data)
|
||||
if len(data_preview) > self.PREVIEW_LENGTH: data_preview = data_preview[:self.PREVIEW_LENGTH] + "..."
|
||||
|
||||
# Format the message in Markdown format:
|
||||
formatted_message = f"*Module:*\n`{self._debug_prefix}`\n\n"
|
||||
formatted_message += f"*Query:*\n`{query_preview}`\n\n"
|
||||
formatted_message += f"*Data:*\n`{data_preview}`\n\n"
|
||||
formatted_message += f"*Rows Affected:*\n`{rows_affected}`\n\n"
|
||||
formatted_message += f"*DB Response:*\n`{db_response}`\n\n"
|
||||
formatted_message += f"*Exception:*\n`{exception}`\n\n"
|
||||
|
||||
# Send the alert:
|
||||
await self.send_alert(formatted_message, session_token = session_token)
|
||||
|
||||
# Return the response:
|
||||
return rows_affected, db_response
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,444 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Thursday, 12th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To define all file-management activities in one place.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level:
|
||||
import io
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.system import files
|
||||
from utils_v2.security.hash import Hasher
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.core.user import CoreUserInfoModel
|
||||
from models.data.core.file import CoreFileInfoModel, CoreFileAccessResponseModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, Literal, List
|
||||
|
||||
# To make deep copies:
|
||||
import copy
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class FileManagementModel:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
debug = True,
|
||||
debug_prefix = "File Objs. | ",
|
||||
):
|
||||
|
||||
# Debugging:
|
||||
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
|
||||
if not debug: self._printer.disable()
|
||||
|
||||
def enable_debug(self):
|
||||
self._printer.enable()
|
||||
|
||||
def disable_debug(self):
|
||||
self._printer.disable()
|
||||
|
||||
# ┏┓ • ┓ ┏┓ •
|
||||
# ┃┃┓┏┓┏┃┏ ┃┃┓┏┏┓┏┓┓┏┓┏
|
||||
# ┗┻┗┻┗┗┛┗ ┗┻┗┻┗ ┛ ┗┗ ┛
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId | str
|
||||
) -> CoreFileAccessResponseModel:
|
||||
|
||||
"""
|
||||
To check whether, or not, a particular file's record exists in the database.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file to check.
|
||||
:return: A structured response where the existence of the file is noted in the 'result' field.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Run the query:
|
||||
record = await mongo_conn.find_one_file(
|
||||
filter = {"_id": ObjectId(file_id)},
|
||||
projection = {"_id": True, "user": True},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the result:
|
||||
if record:
|
||||
response.data = True
|
||||
response.success = True
|
||||
response.message = "ok"
|
||||
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.data = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
async def info(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId | str
|
||||
) -> CoreFileAccessResponseModel:
|
||||
|
||||
"""
|
||||
To get the information about this file.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file to check.
|
||||
:return: A structured response where the info of the file is noted in the 'result' field.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Run the query:
|
||||
record = await mongo_conn.find_one_file(
|
||||
filter = {"_id": ObjectId(file_id)},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the result:
|
||||
response.success = True
|
||||
if record:
|
||||
response.data = CoreFileInfoModel(**record["metadata"])
|
||||
response.message = "ok"
|
||||
else:
|
||||
response.message = "no such file object"
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.data = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
async def is_private(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId | str
|
||||
) -> bool | None:
|
||||
|
||||
"""
|
||||
To check whether, or not, a particular file is publicly readable.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file to check.
|
||||
:return: True if private, else False. None if it doesn't exist at all.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Run the query:
|
||||
record = await mongo_conn.find_one_file(
|
||||
filter = {"_id": ObjectId(file_id)},
|
||||
projection = {"_id": False, "isPrivate": True},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the result:
|
||||
if record:
|
||||
response.data = record["metadata"]["isPrivate"]
|
||||
response.success = True
|
||||
response.message = "ok"
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.data = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
# ┓ • •
|
||||
# ┃ ┓┏╋┓┏┓┏┓
|
||||
# ┗┛┗┛┗┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
pass
|
||||
|
||||
# ┳┓ ┓•
|
||||
# ┣┫┏┓┏┓┏┫┓┏┓┏┓
|
||||
# ┛┗┗ ┗┻┗┻┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
async def download_file(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId | str
|
||||
) -> CoreFileAccessResponseModel:
|
||||
|
||||
"""
|
||||
To quickly download small files. Do not use this for larger files because the file will be held in RAM first
|
||||
and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file to fetch.
|
||||
:return: The file in a BytesIO buffer in the 'data' field of the structured response.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Get the file from the database:
|
||||
buffer = io.BytesIO()
|
||||
response.success = await mongo_conn.easy_download(
|
||||
destination = buffer,
|
||||
file_id = ObjectId(file_id),
|
||||
raise_exception = True
|
||||
)
|
||||
buffer.seek(0)
|
||||
|
||||
# Note down the results:
|
||||
response.message = (
|
||||
"file fetched successfully" if response.success
|
||||
else "file fetching failed"
|
||||
)
|
||||
response.data = buffer if response.success else None
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.data = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
async def get_file_download_stream(
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId
|
||||
) -> Any:
|
||||
|
||||
"""
|
||||
To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM. Once
|
||||
the stream is created you can read from it, and terminate it like this:
|
||||
READ: await stream.read(chunk_size)
|
||||
CLOSE (without awaiting): stream.close()
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file whose stream you would like to fetch.
|
||||
:return: Returns the stream object that will allow more efficient downloads of files on the user's end.
|
||||
"""
|
||||
|
||||
return await mongo_conn.get_download_stream(file_id = file_id)
|
||||
|
||||
# ┓ ┏ • •
|
||||
# ┃┃┃┏┓┓╋┓┏┓┏┓
|
||||
# ┗┻┛┛ ┗┗┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_info: CoreFileInfoModel,
|
||||
file_data: io.BytesIO | str,
|
||||
chunk_size: int = None
|
||||
):
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Read the file's data into a BytesIO object:
|
||||
if isinstance(file_data, str):
|
||||
file_data = io.BytesIO(files.read_file(file_data, mode = "rb"))
|
||||
file_data.seek(0)
|
||||
|
||||
# Hash the file's data:
|
||||
hasher = Hasher()
|
||||
hasher.update(file_data.getvalue())
|
||||
file_info.hash = hasher.hexdigest()
|
||||
|
||||
# Save the file to the database:
|
||||
file_data.seek(0)
|
||||
response.success = await mongo_conn.easy_upload(
|
||||
source = file_data,
|
||||
file_name = file_info.filename,
|
||||
file_metadata = file_info.model_dump(),
|
||||
file_id = file_info.fileId,
|
||||
chunk_size = chunk_size,
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the results:
|
||||
response.message = (
|
||||
"file saved successfully" if response.success
|
||||
else "file saving failed"
|
||||
)
|
||||
response.data = True if response.success else False
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.data = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
async def upload_from_stream(self): pass
|
||||
|
||||
# ┳┓ ┓ •
|
||||
# ┃┃┏┓┃┏┓╋┓┏┓┏┓
|
||||
# ┻┛┗ ┗┗ ┗┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
async def delete_file(self): pass
|
||||
|
||||
# ┏┓ • •
|
||||
# ┃┃┏┓┏┓┏┳┓┓┏┏┓┏┓┏┓┏
|
||||
# ┣┛┗ ┛ ┛┗┗┗┛┛┗┗┛┛┗┛
|
||||
|
||||
async def make_public(self): pass
|
||||
|
||||
async def make_private(self): pass
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
async def main():
|
||||
|
||||
files_mongo = AsyncMongoStorage(
|
||||
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
|
||||
database_name = "converseStore",
|
||||
max_connections = 10,
|
||||
debug = True
|
||||
)
|
||||
await files_mongo.connect()
|
||||
my_files = FileManagementModel()
|
||||
|
||||
user_bhopli = CoreUserInfoModel(
|
||||
fullName = "Bhopli Narangi",
|
||||
userId = 1,
|
||||
entityId = 2,
|
||||
billingAccountId = 3,
|
||||
departmentId = 4,
|
||||
branchId = 5,
|
||||
industry = "technology"
|
||||
)
|
||||
|
||||
file_info = await my_files.info(
|
||||
mongo_conn = files_mongo,
|
||||
file_id = "67598d48c1bf89b25695f20b"
|
||||
)
|
||||
print("FILE INFO:", json.to_string(file_info.model_dump(), default = str))
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,807 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Tuesday, 10th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To define all file-management activities in one place.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
import io
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.system import files
|
||||
from utils_v2.security.hash import Hasher
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.core.user import CoreUserInfoModel
|
||||
from models.data.core.file_object import (
|
||||
CoreFileObjectInfoModel,
|
||||
CoreFileObjectSharingModel,
|
||||
CoreFileObjectPermissionsModel,
|
||||
CoreFileObjectAccessResponseModel
|
||||
)
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, Literal, List
|
||||
|
||||
# To make deep copies:
|
||||
import copy
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class FileObjectManagementModel:
|
||||
|
||||
# Define class-level variables:
|
||||
FILE_OBJECTS_COLLECTION = "_fileObjects"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
debug = True,
|
||||
debug_prefix = "File Objs. | ",
|
||||
):
|
||||
|
||||
# Debugging:
|
||||
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
|
||||
if not debug: self._printer.disable()
|
||||
|
||||
def enable_debug(self):
|
||||
self._printer.enable()
|
||||
|
||||
def disable_debug(self):
|
||||
self._printer.disable()
|
||||
|
||||
# ┓┏ ┓
|
||||
# ┣┫┏┓┃┏┓┏┓┏┓┏
|
||||
# ┛┗┗ ┗┣┛┗ ┛ ┛
|
||||
# ┛
|
||||
|
||||
@staticmethod
|
||||
def users_match(
|
||||
user_p: CoreUserInfoModel,
|
||||
user_r: CoreUserInfoModel,
|
||||
ignore_null: bool = True
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
To match if a user that is requesting a resource is the same as the user known to have access to the resource.
|
||||
:param user_p: One of the dicts to check.
|
||||
:param user_r: The other dict to check.
|
||||
:param ignore_null: Whether to consider only non-null values, or all values.
|
||||
:return: True if they match, else False.
|
||||
"""
|
||||
|
||||
# Start by assuming success:
|
||||
are_matching = True
|
||||
|
||||
# Iterate through the required items:
|
||||
for rk, rv in user_r.model_dump().items():
|
||||
|
||||
# Do not consider fields that are nulls if asked to ignore them:
|
||||
if ignore_null and rv is None: continue
|
||||
|
||||
# Extract the corresponding value from the other user,
|
||||
# and test it for being equal:
|
||||
pv = getattr(user_p, rk, None)
|
||||
if (
|
||||
(not isinstance(rv, type(pv))) or
|
||||
(rv != pv)
|
||||
):
|
||||
are_matching = False
|
||||
break
|
||||
|
||||
# Done here:
|
||||
return are_matching
|
||||
|
||||
# ┏┓ • ┓ ┏┓ •
|
||||
# ┃┃┓┏┓┏┃┏ ┃┃┓┏┏┓┏┓┓┏┓┏
|
||||
# ┗┻┗┻┗┗┛┗ ┗┻┗┻┗ ┛ ┗┗ ┛
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_object_id: ObjectId | str
|
||||
) -> CoreFileObjectAccessResponseModel:
|
||||
|
||||
"""
|
||||
To check whether, or not, a particular file's record exists in the database.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_object_id: The id of the file to check.
|
||||
:return: A structured response where the existence of the file is noted in the 'result' field.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileObjectAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Run the query:
|
||||
record = await mongo_conn.find_one(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = {"_id": ObjectId(file_object_id)},
|
||||
projection = {"_id": True, "user": True, "isDir": True},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the result:
|
||||
if record:
|
||||
response.result = True
|
||||
response.success = True
|
||||
response.message = "ok"
|
||||
|
||||
except Exception as exception:
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.result = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
async def info(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_object_id: ObjectId | str
|
||||
) -> CoreFileObjectAccessResponseModel:
|
||||
|
||||
"""
|
||||
To get the information about this file.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_object_id: The id of the file to check.
|
||||
:return: A structured response where the info of the file is noted in the 'result' field.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileObjectAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Run the query:
|
||||
record = await mongo_conn.find_one(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = {"_id": ObjectId(file_object_id)},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the result:
|
||||
response.success = True
|
||||
if record:
|
||||
response.result = CoreFileObjectInfoModel(**record)
|
||||
response.message = "ok"
|
||||
else:
|
||||
response.message = "no such file object"
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.result = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
async def is_private(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_object_id: ObjectId | str
|
||||
) -> bool | None:
|
||||
|
||||
"""
|
||||
To check whether, or not, a particular file is publicly readable.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_object_id: The id of the file to check.
|
||||
:return: True if private, else False. None if it doesn't exist at all.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileObjectAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Run the query:
|
||||
record = await mongo_conn.find_one(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = {"_id": ObjectId(file_object_id)},
|
||||
projection = {"_id": False, "isPrivate": True},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Note down the result:
|
||||
if record:
|
||||
response.result = record["isPrivate"]
|
||||
response.success = True
|
||||
response.message = "ok"
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.result = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
def is_owner(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
file_object_info: CoreFileObjectInfoModel,
|
||||
ignore_null: bool = True
|
||||
) -> CoreFileObjectAccessResponseModel:
|
||||
|
||||
"""
|
||||
To check if a specific user is the owner of a specific file.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param user_info: The details of the user who needs to have permissions to this file.
|
||||
:param file_object_info: The information about the file/dir. Fetch it from the 'info' method.
|
||||
:param ignore_null: Whether to consider only non-null values, or all values.
|
||||
:return: True if owner, else False. None if something goes wrong.
|
||||
"""
|
||||
|
||||
# Create a response:
|
||||
response = CoreFileObjectAccessResponseModel()
|
||||
|
||||
try:
|
||||
|
||||
# Test for a match:
|
||||
if self.users_match(
|
||||
user_p = file_object_info.user,
|
||||
user_r = user_info,
|
||||
ignore_null = ignore_null
|
||||
):
|
||||
response.result = True
|
||||
response.message = "user is the owner of this resource"
|
||||
else:
|
||||
response.result = False
|
||||
response.message = "user is not the owner of this resource"
|
||||
response.success = True
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
response.exception = exception
|
||||
response.message = str(exception)
|
||||
response.success = False
|
||||
response.result = None
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
def has_permission(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
file_object_info: CoreFileObjectInfoModel,
|
||||
permission: Literal["read", "write", "delete", "changePermissions"],
|
||||
ignore_null: bool = True
|
||||
) -> CoreFileObjectAccessResponseModel:
|
||||
|
||||
"""
|
||||
To check if a particular user has permissions to a given file obj. You may pass either an instance of the file's
|
||||
info, or you may send the file's id to check. If you pass just the file's id, a database call will be needed.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param user_info: The details of the user who needs to have permissions to this file.
|
||||
:param permission: The name of the permission that the said user must have on this file.
|
||||
:param file_object_info: The information about the file/dir. Fetch it from the 'info' method.
|
||||
:param ignore_null: Whether to consider only non-null values, or all values.
|
||||
:return: True if the user has said permission, else False. None if something goes wrong.
|
||||
"""
|
||||
|
||||
# If this is a public file/dir,
|
||||
# and the permission requested is 'read':
|
||||
if (not file_object_info.isPrivate) and permission == "read":
|
||||
return CoreFileObjectAccessResponseModel(
|
||||
success = True,
|
||||
message = "this resource is publicly available",
|
||||
result = True,
|
||||
exception = None
|
||||
)
|
||||
|
||||
# The owner always has all permissions:
|
||||
response = self.is_owner(
|
||||
mongo_conn = mongo_conn,
|
||||
user_info = user_info,
|
||||
file_object_info = file_object_info,
|
||||
ignore_null = ignore_null
|
||||
)
|
||||
if not response.success: return response
|
||||
if response.result is True: return response
|
||||
|
||||
# Note down the failure of the ownership test:
|
||||
response.message = "this user does not have the requested permission over this resource"
|
||||
|
||||
# Since The person requesting this is not the owner,
|
||||
# we check with the sharing details:
|
||||
for sharing_data in file_object_info.sharedWith:
|
||||
|
||||
# We match the users.
|
||||
# If they don't match, we move to the next user:
|
||||
if not self.users_match(
|
||||
user_p = sharing_data.user,
|
||||
user_r = user_info,
|
||||
ignore_null = ignore_null
|
||||
): continue
|
||||
|
||||
# If we found a matching user,
|
||||
# we check for the permission:
|
||||
if getattr(sharing_data.permissions, permission, False):
|
||||
response.result = True
|
||||
response.success = True
|
||||
response.message = "this user has the requested permission over this resource"
|
||||
break
|
||||
|
||||
# Done here:
|
||||
return response
|
||||
|
||||
# ┓ • •
|
||||
# ┃ ┓┏╋┓┏┓┏┓
|
||||
# ┗┛┗┛┗┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
async def list_owned_dirs(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
limit: int = 50,
|
||||
skip: int = 0,
|
||||
) -> List[CoreFileObjectInfoModel] | None:
|
||||
|
||||
"""
|
||||
To list all the dirs that are owned by the described user.
|
||||
:param mongo_conn: The connection instance to use to make the check.
|
||||
:param user_info: The information about the user that we must match.
|
||||
:param limit: How many max. records to fetch.
|
||||
:param skip: How many initial records to skip. Useful for pagination.
|
||||
:return: The list of files owned by the user (can be empty), or None if something goes wrong.
|
||||
"""
|
||||
|
||||
# Run the query:
|
||||
records = await mongo_conn.find_many(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"user": {k: v for k, v in user_info.model_dump().items() if v is not None},
|
||||
"isDir": True
|
||||
}),
|
||||
limit = limit,
|
||||
skip = skip
|
||||
)
|
||||
|
||||
# If something went wrong, we receive null for the records.
|
||||
# We pass that null on:
|
||||
if records is None: return None
|
||||
|
||||
# Otherwise, we format and return the records:
|
||||
return [CoreFileObjectInfoModel(**record) for record in records]
|
||||
|
||||
async def list_shared_dirs(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
limit: int = 50,
|
||||
skip: int = 0,
|
||||
) -> List[CoreFileObjectInfoModel] | None:
|
||||
|
||||
"""
|
||||
To list all the dirs that have been shared with the described user.
|
||||
:param mongo_conn: The connection instance to use to make the check.
|
||||
:param user_info: The information about the user that we must match.
|
||||
:param limit: How many max. records to fetch.
|
||||
:param skip: How many initial records to skip. Useful for pagination.
|
||||
:return: The list of files owned by the user (can be empty), or None if something goes wrong.
|
||||
"""
|
||||
|
||||
# Run the query:
|
||||
records = await mongo_conn.find_many(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"sharedWith.user": {k: v for k, v in user_info.model_dump().items() if v is not None},
|
||||
"isDir": True
|
||||
}),
|
||||
limit = limit,
|
||||
skip = skip
|
||||
)
|
||||
|
||||
# If something went wrong, we receive null for the records.
|
||||
# We pass that null on:
|
||||
if records is None: return None
|
||||
|
||||
# Otherwise, we format and return the records:
|
||||
return [CoreFileObjectInfoModel(**record) for record in records]
|
||||
|
||||
async def list_owned_files(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
limit: int = 50,
|
||||
skip: int = 0,
|
||||
) -> List[CoreFileObjectInfoModel] | None:
|
||||
|
||||
"""
|
||||
To list all the files that are owned by the described user.
|
||||
:param mongo_conn: The connection instance to use to make the check.
|
||||
:param user_info: The information about the user that we must match.
|
||||
:param limit: How many max. records to fetch.
|
||||
:param skip: How many initial records to skip. Useful for pagination.
|
||||
:return: The list of files owned by the user (can be empty), or None if something goes wrong.
|
||||
"""
|
||||
|
||||
# Run the query:
|
||||
records = await mongo_conn.find_many(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"user": {k: v for k, v in user_info.model_dump().items() if v is not None},
|
||||
"isDir": False
|
||||
}),
|
||||
limit = limit,
|
||||
skip = skip
|
||||
)
|
||||
|
||||
# If something went wrong, we receive null for the records.
|
||||
# We pass that null on:
|
||||
if records is None: return None
|
||||
|
||||
# Otherwise, we format and return the records:
|
||||
return [CoreFileObjectInfoModel(**record) for record in records]
|
||||
|
||||
async def list_shared_files(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
limit: int = 50,
|
||||
skip: int = 0,
|
||||
) -> List[CoreFileObjectInfoModel] | None:
|
||||
|
||||
"""
|
||||
To list all the files that have been shared with the described user.
|
||||
:param mongo_conn: The connection instance to use to make the check.
|
||||
:param user_info: The information about the user that we must match.
|
||||
:param limit: How many max. records to fetch.
|
||||
:param skip: How many initial records to skip. Useful for pagination.
|
||||
:return: The list of files owned by the user (can be empty), or None if something goes wrong.
|
||||
"""
|
||||
|
||||
# Run the query:
|
||||
records = await mongo_conn.find_many(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"sharedWith.user": {k: v for k, v in user_info.model_dump().items() if v is not None},
|
||||
"isDir": False
|
||||
}),
|
||||
limit = limit,
|
||||
skip = skip
|
||||
)
|
||||
|
||||
# If something went wrong, we receive null for the records.
|
||||
# We pass that null on:
|
||||
if records is None: return None
|
||||
|
||||
# Otherwise, we format and return the records:
|
||||
return [CoreFileObjectInfoModel(**record) for record in records]
|
||||
|
||||
# ┳┓ ┓•
|
||||
# ┣┫┏┓┏┓┏┫┓┏┓┏┓
|
||||
# ┛┗┗ ┗┻┗┻┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
@staticmethod
|
||||
async def download_file(
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId
|
||||
) -> io.BytesIO | None:
|
||||
|
||||
"""
|
||||
To quickly download small files. Do not use this for larger files because the file will be held in RAM first
|
||||
and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory.
|
||||
WARNING: Check for permissions before using this method.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file to fetch.
|
||||
:return: The file in a BytesIO buffer, or None if the file doesn't exist.
|
||||
"""
|
||||
|
||||
# Get the file from the database:
|
||||
buffer = io.BytesIO()
|
||||
success = await mongo_conn.easy_download(
|
||||
destination = buffer,
|
||||
file_id = ObjectId(file_id),
|
||||
raise_exception = True
|
||||
)
|
||||
buffer.seek(0)
|
||||
|
||||
# Return the result:
|
||||
if not success: return None
|
||||
else: return buffer
|
||||
|
||||
@staticmethod
|
||||
async def get_file_download_stream(
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_id: ObjectId
|
||||
) -> Any:
|
||||
|
||||
"""
|
||||
To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM. Once
|
||||
the stream is created you can read from it, and terminate it like this:
|
||||
READ: await stream.read(chunk_size)
|
||||
CLOSE (without awaiting): stream.close()
|
||||
WARNING: Check for permissions before using this method.
|
||||
:param mongo_conn: The instance of the database connection to perform this action.
|
||||
:param file_id: The id of the file whose stream you would like to fetch.
|
||||
:return: Returns the stream object that will allow more efficient downloads of files on the user's end.
|
||||
"""
|
||||
|
||||
return await mongo_conn.get_download_stream(file_id = file_id)
|
||||
|
||||
# ┓ ┏ • •
|
||||
# ┃┃┃┏┓┓╋┓┏┓┏┓
|
||||
# ┗┻┛┛ ┗┗┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
async def make_dir(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
user_info: CoreUserInfoModel,
|
||||
dir_name: str,
|
||||
dir_metadata: dict = None,
|
||||
dir_tags: list = None,
|
||||
parent_id: ObjectId | str = None
|
||||
) -> bool:
|
||||
|
||||
# Create an instance of the directory's model:
|
||||
dir_model = CoreFileObjectInfoModel(
|
||||
_id = mongo_conn.generate_id(),
|
||||
user = user_info,
|
||||
isDir = True,
|
||||
name = dir_name,
|
||||
createTs = date_time.get_current_utc_date_time(as_string = False),
|
||||
metadata = dir_metadata,
|
||||
tags = dir_tags,
|
||||
parentId = parent_id,
|
||||
isPrivate = True
|
||||
)
|
||||
|
||||
print("DIRECTORY:", json.to_string(dir_model.model_dump(), default = str))
|
||||
|
||||
# Insert this document into the database:
|
||||
inserted_id = await mongo_conn.insert_one(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
document = dir_model.model_dump()
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return True if inserted_id else False
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
mongo_conn: AsyncMongoStorage,
|
||||
file_info: CoreFileObjectInfoModel,
|
||||
file_data: io.BytesIO | str
|
||||
):
|
||||
|
||||
# Start by assuming failure:
|
||||
file_uploaded = False
|
||||
|
||||
# Start a session:
|
||||
async with await (await mongo_conn.client).start_session() as session:
|
||||
|
||||
# Define the transaction options:
|
||||
options = {
|
||||
# "read_concern": {"level": "snapshot"}, # ... Optional: ensures consistent reads.
|
||||
# "write_concern": {"w": "majority"}, # ...... Ensures writes are acknowledged.
|
||||
# "read_preference": "primary", # ............ Specify where to read from (e.g., primary).
|
||||
}
|
||||
|
||||
# Start the transaction:
|
||||
async with session.start_transaction(**options):
|
||||
|
||||
try:
|
||||
|
||||
# Check that we indeed have a file that we are uploading:
|
||||
if file_info.isDir: raise ValueError("cannot data to upload directory object")
|
||||
|
||||
# Generate an ObjectId:
|
||||
file_info.fileObjectId = mongo_conn.generate_id(as_str = False)
|
||||
|
||||
# Read the file's data into a BytesIO object:
|
||||
if isinstance(file_data, str):
|
||||
file_data = io.BytesIO(files.read_file(file_data, mode = "rb"))
|
||||
file_data.seek(0)
|
||||
|
||||
# Hash the file's data:
|
||||
hasher = Hasher()
|
||||
hasher.update(file_data.getvalue())
|
||||
file_info.hash = hasher.hexdigest()
|
||||
|
||||
# Now we upload the actual content of the file with the same id:
|
||||
success = await mongo_conn.easy_upload(
|
||||
source = file_data,
|
||||
file_name = file_info.name,
|
||||
file_metadata = None,
|
||||
file_id = file_info.fileObjectId,
|
||||
session = session,
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Safety check to ensure that the data was written:
|
||||
if not success:
|
||||
raise ValueError("file object's bytes weren't uploaded")
|
||||
|
||||
# Write the file object's info model now with the same id:
|
||||
inserted_id = await mongo_conn.insert_one(
|
||||
collection = self.FILE_OBJECTS_COLLECTION,
|
||||
document = file_info.model_dump(),
|
||||
session = session,
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Safety check to ensure that the info was written:
|
||||
if inserted_id is None:
|
||||
raise ValueError("file object's info wasn't inserted")
|
||||
|
||||
# If we've reached this far:
|
||||
file_uploaded = True
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
session.abort_transaction()
|
||||
file_uploaded = False
|
||||
self._printer(exception)
|
||||
|
||||
# Done here:
|
||||
return file_uploaded
|
||||
|
||||
async def upload_from_stream(self): pass
|
||||
|
||||
# ┳┓ ┓ •
|
||||
# ┃┃┏┓┃┏┓╋┓┏┓┏┓
|
||||
# ┻┛┗ ┗┗ ┗┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
async def delete_dir(self): pass
|
||||
|
||||
async def delete_file(self): pass
|
||||
|
||||
# ┏┓ • •
|
||||
# ┃┃┏┓┏┓┏┳┓┓┏┏┓┏┓┏┓┏
|
||||
# ┣┛┗ ┛ ┛┗┗┗┛┛┗┗┛┛┗┛
|
||||
|
||||
async def update_permissions(self): pass
|
||||
|
||||
async def make_public(self): pass
|
||||
|
||||
async def make_private(self): pass
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
async def main():
|
||||
|
||||
files_mongo = AsyncMongoStorage(
|
||||
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
|
||||
database_name = "converseStore",
|
||||
max_connections = 10,
|
||||
debug = True
|
||||
)
|
||||
await files_mongo.connect()
|
||||
my_fs = FileObjectManagementModel()
|
||||
|
||||
user_bhopli = CoreUserInfoModel(
|
||||
fullName = "Bhopli Narangi",
|
||||
userId = 1,
|
||||
entityId = 2,
|
||||
billingAccountId = 3,
|
||||
departmentId = 4,
|
||||
branchId = 5,
|
||||
industry = "technology"
|
||||
)
|
||||
|
||||
user_polki = CoreUserInfoModel(
|
||||
fullName = "Polki Muchhwaali",
|
||||
userId = 6,
|
||||
entityId = 7,
|
||||
billingAccountId = 8,
|
||||
departmentId = 9,
|
||||
branchId = 10,
|
||||
# industry = "finance"
|
||||
)
|
||||
|
||||
file_info = await my_fs.info(
|
||||
mongo_conn = files_mongo,
|
||||
file_object_id = "67598d48c1bf89b25695f20b"
|
||||
)
|
||||
print("SUCCESS:", file_info.success)
|
||||
print("MESSAGE:", file_info.message)
|
||||
print("FILE INFO:", json.to_string(file_info.result.model_dump(), default = str))
|
||||
is_owner = my_fs.has_permission(
|
||||
mongo_conn = files_mongo,
|
||||
user_info = user_polki,
|
||||
file_object_info = file_info.result,
|
||||
permission = "write"
|
||||
)
|
||||
print("HAS PERMISSION:", is_owner.model_dump_json(indent = 4))
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -1,314 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
ORIGINAL: Monday, 2nd Dec., 2024
|
||||
UPGRADED: Monday, 9th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To define the interaction between the UI layer and the database connectivity in one place. Here we shall handle
|
||||
all the activities for OAuth2.0 authorization requests for all the users of our service.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.core.auth_token import CoreAuthTokenModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class MailOAuthModel(BaseModel):
|
||||
|
||||
AUTH_COLLECTION = "_authTokens"
|
||||
|
||||
async def get_token_id(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
mongo_conn: AsyncMongo,
|
||||
auth_token: CoreAuthTokenModel,
|
||||
session_token: str = None
|
||||
) -> ObjectId:
|
||||
|
||||
"""
|
||||
Stores params from the session info and gives an identifier to use in the authorization URL. Use this when the
|
||||
user requests an authorization URL to link your service to another service (like GMail).
|
||||
:param db_conn: The database connection (MariaDB) to use to perform the action.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param auth_token: An instance of the core auth-token model that holds data in the database.
|
||||
:param session_token: The session token of the user who requested this service.
|
||||
:return: An ObjectId to later store the granted tokens.
|
||||
"""
|
||||
|
||||
# Note down the timestamp at which this event occurred:
|
||||
request_ts = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
# Get the identifier from the database.
|
||||
# BE CAREFUL WITH THE KEYS HERE, THEY SHOULD MATCH THE FIELDS OF THE CORE AUTH-TOKEN MODEL:
|
||||
mongo_json = await mongo_conn.find_one_and_update(
|
||||
collection = MailOAuthModel.AUTH_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"serviceType": auth_token.serviceType,
|
||||
"user": {
|
||||
"entityId": auth_token.user.entityId,
|
||||
"billingAccountId": auth_token.user.billingAccountId
|
||||
},
|
||||
"clientUserId": auth_token.clientUserId
|
||||
}),
|
||||
update = {
|
||||
"$set": {
|
||||
"lastRequestTs": auth_token.lastRequestTs,
|
||||
"status": auth_token.status,
|
||||
"syncFreq": auth_token.syncFreq
|
||||
},
|
||||
"$setOnInsert": {
|
||||
"serviceType": auth_token.serviceType,
|
||||
"client": auth_token.client,
|
||||
"authType": auth_token.authType,
|
||||
"user": auth_token.user.model_dump(),
|
||||
"clientUserId": auth_token.clientUserId,
|
||||
"auth": auth_token.auth,
|
||||
"token": auth_token.token,
|
||||
"firstRefreshTs": auth_token.firstRefreshTs,
|
||||
"lastRefreshTs": auth_token.lastRefreshTs,
|
||||
"firstRequestTs": auth_token.firstRequestTs or request_ts,
|
||||
}
|
||||
},
|
||||
projection = {
|
||||
"_id": True
|
||||
},
|
||||
upsert = True,
|
||||
return_updated = True
|
||||
)
|
||||
|
||||
# Tell MariaDB that an authorization request was initiated:
|
||||
db_json = {}
|
||||
if mongo_json is not None:
|
||||
db_json = await self.call_procedure(
|
||||
db_conn = db_conn,
|
||||
proc_name = "entity_integration_save",
|
||||
proc_args = (
|
||||
auth_token.user.entityId, # ......................................... 'p_entity_id'
|
||||
auth_token.client, # ................................................ 'p_provider'
|
||||
auth_token.status, # ................................................ 'p_current_status'
|
||||
"Auth Requested", # ................................................. 'p_last_action'
|
||||
None, # ............................................................. 'p_display_name'
|
||||
None, # ............................................................. 'p_display_picture'
|
||||
str(mongo_json["_id"]), # ........................................... 'p_token_id'
|
||||
json.to_string(python_data = {"email": None}, no_space = True), # ... 'p_notes'
|
||||
auth_token.user.userId # ............................................ 'p_created_by'
|
||||
),
|
||||
session_token = session_token
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
|
||||
|
||||
async def set_token(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId | str,
|
||||
auth_token: CoreAuthTokenModel,
|
||||
session_token: str = None
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
This method is to be called when the end user authorizes your service to connect to his third-party account. For
|
||||
example, when the end user allows you to access his GMail account. USE THIS FOR UPDATING (REFRESHING) TOKENS
|
||||
ALSO.
|
||||
:param db_conn: The database connection (MariaDB) to use to perform the action.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param token_id: The identifier granted by the 'get_token_id' method.
|
||||
:param auth_token: The actual auth/token data to be saved to the database.
|
||||
:param session_token: The session token of the user who requested this service.
|
||||
:return: True if saved, False if failed.
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
token_saved = False
|
||||
|
||||
# Note down the timestamp at which this event occurred:
|
||||
request_ts = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
# Save the token to MongoDB.
|
||||
# BE CAREFUL WITH THE KEYS HERE, THEY SHOULD MATCH THE FIELDS OF THE CORE AUTH-TOKEN MODEL:
|
||||
mongo_json = await mongo_conn.find_one_and_update(
|
||||
collection = MailOAuthModel.AUTH_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"_id": ObjectId(token_id),
|
||||
"clientUserId": auth_token.clientUserId
|
||||
}),
|
||||
update = [{
|
||||
"$set": {
|
||||
"token": auth_token.token,
|
||||
"status": auth_token.status,
|
||||
"lastRefreshTs": request_ts,
|
||||
"firstRefreshTs": {
|
||||
"$cond": {
|
||||
"if": {
|
||||
"$or": [
|
||||
{"$eq": ["$firstRefreshTs", None]},
|
||||
{"$eq": [{"$type": "$firstRefreshTs"}, "missing"]}
|
||||
]
|
||||
},
|
||||
"then": request_ts,
|
||||
"else": "$firstRefreshTs"
|
||||
}
|
||||
}
|
||||
}
|
||||
}],
|
||||
projection = {"token": False},
|
||||
return_updated = True,
|
||||
upsert = False
|
||||
)
|
||||
|
||||
# Tell MariaDB that the token was saved:
|
||||
if mongo_json is not None:
|
||||
token_notes = {
|
||||
"email": auth_token.token["email"],
|
||||
"displayName": auth_token.token.get("displayName"),
|
||||
"displayPictureUrl": auth_token.token.get("displayPictureUrl"),
|
||||
}
|
||||
db_json = await self.call_procedure(
|
||||
db_conn = db_conn,
|
||||
proc_name = "entity_integration_save",
|
||||
proc_args = (
|
||||
mongo_json["user"]["entityId"], # ............................... 'p_entity_id'
|
||||
mongo_json["client"], # ......................................... 'p_provider'
|
||||
auth_token.status, # ............................................ 'p_current_status'
|
||||
"Auth Granted", # ............................................... 'p_last_action'
|
||||
auth_token.token.get("displayName"), # .......................... 'p_display_name'
|
||||
auth_token.token.get("displayPictureUrl"), # .................... 'p_display_picture'
|
||||
token_id, # ..................................................... 'p_token_id'
|
||||
json.to_string(python_data = token_notes, no_space = True), # ... 'p_notes'
|
||||
auth_token.user.userId # ........................................ 'p_created_by'
|
||||
),
|
||||
session_token = session_token
|
||||
)
|
||||
if db_json["status"] == 1: token_saved = True
|
||||
|
||||
# Done here:
|
||||
return token_saved
|
||||
|
||||
async def get_token(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId | str = None,
|
||||
**kwargs
|
||||
) -> CoreAuthTokenModel | None:
|
||||
|
||||
"""
|
||||
To retrieve stored tokens from the database.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param token_id: The identifier granted by the 'get_token_id' method.
|
||||
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
|
||||
info, the client, the type of authentication used, or even the kind of service.
|
||||
:return: The retrieved record that has the token, and information about the service and client if found, else
|
||||
None when there is no matching record.
|
||||
"""
|
||||
|
||||
# Build the filter:
|
||||
filter_json = {k: v for k, v in kwargs.items()}
|
||||
if token_id: filter_json["_id"] = ObjectId(token_id)
|
||||
|
||||
# If there is no search criteria, we exit with failure:
|
||||
if not filter_json: return None
|
||||
|
||||
# If there is some filtering possible, we fetch the token:
|
||||
token = await mongo_conn.find_one(
|
||||
collection = self.AUTH_COLLECTION,
|
||||
filter = filter_json,
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return CoreAuthTokenModel(**token) if token else None
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,215 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Tuesday, 3rd Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To enlist and retrieve mails for various filtering conditions.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal, List
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class MailRetrieveModel(BaseModel):
|
||||
|
||||
# For MongoDB:
|
||||
AUTH_COLLECTION = "_authTokens"
|
||||
MAIL_COLLECTION = "_messages"
|
||||
|
||||
async def get_mail(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
mail_id: str | ObjectId
|
||||
):
|
||||
|
||||
"""
|
||||
Retrieves one full mail from the database.
|
||||
:param mongo_conn: The instance of the database connector to use to get the mail's data.
|
||||
:param mail_id: The '_id' of the document that holds the mail.
|
||||
:return: Either the JSON that describes the mail or None if such a mail does not exist.
|
||||
"""
|
||||
|
||||
# Get the data from the database:
|
||||
mail_data = await mongo_conn.find_one(
|
||||
collection = self.MAIL_COLLECTION,
|
||||
filter = {"_id": ObjectId(mail_id)},
|
||||
projection = {
|
||||
"_id": True,
|
||||
"serviceType": True,
|
||||
"client": True,
|
||||
"ts": True,
|
||||
"readTs": True,
|
||||
"payload.ts": True,
|
||||
"payload.readTs": True,
|
||||
"payload.from": True,
|
||||
"payload.to": True,
|
||||
"payload.cc": True,
|
||||
"payload.bcc": True,
|
||||
"payload.parts": True,
|
||||
"payload.attachments": True,
|
||||
"payload.labels": True,
|
||||
"payload.snippet": True,
|
||||
"payload.aiSnippet": True,
|
||||
}
|
||||
)
|
||||
|
||||
# Format the data:
|
||||
if mail_data:
|
||||
mail_data["mailId"] = str(mail_data.pop("_id"))
|
||||
# mail_data["payload"]["ts"] = mail_data["payload"]["ts"].isoformat()
|
||||
# mail_data["payload"]["readTs"] = mail_data["payload"]["readTs"].isoformat()
|
||||
mail_data["ts"] = mail_data["ts"].isoformat()
|
||||
mail_data["readTs"] = mail_data["readTs"].isoformat()
|
||||
|
||||
# Done here:
|
||||
return mail_data
|
||||
|
||||
async def list_for_token_id(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: str | ObjectId | List[str | ObjectId],
|
||||
limit: int = 25,
|
||||
skip: int = 0
|
||||
):
|
||||
|
||||
"""
|
||||
To enlist mails for one account.
|
||||
:param mongo_conn: The instance of the database connector to use to get the mail's data.
|
||||
:param token_id: The id(s) of the document in the database that holds the tokens to access the account.
|
||||
:param limit: How many records to fetch.
|
||||
:param skip: How many initial records to skip. useful for pagination.
|
||||
:return: Either the JSON that describes the mails or None if something failed.
|
||||
"""
|
||||
|
||||
# Ensure that the token ids are in expected format:
|
||||
if not isinstance(token_id, list): token_id = [token_id]
|
||||
token_id = [ObjectId(t) for t in token_id]
|
||||
|
||||
# Get the data from the database:
|
||||
mails_list = await mongo_conn.find_many(
|
||||
collection = self.MAIL_COLLECTION,
|
||||
filter = {
|
||||
"tokenId": {"$in": token_id},
|
||||
"serviceType": "email"
|
||||
},
|
||||
projection = {
|
||||
"_id": True,
|
||||
"serviceType": True,
|
||||
"client": True,
|
||||
"ts": True,
|
||||
"readTs": True,
|
||||
"payload.ts": True,
|
||||
"payload.readTs": True,
|
||||
"payload.from": True,
|
||||
"payload.subject": True,
|
||||
"payload.labels": True,
|
||||
"payload.snippet": True,
|
||||
"payload.aiSnippet": True,
|
||||
},
|
||||
limit = limit,
|
||||
skip = skip,
|
||||
sort = {"payload.ts": -1}
|
||||
)
|
||||
|
||||
# Format the data:
|
||||
if mails_list:
|
||||
for mail_data in mails_list:
|
||||
mail_data["mailId"] = str(mail_data.pop("_id"))
|
||||
# mail_data["payload"]["ts"] = mail_data["payload"]["ts"].isoformat()
|
||||
# mail_data["payload"]["readTs"] = mail_data["payload"]["readTs"].isoformat()
|
||||
mail_data["ts"] = mail_data["ts"].isoformat()
|
||||
mail_data["readTs"] = mail_data["readTs"].isoformat()
|
||||
|
||||
# Done here:
|
||||
return mails_list
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,593 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
tuesday, 3rd Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
From here we sync all mails between the mail client's server and TheCAOffice's database.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For Quart:
|
||||
from quart import current_app
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
|
||||
|
||||
# Mail Clients:
|
||||
from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
|
||||
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.api.mail.sync import MailSyncOneResult, MailSyncManyResults
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
from pymongo import InsertOne, UpdateOne, ReplaceOne
|
||||
|
||||
# To work with LLMs:
|
||||
from models.behaviour.ai.llm.open_ai import LLMOpenAI
|
||||
from models.data.api.ai.llm import LLMInput
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal, List, Dict, Any
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
# To work with base-64 encoding:
|
||||
import base64
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class MailSyncModel(BaseModel):
|
||||
|
||||
# For MongoDB:
|
||||
AUTH_COLLECTION = "_authTokens"
|
||||
MAIL_COLLECTION = "_messages"
|
||||
|
||||
# For AI Magic through LLMs:
|
||||
PROMPT_TEMPLATE = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You're a mail summary expert that summarizes mails in 150 chars or less. "
|
||||
"If available, show login info like username and OTPs in your summary."
|
||||
"If no login info is provided, please don't worry; just summarize what you see."
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
# ┏┓ ┓
|
||||
# ┣┫╋╋┏┓┏┣┓┏┳┓┏┓┏┓╋┏
|
||||
# ┛┗┗┗┗┻┗┛┗┛┗┗┗ ┛┗┗┛
|
||||
|
||||
@staticmethod
|
||||
async def __save_one_attachment(
|
||||
session_token: str,
|
||||
attachment: Dict[str, Any],
|
||||
attachment_tags: List[str],
|
||||
attachment_metadata: dict,
|
||||
retry_count: int = 1,
|
||||
retry_delay: int = 1,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
"""
|
||||
Saves one attachment and generates a URL that can be later used to retrieve it.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param attachment: The JSON that describes the attachment.
|
||||
:param attachment_tags: Any tags to put on the file for easy search later.
|
||||
:param attachment_metadata: Any metadata to put on the file for easy search later.
|
||||
:param retry_count: How many max. retries to do in case of failure.
|
||||
:param retry_delay: The interval between the delays.
|
||||
:param backoff_multiplier: By what rate the delay between 2 attempts must change.
|
||||
:return: The JSON that describes the same attachment, except that the payload's data is replaced by the id and
|
||||
url of where to find the attachment.
|
||||
"""
|
||||
|
||||
# Make a deep-copy of the attachment JSON,
|
||||
# and process the payload in advance:
|
||||
attachment_copy = copy.deepcopy(attachment)
|
||||
attachment_payload = attachment_copy.pop("payload").encode()
|
||||
if attachment_copy.pop("contentTransferEncoding", "?").strip().lower() == "base64":
|
||||
attachment_payload = base64.b64decode(attachment_payload)
|
||||
|
||||
# Start by assuming failure,
|
||||
# and retry as many times as asked:
|
||||
attachment_copy["id"] = None
|
||||
attachment_copy["url"] = None
|
||||
for _ in range(retry_count):
|
||||
|
||||
# Make the upload:
|
||||
api_response = await current_app.http_client.post(
|
||||
url = current_app.script_data["fileUpload"]["url"],
|
||||
headers = {
|
||||
"X-Session-Token": session_token,
|
||||
"X-File-Name": attachment["filename"],
|
||||
"X-File-Private": "false",
|
||||
"X-File-Tags": json.to_string(attachment_tags, no_space = True),
|
||||
"X-File-Metadata": json.to_string(attachment_metadata, no_space = True)
|
||||
},
|
||||
data = attachment_payload
|
||||
)
|
||||
|
||||
# If the upload was successful:
|
||||
if api_response.status_code in [200]:
|
||||
api_data = api_response.json()["data"]
|
||||
attachment_copy["id"] = api_data["id"]
|
||||
attachment_copy["url"] = api_data["url"]
|
||||
break
|
||||
|
||||
# Done here:
|
||||
return attachment_copy
|
||||
|
||||
async def __save_many_attachments(
|
||||
self,
|
||||
session_token: str,
|
||||
attachments: List[Dict[str, Any]],
|
||||
attachment_tags: List[str],
|
||||
attachment_metadata: dict,
|
||||
retry_count: int = 1,
|
||||
retry_delay:int = 1,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
"""
|
||||
Saves all the attachments received in the mail (whether inline or otherwise) and makes them available through
|
||||
simple download URLs.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param attachments: The JSON that describes the attachments.
|
||||
:param attachment_tags: Any tags to put on the file for easy search later.
|
||||
:param attachment_metadata: Any metadata to put on the file for easy search later.
|
||||
:param retry_count: How many max. retries to do in case of failure.
|
||||
:param retry_delay: The interval between the delays.
|
||||
:param backoff_multiplier: By what rate the delay between 2 attempts must change.
|
||||
:return: The JSON that describes the same attachments, except that the payload's data is replaced by the id and
|
||||
url of where to find each attachment.
|
||||
"""
|
||||
|
||||
# Create and fire all the tasks
|
||||
# needed to save the files:
|
||||
tasks = [
|
||||
self.__save_one_attachment(
|
||||
session_token = session_token,
|
||||
attachment = attachment,
|
||||
attachment_tags = attachment_tags,
|
||||
attachment_metadata = attachment_metadata,
|
||||
retry_count = retry_count,
|
||||
retry_delay = retry_delay,
|
||||
backoff_multiplier = backoff_multiplier
|
||||
) for attachment in attachments
|
||||
]
|
||||
uploaded_attachments = await asyncio.gather(*tasks)
|
||||
|
||||
# Done here:
|
||||
return uploaded_attachments
|
||||
|
||||
# ┏┓ ┏┓┳┳┓ •┓
|
||||
# ┣ ┏┓┏┓ ┃┓┃┃┃┏┓┓┃
|
||||
# ┻ ┗┛┛ ┗┛┛ ┗┗┻┗┗
|
||||
|
||||
async def __sync_one_gmail(
|
||||
self,
|
||||
session_token: str,
|
||||
user_info: dict,
|
||||
mongo_conn: AsyncMongo,
|
||||
mail_client: AsyncGMailClient,
|
||||
tokens: GoogleAuthTokens,
|
||||
message_id: str,
|
||||
llm: LLMOpenAI = None,
|
||||
force_sync: bool = False
|
||||
) -> MailSyncOneResult:
|
||||
|
||||
"""
|
||||
Sync on mail from GMail.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param user_info: The information of the user (derived from his session token).
|
||||
:param mongo_conn: The instance of the connection to the database to use.
|
||||
:param mail_client: The instance of the mail client to use to perform the action.
|
||||
:param tokens: The tokens to use to fetch the mails.
|
||||
:param message_id: The id that Google uses to identify this mail. This will be received in the 'list_messages'
|
||||
method.
|
||||
:param llm: The instance of the LLM to use to summarize the mail's content.
|
||||
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
|
||||
database.
|
||||
:return:
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
sync_result = MailSyncOneResult()
|
||||
|
||||
# If we've not been forced to re-sync the mail message,
|
||||
# we first check if the mail already exists in our database:
|
||||
if not force_sync:
|
||||
mail_record = await mongo_conn.find_one(
|
||||
collection = self.MAIL_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"payload": {
|
||||
"messageId": message_id
|
||||
},
|
||||
"user_info": {
|
||||
"entityId": user_info["entityId"],
|
||||
"billingAccountId": user_info["billingAccountId"]
|
||||
}
|
||||
}),
|
||||
projection = {
|
||||
"_id": False,
|
||||
"readTs": "payload.readTs"
|
||||
}
|
||||
)
|
||||
if mail_record:
|
||||
sync_result.success = True
|
||||
sync_result.message = f"gmail message '{message_id}' already sync'd on '{mail_record['readTs']} (UTC)'"
|
||||
return sync_result
|
||||
|
||||
# Now that we know that we have to fetch the mail from GMail:
|
||||
client_response = await mail_client.get_message(
|
||||
tokens = tokens,
|
||||
message_id = message_id,
|
||||
return_raw = False
|
||||
)
|
||||
|
||||
# If we didn't get the mail from GMail;
|
||||
if not client_response.success:
|
||||
sync_result.message = f"gmail (messageId: '{message_id}'): {client_response.message}"
|
||||
return sync_result
|
||||
|
||||
# We upload the attachments:
|
||||
client_response.data["attachments"] = await self.__save_many_attachments(
|
||||
session_token = session_token,
|
||||
attachments = client_response.data["attachments"],
|
||||
attachment_tags = [
|
||||
"email",
|
||||
"gmail",
|
||||
client_response.data["from"][0]["name"],
|
||||
client_response.data["from"][0]["email"],
|
||||
tokens.email,
|
||||
],
|
||||
attachment_metadata = {
|
||||
"project": "tcaoff",
|
||||
"serviceType": "email",
|
||||
"client": "gmail",
|
||||
"from": client_response.data["from"][0]["email"],
|
||||
"to": tokens.email
|
||||
},
|
||||
retry_count = 3
|
||||
)
|
||||
|
||||
# Give a quick indicator of whether this mail is an inbox mail or sent mail:
|
||||
all_recipients = []
|
||||
for field in ["to", "cc", "bcc"]: all_recipients += [item["email"] for item in client_response.data[field]]
|
||||
if tokens.email in all_recipients: client_response.data["isInbox"] = True
|
||||
else: client_response.data["isInbox"] = False
|
||||
|
||||
# If an LLM is given,
|
||||
# we add an AI summary:
|
||||
llm_json = None
|
||||
if llm:
|
||||
|
||||
# Invoke the LLM:
|
||||
llm_response = response = await llm.invoke(
|
||||
mongo_conn = mongo_conn,
|
||||
user_info = user_info,
|
||||
llm_input = LLMInput(
|
||||
messages = self.PROMPT_TEMPLATE + [
|
||||
{
|
||||
"role": "human",
|
||||
"content": f"Please summarize this mail: \"\"\"{client_response.data['unformattedText']}\"\"\""
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# Format the response:
|
||||
llm_json = {
|
||||
"ts": llm_response.ts,
|
||||
"snippet": llm_response.output,
|
||||
"tokens": llm_response.tokens.model_dump()
|
||||
}
|
||||
|
||||
# Add the LLM's response to the main data:
|
||||
client_response.data["aiSnippet"] = llm_json
|
||||
|
||||
# Done here:
|
||||
sync_result.success = True
|
||||
sync_result.mailMessage = client_response.data
|
||||
return sync_result
|
||||
|
||||
async def __sync_many_gmail(
|
||||
self,
|
||||
session_token: str,
|
||||
user_info: dict,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId,
|
||||
mail_client: AsyncGMailClient,
|
||||
tokens: GoogleAuthTokens,
|
||||
llm: LLMOpenAI = None,
|
||||
force_sync: bool = False,
|
||||
start_date: datetime.datetime = None,
|
||||
end_date: datetime.datetime = None,
|
||||
max_count: int = 100
|
||||
) -> MailSyncManyResults:
|
||||
|
||||
"""
|
||||
Sync many mails from GMail in one shot.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param user_info: The information of the user (derived from his session token).
|
||||
:param mongo_conn: The instance of the connection to the database to use.
|
||||
:param token_id: The id of the document in the database that holds the tokens to access the account.
|
||||
Needed only for refreshing the tokens and saving them.
|
||||
:param mail_client: The instance of the mail client to use to perform the action.
|
||||
:param tokens: The tokens to use to fetch the mails.
|
||||
:param llm: The instance of the LLM to use to summarize the mail's content.
|
||||
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
|
||||
database.
|
||||
:param start_date: The starting date (inclusive) from when to sync the mails.
|
||||
:param end_date: The ending date (inclusive) from when to sync the mails.
|
||||
:param max_count: The max. no. of mails to sync.
|
||||
:return: The result of the sync'ing.
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
sync_results = MailSyncManyResults()
|
||||
|
||||
# Refresh the tokens (if needed):
|
||||
tokens_refreshed = await tokens.arefresh(
|
||||
http_client = current_app.http_client,
|
||||
client_id = mail_client.client_id,
|
||||
client_secret = mail_client.client_secret
|
||||
)
|
||||
if tokens_refreshed: await current_app.mail_oauth_model.set_token(
|
||||
db_conn = current_app.sql_writer,
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
client_user_id = tokens.client_user_id,
|
||||
token = tokens,
|
||||
session_token = session_token
|
||||
)
|
||||
|
||||
# Let's build the query:
|
||||
sub_queries = []
|
||||
if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
|
||||
if end_date: sub_queries.append((end_date + datetime.timedelta(days = 1)).strftime("before:%Y/%m/%d"))
|
||||
query_string = " ".join(sub_queries)
|
||||
|
||||
# Let's enlist all the mails that fall in the date range:
|
||||
client_response = await mail_client.list_messages(
|
||||
tokens = tokens,
|
||||
max_count = max_count,
|
||||
query = query_string
|
||||
)
|
||||
if not client_response.success:
|
||||
sync_results["message"] = f"gmail: {client_response.message}"
|
||||
return sync_results
|
||||
messages_list = client_response.data["messages"]
|
||||
|
||||
# Now, for every mail in the list, we fetch the mail and note the results:
|
||||
tasks = [
|
||||
self.__sync_one_gmail(
|
||||
session_token = session_token,
|
||||
user_info = user_info,
|
||||
mongo_conn = mongo_conn,
|
||||
mail_client = mail_client,
|
||||
tokens = tokens,
|
||||
message_id = v["id"],
|
||||
llm = llm,
|
||||
force_sync = force_sync
|
||||
) for v in messages_list.values()
|
||||
]
|
||||
individual_sync_results = await asyncio.gather(*tasks)
|
||||
|
||||
# Now we create operations for each mail,
|
||||
# and maintain success/failure counters:
|
||||
sync_results.totalCount = len(individual_sync_results)
|
||||
mongo_operations = []
|
||||
for result in individual_sync_results:
|
||||
if result.success: sync_results.successCount += 1
|
||||
else: sync_results.failureCount += 1
|
||||
if result.mailMessage: mongo_operations.append(ReplaceOne(
|
||||
filter = {
|
||||
"serviceType": "email",
|
||||
"$or": [
|
||||
{
|
||||
"client": "gmail",
|
||||
"payload.messageId": result.mailMessage["messageId"]
|
||||
}
|
||||
]
|
||||
},
|
||||
replacement = {
|
||||
"version": "1.0.0",
|
||||
"tokenId": ObjectId(token_id),
|
||||
"serviceType": "email",
|
||||
"client": "gmail",
|
||||
"payload": result.mailMessage
|
||||
},
|
||||
upsert = True
|
||||
))
|
||||
|
||||
# Make the bulk write:
|
||||
if mongo_operations:
|
||||
mongo_count = await mongo_conn.bulk_write(
|
||||
collection = self.MAIL_COLLECTION,
|
||||
requests = mongo_operations
|
||||
)
|
||||
|
||||
# Apply the labels to the read messages:
|
||||
try:
|
||||
client_response = await mail_client.modify_messages(
|
||||
tokens = tokens,
|
||||
message_ids = [v["id"] for v in messages_list.values()],
|
||||
add_label_ids = [tokens.labels.get("TCAOFF", {}).get("id")]
|
||||
)
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
|
||||
# Done here:
|
||||
sync_results.message = f"{sync_results.successCount}/{sync_results.totalCount} mail(s) sync'd from gmail"
|
||||
return sync_results
|
||||
|
||||
# ┳┓
|
||||
# ┣┫┏┓┓┏╋┏┓┏┓
|
||||
# ┛┗┗┛┗┻┗┗ ┛
|
||||
|
||||
async def sync(
|
||||
self,
|
||||
session_token: str,
|
||||
user_info: dict,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId,
|
||||
llm: LLMOpenAI = None,
|
||||
force_sync: bool = False,
|
||||
start_date: datetime.datetime = None,
|
||||
end_date: datetime.datetime = None,
|
||||
max_count: int = 100
|
||||
) -> MailSyncManyResults:
|
||||
|
||||
"""
|
||||
Sync many mails at once from many types of clients. Use this as a common entry point after which you internally
|
||||
route the request to the appropriate clients.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param user_info: The information of the user (derived from his session token).
|
||||
:param mongo_conn: The instance of the connection to the database to use.
|
||||
:param token_id: The id of the document in the database that holds the tokens to access the account.
|
||||
Needed only for refreshing the tokens and saving them.
|
||||
:param llm: The instance of the LLM to use to summarize the mail's content.
|
||||
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
|
||||
database.
|
||||
:param start_date: The starting date (inclusive) from when to sync the mails.
|
||||
:param end_date: The ending date (inclusive) from when to sync the mails.
|
||||
:param max_count: The max. no. of mails to sync.
|
||||
:return: The result of the sync'ing.
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
sync_results = MailSyncManyResults()
|
||||
|
||||
# ┏┓ ┓ ┏┳┓ ┓
|
||||
# ┣ ┏┓╋┏┣┓ ┃ ┏┓┃┏┏┓┏┓┏
|
||||
# ┻ ┗ ┗┗┛┗ ┻ ┗┛┛┗┗ ┛┗┛
|
||||
|
||||
# We first load the authorization tokens:
|
||||
auth_json = await current_app.mail_oauth_model.get_token(
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
)
|
||||
|
||||
# If we failed to load the authorization tokens:
|
||||
if not auth_json:
|
||||
sync_results.message = f"no such token id '{token_id}'"
|
||||
return sync_results
|
||||
|
||||
# ┏┓ ┏┓┳┳┓ •┓
|
||||
# ┣ ┏┓┏┓ ┃┓┃┃┃┏┓┓┃
|
||||
# ┻ ┗┛┛ ┗┛┛ ┗┗┻┗┗
|
||||
|
||||
if auth_json["client"] == "gmail":
|
||||
return await self.__sync_many_gmail(
|
||||
session_token = session_token,
|
||||
user_info = user_info,
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
mail_client = current_app.gmail_client,
|
||||
tokens = GoogleAuthTokens(**auth_json["token"]),
|
||||
llm = llm,
|
||||
force_sync = force_sync,
|
||||
start_date = start_date,
|
||||
end_date = end_date,
|
||||
max_count = max_count
|
||||
)
|
||||
|
||||
# ┳ ┓• ┓ ┏┓┓•
|
||||
# ┃┏┓┓┏┏┓┃┓┏┫ ┃ ┃┓┏┓┏┓╋
|
||||
# ┻┛┗┗┛┗┻┗┗┗┻ ┗┛┗┗┗ ┛┗┗
|
||||
|
||||
# If we haven't been able to sync mail due to not entering any 'if' condition:
|
||||
sync_results.message = f"no such mail client '{auth_json['client']}'"
|
||||
return sync_results
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,614 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
ORIGINAL: Tuesday, 3rd Dec., 2024
|
||||
UPGRADED: Monday, 9th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
From here we sync all mails between the mail client's server and our internal database.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
|
||||
from models.data.core.auth_token import CoreAuthTokenModel
|
||||
from models.data.core.message import CoreMessageModel
|
||||
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For Quart:
|
||||
from quart import current_app
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
|
||||
|
||||
# Mail Clients:
|
||||
from utils_v2.goog.gmail.gmail_client import AsyncGMailClient
|
||||
from utils_v2.goog.models.data.auth_tokens import GoogleAuthTokens
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.api.mail.sync import MailSyncOneResult, MailSyncManyResults
|
||||
from models.data.core.user import CoreUserInfoModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
from pymongo import InsertOne, UpdateOne, ReplaceOne
|
||||
|
||||
# To work with LLMs:
|
||||
from models.behaviour.ai.llm.open_ai import LLMOpenAI
|
||||
from models.data.api.ai.llm import LLMInput
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal, List, Dict, Any
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
# To work with base-64 encoding:
|
||||
import base64
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class MailSyncModel(BaseModel):
|
||||
|
||||
# For MongoDB:
|
||||
AUTH_COLLECTION = "_authTokens"
|
||||
MAIL_COLLECTION = "_messages"
|
||||
|
||||
# For AI Magic through LLMs:
|
||||
PROMPT_TEMPLATE = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You're a mail summary expert that summarizes mails in 150 chars or less. "
|
||||
"If available, show login info like username and OTPs in your summary."
|
||||
"If no login info is provided, please don't worry; just summarize what you see."
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
# ┏┓ ┓
|
||||
# ┣┫╋╋┏┓┏┣┓┏┳┓┏┓┏┓╋┏
|
||||
# ┛┗┗┗┗┻┗┛┗┛┗┗┗ ┛┗┗┛
|
||||
|
||||
@staticmethod
|
||||
async def __save_one_attachment(
|
||||
session_token: str,
|
||||
attachment: Dict[str, Any],
|
||||
attachment_tags: List[str],
|
||||
attachment_metadata: dict,
|
||||
retry_count: int = 1,
|
||||
retry_delay: int = 1,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
"""
|
||||
Saves one attachment and generates a URL that can be later used to retrieve it.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param attachment: The JSON that describes the attachment.
|
||||
:param attachment_tags: Any tags to put on the file for easy search later.
|
||||
:param attachment_metadata: Any metadata to put on the file for easy search later.
|
||||
:param retry_count: How many max. retries to do in case of failure.
|
||||
:param retry_delay: The interval between the delays.
|
||||
:param backoff_multiplier: By what rate the delay between 2 attempts must change.
|
||||
:return: The JSON that describes the same attachment, except that the payload's data is replaced by the id and
|
||||
url of where to find the attachment.
|
||||
"""
|
||||
|
||||
# Make a deep-copy of the attachment JSON,
|
||||
# and process the payload in advance:
|
||||
attachment_copy = copy.deepcopy(attachment)
|
||||
attachment_payload = attachment_copy.pop("payload").encode()
|
||||
if attachment_copy.pop("contentTransferEncoding", "?").strip().lower() == "base64":
|
||||
attachment_payload = base64.b64decode(attachment_payload)
|
||||
|
||||
# Start by assuming failure,
|
||||
# and retry as many times as asked:
|
||||
attachment_copy["id"] = None
|
||||
attachment_copy["url"] = None
|
||||
for _ in range(retry_count):
|
||||
|
||||
# Make the upload:
|
||||
api_response = await current_app.http_client.post(
|
||||
url = current_app.script_data["fileUpload"]["url"],
|
||||
headers = {
|
||||
"X-Session-Token": session_token,
|
||||
"X-File-Name": attachment["filename"],
|
||||
"X-File-Private": "false",
|
||||
"X-File-Tags": json.to_string(attachment_tags, no_space = True),
|
||||
"X-File-Metadata": json.to_string(attachment_metadata, no_space = True)
|
||||
},
|
||||
data = attachment_payload
|
||||
)
|
||||
|
||||
# If the upload was successful:
|
||||
if api_response.status_code in [200]:
|
||||
api_data = api_response.json()["data"]
|
||||
attachment_copy["id"] = api_data["id"]
|
||||
attachment_copy["url"] = api_data["url"]
|
||||
break
|
||||
|
||||
# If the upload failed:
|
||||
await asyncio.sleep(retry_delay)
|
||||
retry_delay = retry_delay * backoff_multiplier
|
||||
|
||||
# Done here:
|
||||
return attachment_copy
|
||||
|
||||
async def __save_many_attachments(
|
||||
self,
|
||||
session_token: str,
|
||||
attachments: List[Dict[str, Any]],
|
||||
attachment_tags: List[str],
|
||||
attachment_metadata: dict,
|
||||
retry_count: int = 1,
|
||||
retry_delay:int = 1,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> List[Dict[str, Any]]:
|
||||
|
||||
"""
|
||||
Saves all the attachments received in the mail (whether inline or otherwise) and makes them available through
|
||||
simple download URLs.
|
||||
:param session_token: The session token of the uer who is trying to upload this file.
|
||||
:param attachments: The JSON that describes the attachments.
|
||||
:param attachment_tags: Any tags to put on the file for easy search later.
|
||||
:param attachment_metadata: Any metadata to put on the file for easy search later.
|
||||
:param retry_count: How many max. retries to do in case of failure.
|
||||
:param retry_delay: The interval between the delays.
|
||||
:param backoff_multiplier: By what rate the delay between 2 attempts must change.
|
||||
:return: The JSON that describes the same attachments, except that the payload's data is replaced by the id and
|
||||
url of where to find each attachment.
|
||||
"""
|
||||
|
||||
# Create and fire all the tasks
|
||||
# needed to save the files:
|
||||
tasks = [
|
||||
self.__save_one_attachment(
|
||||
session_token = session_token,
|
||||
attachment = attachment,
|
||||
attachment_tags = attachment_tags,
|
||||
attachment_metadata = attachment_metadata,
|
||||
retry_count = retry_count,
|
||||
retry_delay = retry_delay,
|
||||
backoff_multiplier = backoff_multiplier
|
||||
) for attachment in attachments
|
||||
]
|
||||
uploaded_attachments = await asyncio.gather(*tasks)
|
||||
|
||||
# Done here:
|
||||
return uploaded_attachments
|
||||
|
||||
# ┏┓ ┏┓┳┳┓ •┓
|
||||
# ┣ ┏┓┏┓ ┃┓┃┃┃┏┓┓┃
|
||||
# ┻ ┗┛┛ ┗┛┛ ┗┗┻┗┗
|
||||
|
||||
async def __sync_one_gmail(
|
||||
self,
|
||||
session_token: str,
|
||||
user_info: CoreUserInfoModel,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId,
|
||||
auth_token: CoreAuthTokenModel,
|
||||
mail_client: AsyncGMailClient,
|
||||
google_tokens: GoogleAuthTokens,
|
||||
message_id: str,
|
||||
llm: LLMOpenAI = None,
|
||||
force_sync: bool = False
|
||||
) -> MailSyncOneResult:
|
||||
|
||||
"""
|
||||
Sync on mail from GMail.
|
||||
:param mongo_conn: The instance of the connection to the database to use.
|
||||
:param mail_client: The instance of the mail client to use to perform the action.
|
||||
:param google_tokens: The tokens to use to fetch the mails.
|
||||
:param message_id: The id that Google uses to identify this mail. This will be received in the 'list_messages'
|
||||
method.
|
||||
:param llm: The instance of the LLM to use to summarize the mail's content.
|
||||
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
|
||||
database.
|
||||
:return:
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
sync_result = MailSyncOneResult()
|
||||
|
||||
# If we've not been forced to re-sync the mail message,
|
||||
# we first check if the mail already exists in our database:
|
||||
if not force_sync:
|
||||
mail_record = await mongo_conn.find_one(
|
||||
collection = self.MAIL_COLLECTION,
|
||||
filter = {
|
||||
"tokenId": ObjectId(token_id),
|
||||
"serviceType": auth_token.serviceType,
|
||||
"client": auth_token.client,
|
||||
"clientMessageId": message_id
|
||||
},
|
||||
projection = {
|
||||
"_id": False,
|
||||
"readTs": True
|
||||
},
|
||||
raise_exception = True
|
||||
)
|
||||
if mail_record:
|
||||
sync_result.success = True
|
||||
sync_result.message = f"gmail message '{message_id}' already sync'd on '{mail_record['readTs']} (UTC)'"
|
||||
return sync_result
|
||||
|
||||
# Now that we know that we have to fetch the mail from GMail:
|
||||
client_response = await mail_client.get_message(
|
||||
tokens = google_tokens,
|
||||
message_id = message_id,
|
||||
return_raw = False
|
||||
)
|
||||
|
||||
# If we didn't get the mail from GMail;
|
||||
if not client_response.success:
|
||||
sync_result.message = f"gmail (messageId: '{message_id}'): {client_response.message}"
|
||||
return sync_result
|
||||
|
||||
# We upload the attachments:
|
||||
client_response.data["attachments"] = await self.__save_many_attachments(
|
||||
session_token = session_token,
|
||||
attachments = client_response.data["attachments"],
|
||||
attachment_tags = [
|
||||
auth_token.serviceType,
|
||||
auth_token.client,
|
||||
client_response.data["from"][0]["name"],
|
||||
client_response.data["from"][0]["email"],
|
||||
google_tokens.email,
|
||||
],
|
||||
attachment_metadata = {
|
||||
"project": "tcaoff",
|
||||
"serviceType": auth_token.serviceType,
|
||||
"client": auth_token.client,
|
||||
"from": client_response.data["from"][0]["email"],
|
||||
"to": google_tokens.email
|
||||
},
|
||||
retry_count = 3
|
||||
)
|
||||
|
||||
# Give a quick indicator of whether this mail is an inbox mail or sent mail:
|
||||
all_recipients = []
|
||||
for field in ["to", "cc", "bcc"]: all_recipients += [item["email"] for item in client_response.data[field]]
|
||||
if google_tokens.email in all_recipients: client_response.data["isInbox"] = True
|
||||
else: client_response.data["isInbox"] = False
|
||||
|
||||
# If an LLM is given,
|
||||
# we add an AI summary:
|
||||
llm_json = None
|
||||
if llm:
|
||||
|
||||
# Invoke the LLM:
|
||||
llm_response = await llm.invoke(
|
||||
mongo_conn = mongo_conn,
|
||||
user_info = user_info,
|
||||
llm_input = LLMInput(
|
||||
messages = self.PROMPT_TEMPLATE + [
|
||||
{
|
||||
"role": "human",
|
||||
"content": (
|
||||
"Please summarize this mail: "
|
||||
f"\"\"\"{client_response.data['unformattedText']}\"\"\""
|
||||
)
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# Format the response:
|
||||
llm_json = {
|
||||
"ts": llm_response.ts,
|
||||
"snippet": llm_response.output,
|
||||
"tokens": llm_response.tokens.model_dump()
|
||||
}
|
||||
|
||||
# Add the LLM's response to the main data:
|
||||
client_response.data["aiSnippet"] = llm_json
|
||||
|
||||
# Fit the mail message into the model:
|
||||
sync_result.mailMessage = CoreMessageModel(
|
||||
ts = client_response.data["ts"],
|
||||
readTs = date_time.get_current_utc_date_time(as_string = False),
|
||||
tokenId = token_id,
|
||||
serviceType = auth_token.serviceType,
|
||||
client = auth_token.client,
|
||||
clientMessageId = message_id,
|
||||
clientThreadId = client_response.data["threadId"],
|
||||
payload = client_response.data
|
||||
)
|
||||
|
||||
# Done here:
|
||||
sync_result.success = True
|
||||
return sync_result
|
||||
|
||||
async def __sync_many_gmail(
|
||||
self,
|
||||
session_token: str,
|
||||
user_info: CoreUserInfoModel,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId,
|
||||
auth_token: CoreAuthTokenModel,
|
||||
mail_client: AsyncGMailClient,
|
||||
llm: LLMOpenAI = None,
|
||||
force_sync: bool = False,
|
||||
start_date: datetime.datetime = None,
|
||||
end_date: datetime.datetime = None,
|
||||
max_count: int = 100
|
||||
) -> MailSyncManyResults:
|
||||
|
||||
"""
|
||||
Sync many mails from GMail in one shot.
|
||||
:param mongo_conn: The instance of the connection to the database to use.
|
||||
:param token_id: The id of the document in the database that holds the tokens to access the account.
|
||||
Needed only for refreshing the tokens and saving them.
|
||||
:param mail_client: The instance of the mail client to use to perform the action.
|
||||
:param llm: The instance of the LLM to use to summarize the mail's content.
|
||||
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
|
||||
database.
|
||||
:param start_date: The starting date (inclusive) from when to sync the mails.
|
||||
:param end_date: The ending date (inclusive) from when to sync the mails.
|
||||
:param max_count: The max. no. of mails to sync.
|
||||
:return: The result of the sync'ing.
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
sync_results = MailSyncManyResults()
|
||||
|
||||
# Extract the client's tokens from the full token payload given by the database:
|
||||
google_tokens = GoogleAuthTokens(**auth_token.token)
|
||||
|
||||
# Refresh the tokens (if needed):
|
||||
tokens_refreshed = await google_tokens.arefresh(
|
||||
http_client = current_app.http_client,
|
||||
client_id = mail_client.client_id,
|
||||
client_secret = mail_client.client_secret
|
||||
)
|
||||
if tokens_refreshed:
|
||||
auth_token.token = google_tokens.model_dump()
|
||||
auth_token.lastRefreshTs = date_time.get_current_utc_date_time(as_string = True)
|
||||
await current_app.mail_oauth_model.set_token(
|
||||
db_conn = current_app.sql_writer,
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
auth_token = auth_token
|
||||
)
|
||||
|
||||
# Let's build the query:
|
||||
sub_queries = []
|
||||
if start_date: sub_queries.append(start_date.strftime("after:%Y/%m/%d"))
|
||||
if end_date: sub_queries.append((end_date + datetime.timedelta(days = 1)).strftime("before:%Y/%m/%d"))
|
||||
query_string = " ".join(sub_queries)
|
||||
|
||||
# Let's enlist all the mails that fall in the date range:
|
||||
client_response = await mail_client.list_messages(
|
||||
tokens = google_tokens,
|
||||
max_count = max_count,
|
||||
query = query_string
|
||||
)
|
||||
if not client_response.success:
|
||||
sync_results["message"] = f"gmail: {client_response.message}"
|
||||
return sync_results
|
||||
messages_list = client_response.data["messages"]
|
||||
|
||||
# Now, for every mail in the list, we fetch the mail and note the results:
|
||||
tasks = [
|
||||
self.__sync_one_gmail(
|
||||
session_token = session_token,
|
||||
user_info = user_info,
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
auth_token = auth_token,
|
||||
mail_client = mail_client,
|
||||
google_tokens = google_tokens,
|
||||
message_id = v["id"],
|
||||
llm = llm,
|
||||
force_sync = force_sync
|
||||
) for v in messages_list.values()
|
||||
]
|
||||
individual_sync_results = await asyncio.gather(*tasks)
|
||||
|
||||
# Now we create operations for each mail,
|
||||
# and maintain success/failure counters:
|
||||
sync_results.totalCount = len(individual_sync_results)
|
||||
mongo_operations = []
|
||||
for result in individual_sync_results:
|
||||
if result.success: sync_results.successCount += 1
|
||||
else: sync_results.failureCount += 1
|
||||
if result.mailMessage: mongo_operations.append(ReplaceOne(
|
||||
filter = {
|
||||
"tokenId": token_id,
|
||||
"serviceType": auth_token.serviceType,
|
||||
"client": auth_token.client,
|
||||
"clientMessageId": result.mailMessage.clientMessageId
|
||||
# "serviceType": auth_token.serviceType,
|
||||
# "$or": [
|
||||
# {
|
||||
# "client": auth_token.client,
|
||||
# "messageId": result.mailMessage.clientMessageId
|
||||
# }
|
||||
# ]
|
||||
},
|
||||
replacement = result.mailMessage.model_dump(),
|
||||
upsert = True
|
||||
))
|
||||
|
||||
# Make the bulk write:
|
||||
if mongo_operations:
|
||||
mongo_count = await mongo_conn.bulk_write(
|
||||
collection = self.MAIL_COLLECTION,
|
||||
requests = mongo_operations
|
||||
)
|
||||
|
||||
# Apply the labels to the read messages:
|
||||
try:
|
||||
client_response = await mail_client.modify_messages(
|
||||
tokens = google_tokens,
|
||||
message_ids = [v["id"] for v in messages_list.values()],
|
||||
add_label_ids = [google_tokens.labels.get("TCAOFF", {}).get("id")]
|
||||
)
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
|
||||
# Done here:
|
||||
sync_results.message = f"{sync_results.successCount}/{sync_results.totalCount} mail(s) sync'd from gmail"
|
||||
return sync_results
|
||||
|
||||
# ┳┓
|
||||
# ┣┫┏┓┓┏╋┏┓┏┓
|
||||
# ┛┗┗┛┗┻┗┗ ┛
|
||||
|
||||
async def sync(
|
||||
self,
|
||||
session_token: str,
|
||||
user_info: CoreUserInfoModel,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId,
|
||||
llm: LLMOpenAI = None,
|
||||
force_sync: bool = False,
|
||||
start_date: datetime.datetime = None,
|
||||
end_date: datetime.datetime = None,
|
||||
max_count: int = 100
|
||||
) -> MailSyncManyResults:
|
||||
|
||||
"""
|
||||
Sync many mails at once from many types of clients. Use this as a common entry point after which you internally
|
||||
route the request to the appropriate clients.
|
||||
:param mongo_conn: The instance of the connection to the database to use.
|
||||
:param token_id: The id of the document in the database that holds the tokens to access the account.
|
||||
Needed only for refreshing the tokens and saving them.
|
||||
:param llm: The instance of the LLM to use to summarize the mail's content.
|
||||
:param force_sync: Whether you would like to forcefully re-sync the mail even if it is already present in the
|
||||
database.
|
||||
:param start_date: The starting date (inclusive) from when to sync the mails.
|
||||
:param end_date: The ending date (inclusive) from when to sync the mails.
|
||||
:param max_count: The max. no. of mails to sync.
|
||||
:return: The result of the sync'ing.
|
||||
"""
|
||||
|
||||
# Start by assuming failure:
|
||||
sync_results = MailSyncManyResults()
|
||||
|
||||
# ┏┓ ┓ ┏┳┓ ┓
|
||||
# ┣ ┏┓╋┏┣┓ ┃ ┏┓┃┏┏┓┏┓┏
|
||||
# ┻ ┗ ┗┗┛┗ ┻ ┗┛┛┗┗ ┛┗┛
|
||||
|
||||
# We first load the authorization tokens:
|
||||
auth_token = await current_app.mail_oauth_model.get_token(
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
)
|
||||
|
||||
# If we failed to load the authorization tokens:
|
||||
if not auth_token:
|
||||
sync_results.message = f"no such token id '{token_id}'"
|
||||
return sync_results
|
||||
|
||||
# ┏┓ ┏┓┳┳┓ •┓
|
||||
# ┣ ┏┓┏┓ ┃┓┃┃┃┏┓┓┃
|
||||
# ┻ ┗┛┛ ┗┛┛ ┗┗┻┗┗
|
||||
|
||||
if auth_token.client == "gmail":
|
||||
return await self.__sync_many_gmail(
|
||||
session_token = session_token,
|
||||
user_info = user_info,
|
||||
mongo_conn = mongo_conn,
|
||||
token_id = token_id,
|
||||
auth_token = auth_token,
|
||||
mail_client = current_app.gmail_client,
|
||||
llm = llm,
|
||||
force_sync = force_sync,
|
||||
start_date = start_date,
|
||||
end_date = end_date,
|
||||
max_count = max_count
|
||||
)
|
||||
|
||||
# ┳ ┓• ┓ ┏┓┓•
|
||||
# ┃┏┓┓┏┏┓┃┓┏┫ ┃ ┃┓┏┓┏┓╋
|
||||
# ┻┛┗┗┛┗┻┗┗┗┻ ┗┛┗┗┗ ┛┗┗
|
||||
|
||||
# If we haven't been able to sync mail due to not entering any 'if' condition:
|
||||
sync_results.message = f"no such mail client '{auth_token.client}'"
|
||||
return sync_results
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,240 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Thursday, 5th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To work with auth details of SMS clients like Nimbus SMS (India) and Savvy Bulk SMS (Kenya).
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class SMSAuthModel(BaseModel):
|
||||
|
||||
AUTH_COLLECTION = "_authTokens"
|
||||
|
||||
async def set(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
mongo_conn: AsyncMongo,
|
||||
user_info: dict,
|
||||
client_user_id: dict,
|
||||
auth: dict,
|
||||
token: dict,
|
||||
service_client: Literal["nimbusSmsIndia", "savvyBulkSmsKenya"],
|
||||
auth_type: Literal["auth"],
|
||||
sync_freq: Literal[60, 300, 900] = 300,
|
||||
session_token: str = None
|
||||
) -> ObjectId | None:
|
||||
|
||||
"""
|
||||
To store auth/tokens for a particular service to the database.
|
||||
:param db_conn: The database connection (MariaDB) to use to perform the action.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param user_info: The dictionary that has the user's session information.
|
||||
:param client_user_id: The way the third-party client recognizes your user.
|
||||
:param auth: The authentication details of the account.
|
||||
:param token: The token granted by the third-party service.
|
||||
:param service_client: The name of the company or brand that is providing this service that is being integrated.
|
||||
:param auth_type: To identify the type of authentication being done here. This could indicate simple password
|
||||
authentication, more advance OAuth2.0 authentication, etc.
|
||||
:param sync_freq: The time interval in which mails need to be sync'd. Specify this in seconds.
|
||||
:param session_token: The session token of the user who requested this service.
|
||||
:return: An ObjectId to later store the granted tokens.
|
||||
"""
|
||||
|
||||
# Note down the timestamp at which this event occurred:
|
||||
request_ts = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
# Get the identifier from the database:
|
||||
mongo_json = await mongo_conn.find_one_and_update(
|
||||
collection = self.AUTH_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"serviceType": "email",
|
||||
"user": {
|
||||
"entityId": user_info["entityId"],
|
||||
"billingAccountId": user_info["billingAccountId"]
|
||||
},
|
||||
"clientUserId": client_user_id
|
||||
}),
|
||||
update = {
|
||||
"$set": {
|
||||
"lastRequestTs": request_ts,
|
||||
"status": "active",
|
||||
"syncFreq": max(sync_freq, 60)
|
||||
},
|
||||
"$setOnInsert": {
|
||||
"version": "1.0.0",
|
||||
"serviceType": "sms",
|
||||
"client": service_client,
|
||||
"authType": auth_type,
|
||||
"user": user_info,
|
||||
"clientUserId": client_user_id,
|
||||
"auth": auth,
|
||||
"token": token,
|
||||
"firstRefreshTs": None,
|
||||
"lastRefreshTs": None,
|
||||
"firstRequestTs": request_ts
|
||||
}
|
||||
},
|
||||
projection = {
|
||||
"_id": True
|
||||
},
|
||||
upsert = True,
|
||||
return_updated = True
|
||||
)
|
||||
|
||||
# Tell MariaDB that an authorization request was initiated:
|
||||
db_json = {}
|
||||
if mongo_json is not None:
|
||||
db_json = await self.call_procedure(
|
||||
db_conn = db_conn,
|
||||
proc_name = "entity_integration_save",
|
||||
proc_args = (
|
||||
user_info["entityId"], # ........................................... 'p_entity_id'
|
||||
service_client, # .................................................. 'p_provider'
|
||||
"Active", # ........................................................ 'p_current_status'
|
||||
"Auth Details Accepted", # ......................................... 'p_last_action'
|
||||
None, # ............................................................ 'p_display_name'
|
||||
None, # ............................................................ 'p_display_picture'
|
||||
str(mongo_json["_id"]), # .......................................... 'p_token_id'
|
||||
json.to_string(python_data = client_user_id, no_space = True), # ... 'p_notes'
|
||||
user_info["userId"] # .............................................. 'p_created_by'
|
||||
),
|
||||
session_token = session_token
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
|
||||
|
||||
async def get(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId | str = None,
|
||||
**kwargs
|
||||
) -> dict | None:
|
||||
|
||||
"""
|
||||
To retrieve stored auth/tokens from the database.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param token_id: The identifier granted providing auth details for the first time in 'set_token'.
|
||||
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
|
||||
info, the client, the type of authentication used, or even the kind of service.
|
||||
:return: The retrieved record that has the token, and information about the service and client if found, else
|
||||
None when there is no matching record.
|
||||
"""
|
||||
|
||||
# Build the filter:
|
||||
filter_json = {k: v for k, v in kwargs.items()}
|
||||
if token_id: filter_json["_id"] = ObjectId(token_id)
|
||||
|
||||
# If there is no search criteria, we exit with failure:
|
||||
if not filter_json: return None
|
||||
|
||||
# If there is some filtering possible,
|
||||
# we fetch and return the token:
|
||||
return await mongo_conn.find_one(
|
||||
collection = self.AUTH_COLLECTION,
|
||||
filter = filter_json,
|
||||
projection = {
|
||||
"_id": True,
|
||||
"serviceType": True,
|
||||
"authType": True,
|
||||
"client": True,
|
||||
"clientUserId": True,
|
||||
"token": True
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,227 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
ORIGINAL: Thursday, 5th Dec., 2024
|
||||
UPGRADED: Monday, 9th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To work with auth details of SMS clients like Nimbus SMS (India) and Savvy Bulk SMS (Kenya).
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.core.auth_token import CoreAuthTokenModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class SMSAuthModel(BaseModel):
|
||||
|
||||
AUTH_COLLECTION = "_authTokens"
|
||||
|
||||
async def set(
|
||||
self,
|
||||
db_conn: AsyncMySQL,
|
||||
mongo_conn: AsyncMongo,
|
||||
auth_token: CoreAuthTokenModel,
|
||||
session_token: str = None
|
||||
) -> ObjectId | None:
|
||||
|
||||
"""
|
||||
To store auth/tokens for a particular service to the database.
|
||||
:param db_conn: The database connection (MariaDB) to use to perform the action.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param auth_token: An instance of the core auth-token model that holds data in the database.
|
||||
:param session_token: The session token of the user who requested this service.
|
||||
:return: An ObjectId to later store the granted tokens.
|
||||
"""
|
||||
|
||||
# Note down the timestamp at which this event occurred:
|
||||
request_ts = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
# Get the identifier from the database:
|
||||
# BE CAREFUL WITH THE KEYS HERE, THEY SHOULD MATCH THE FIELDS OF THE CORE AUTH-TOKEN MODEL:
|
||||
mongo_json = await mongo_conn.find_one_and_update(
|
||||
collection = self.AUTH_COLLECTION,
|
||||
filter = mongo_conn.dict_to_dot_notation({
|
||||
"serviceType": auth_token.serviceType,
|
||||
"user": {
|
||||
"entityId": auth_token.user.entityId,
|
||||
"billingAccountId": auth_token.user.billingAccountId
|
||||
},
|
||||
"clientUserId": auth_token.clientUserId
|
||||
}),
|
||||
update = {
|
||||
"$set": {
|
||||
"lastRequestTs": auth_token.lastRequestTs,
|
||||
"status": auth_token.status,
|
||||
"syncFreq": auth_token.syncFreq
|
||||
},
|
||||
"$setOnInsert": {
|
||||
"version": auth_token.version,
|
||||
"serviceType": auth_token.serviceType,
|
||||
"client": auth_token.client,
|
||||
"authType": auth_token.authType,
|
||||
"user": auth_token.user.model_dump(),
|
||||
"clientUserId": auth_token.clientUserId,
|
||||
"auth": auth_token.auth,
|
||||
"token": auth_token.token,
|
||||
"firstRefreshTs": auth_token.firstRefreshTs,
|
||||
"lastRefreshTs": auth_token.lastRefreshTs,
|
||||
"firstRequestTs": auth_token.firstRequestTs or request_ts
|
||||
}
|
||||
},
|
||||
projection = {
|
||||
"_id": True
|
||||
},
|
||||
upsert = True,
|
||||
return_updated = True
|
||||
)
|
||||
|
||||
# Tell MariaDB that an authorization request was initiated:
|
||||
db_json = {}
|
||||
if mongo_json is not None:
|
||||
token_notes = auth_token.clientUserId
|
||||
db_json = await self.call_procedure(
|
||||
db_conn = db_conn,
|
||||
proc_name = "entity_integration_save",
|
||||
proc_args = (
|
||||
auth_token.user.entityId, # ..................................... 'p_entity_id'
|
||||
auth_token.client, # ............................................ 'p_provider'
|
||||
auth_token.status, # ............................................ 'p_current_status'
|
||||
"Auth Details Accepted", # ...................................... 'p_last_action'
|
||||
None, # ......................................................... 'p_display_name'
|
||||
None, # ......................................................... 'p_display_picture'
|
||||
str(mongo_json["_id"]), # ....................................... 'p_token_id'
|
||||
json.to_string(python_data = token_notes, no_space = True), # ... 'p_notes'
|
||||
auth_token.user.userId # ........................................ 'p_created_by'
|
||||
),
|
||||
session_token = session_token
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return mongo_json["_id"] if mongo_json and db_json.get("status") == 1 else None
|
||||
|
||||
async def get(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId | str = None,
|
||||
**kwargs
|
||||
) -> dict | None:
|
||||
|
||||
"""
|
||||
To retrieve stored auth/tokens from the database.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param token_id: The identifier granted providing auth details for the first time in 'set_token'.
|
||||
:param kwargs: Any set of key-value pairs to build custom search criteria. This could be things like the user
|
||||
info, the client, the type of authentication used, or even the kind of service.
|
||||
:return: The retrieved record that has the token, and information about the service and client if found, else
|
||||
None when there is no matching record.
|
||||
"""
|
||||
|
||||
# Build the filter:
|
||||
filter_json = {k: v for k, v in kwargs.items()}
|
||||
if token_id: filter_json["_id"] = ObjectId(token_id)
|
||||
|
||||
# If there is no search criteria, we exit with failure:
|
||||
if not filter_json: return None
|
||||
|
||||
# If there is some filtering possible, we fetch the token:
|
||||
token = await mongo_conn.find_one(
|
||||
collection = self.AUTH_COLLECTION,
|
||||
filter = filter_json,
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return CoreAuthTokenModel(**token) if token else None
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -1,214 +0,0 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Monday, 9th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To send SMS from clients like Nimbus SMS (India) and Savvy Bulk SMS (Kenya).
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# My async utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
|
||||
# SMS-related utils:
|
||||
from utils_v2.sms.models.behaviour.nimbus.async_nimbus import AsyncNimbusSMS
|
||||
from utils_v2.sms.models.behaviour.savvy_bulk_sms.async_savvy_bulk_sms import AsyncSavvyBulkSMS
|
||||
|
||||
# Base model:
|
||||
from models.behaviour.base import BaseModel
|
||||
|
||||
# Data models:
|
||||
from models.data.core.auth_token import CoreAuthTokenModel
|
||||
from models.data.core.message import CoreMessageModel
|
||||
from models.data.api.sms.send import (
|
||||
SMSSendRequestHeaders,
|
||||
SMSSendRequestData,
|
||||
NimbusSMSIndiaMessage,
|
||||
SavvyBulkSMSKenyaMessage
|
||||
)
|
||||
from utils_v2.sms.models.data.sms_message import SentSMSMessageModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson import ObjectId
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Literal
|
||||
|
||||
# To make deep-copies:
|
||||
import copy
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class SMSSendModel(BaseModel):
|
||||
|
||||
MESSAGES_COLLECTION = "_messages"
|
||||
|
||||
async def send_sms(
|
||||
self,
|
||||
mongo_conn: AsyncMongo,
|
||||
token_id: ObjectId | str,
|
||||
auth_token: CoreAuthTokenModel,
|
||||
inbound_data: SMSSendRequestData,
|
||||
session_token: str = None
|
||||
) -> SentSMSMessageModel:
|
||||
|
||||
"""
|
||||
To store auth/tokens for a particular service to the database.
|
||||
:param mongo_conn: The database connection (MongoDB) to use to perform the action.
|
||||
:param auth_token: An instance of the core auth-token model that holds data in the database.
|
||||
:param session_token: The session token of the user who requested this service.
|
||||
:return: An ObjectId to later store the granted tokens.
|
||||
"""
|
||||
|
||||
# Basic prep:
|
||||
event_ts = date_time.get_current_utc_date_time(as_string = False)
|
||||
client_response = None
|
||||
message_id = None
|
||||
sms_sent = None
|
||||
|
||||
# ┏┓ ┳┓• ┓ ┏┓┳┳┓┏┓ ┳ ┓•
|
||||
# ┣ ┏┓┏┓ ┃┃┓┏┳┓┣┓┓┏┏ ┗┓┃┃┃┗┓ ┃┏┓┏┫┓┏┓
|
||||
# ┻ ┗┛┛ ┛┗┗┛┗┗┗┛┗┻┛ ┗┛┛ ┗┗┛ ┻┛┗┗┻┗┗┻
|
||||
|
||||
if isinstance(inbound_data.message, NimbusSMSIndiaMessage):
|
||||
|
||||
# Prepare the client:
|
||||
sms_client = AsyncNimbusSMS(
|
||||
entity_id = auth_token.auth.get("entityId"),
|
||||
sender_id = auth_token.auth.get("senderId"),
|
||||
user_id = auth_token.auth.get("userId"),
|
||||
api_key = auth_token.auth.get("apiKey"),
|
||||
http_client = self._http_client
|
||||
)
|
||||
|
||||
# Send the SMS:
|
||||
client_response = await sms_client.send_sms(
|
||||
recipient_number = inbound_data.message.recipientNo,
|
||||
message = inbound_data.message.text,
|
||||
template_id = inbound_data.message.templateId
|
||||
)
|
||||
|
||||
# ┏┓ ┏┓ ┳┓ ┓┓ ┏┓┳┳┓┏┓ ┓┏┓
|
||||
# ┣ ┏┓┏┓ ┗┓┏┓┓┏┓┏┓┏ ┣┫┓┏┃┃┏ ┗┓┃┃┃┗┓ ┃┫ ┏┓┏┓┓┏┏┓
|
||||
# ┻ ┗┛┛ ┗┛┗┻┗┛┗┛┗┫ ┻┛┗┻┗┛┗ ┗┛┛ ┗┗┛ ┛┗┛┗ ┛┗┗┫┗┻
|
||||
# ┛ ┛
|
||||
|
||||
elif isinstance(inbound_data.message, SavvyBulkSMSKenyaMessage):
|
||||
|
||||
# Prepare the client:
|
||||
sms_client = AsyncSavvyBulkSMS(
|
||||
api_key = auth_token.auth.get("apiKey"),
|
||||
partner_id = auth_token.auth.get("partnerId"),
|
||||
short_code = auth_token.auth.get("shortCode"),
|
||||
http_client = self._http_client
|
||||
)
|
||||
|
||||
# Send the SMS:
|
||||
client_response = await sms_client.send_sms(
|
||||
recipient_number = inbound_data.message.recipientNo,
|
||||
message = inbound_data.message.text
|
||||
)
|
||||
|
||||
# ┏┓ ┏┳┓┓ ┳┳┓
|
||||
# ┗┓┏┓┓┏┏┓ ┃ ┣┓┏┓ ┃┃┃┏┓┏┏┏┓┏┓┏┓
|
||||
# ┗┛┗┻┗┛┗ ┻ ┛┗┗ ┛ ┗┗ ┛┛┗┻┗┫┗
|
||||
# ┛
|
||||
|
||||
# Save the message:
|
||||
if client_response:
|
||||
message_id = await mongo_conn.insert_one(
|
||||
collection = self.MESSAGES_COLLECTION,
|
||||
document = CoreMessageModel(
|
||||
ts = event_ts,
|
||||
readTs = event_ts,
|
||||
tokenId = ObjectId(token_id),
|
||||
serviceType = auth_token.serviceType,
|
||||
client = auth_token.client,
|
||||
clientMessageId = client_response.messageId,
|
||||
clientThreadId = None,
|
||||
isInward = False,
|
||||
sentSuccessfully = client_response.success,
|
||||
payload = client_response.model_dump()
|
||||
).model_dump()
|
||||
)
|
||||
|
||||
# Done here:
|
||||
return client_response
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,255 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Thursday, 5th Dec., 2024.
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To provide a structure to normalize input to and output from a standardized LLM wrapper.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For making data behaviour_models:
|
||||
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
|
||||
from typing import Optional, Literal, Union, List, Any
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import regex
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# RegEx Patterns:
|
||||
REGEX_SESSION_TOKEN = r"^[a-f0-9]{8}-[a-f0-9]{4}-[1-5][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class LLMInputMessage(BaseModel):
|
||||
|
||||
role: Literal["system", "ai", "human"] = Field(
|
||||
description = "the role of this message",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
content: str = Field(
|
||||
description = "the message sent by the 'role'",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
# ┏┓ ┏•
|
||||
# ┃ ┏┓┏┓╋┓┏┓
|
||||
# ┗┛┗┛┛┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LLMInput(BaseModel):
|
||||
|
||||
messages: List[LLMInputMessage]
|
||||
|
||||
# ┏┓ ┏•
|
||||
# ┃ ┏┓┏┓╋┓┏┓
|
||||
# ┗┛┗┛┛┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
# ┓┏ ┓• ┓ •
|
||||
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
|
||||
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
|
||||
|
||||
@field_validator("messages")
|
||||
def validate_messages(cls, value):
|
||||
|
||||
# Maintain counter(s):
|
||||
system_message_index = -1
|
||||
system_message_count = 0
|
||||
|
||||
# Loop through the messages and check them:
|
||||
for index, message in enumerate(value):
|
||||
|
||||
# For 'system' messages:
|
||||
if message.role == "system":
|
||||
system_message_index = index
|
||||
system_message_count += 1
|
||||
|
||||
# Verify that there is AT MOST ONE 'system' message,
|
||||
# and verify that the 'system' message is the first message:
|
||||
if system_message_count > 1: raise ValueError(f"there can be at most 1 'system' message, found {system_message_count}")
|
||||
if system_message_index > 0: raise ValueError(f"'system' message must always be at index 0, found it at index {system_message_index}")
|
||||
|
||||
# Done here:
|
||||
return value
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LLMUsageTokens(BaseModel):
|
||||
|
||||
input: int = Field(
|
||||
description = "how many tokens were given in the input",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
output: int = Field(
|
||||
description = "how many tokens were generated as the output",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
total: int = Field(
|
||||
description = "the sum of the input and output tokens",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
# ┏┓ ┏•
|
||||
# ┃ ┏┓┏┓╋┓┏┓
|
||||
# ┗┛┗┛┛┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
class LLMOutput(BaseModel):
|
||||
|
||||
ts: AwareDatetime = Field(
|
||||
description = "the time at which the llm was invoked",
|
||||
default_factory = date_time.get_current_utc_date_time,
|
||||
frozen = True
|
||||
)
|
||||
|
||||
messages: List[LLMInputMessage] = Field(
|
||||
description = "the messages that came in that invoked the llm",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
output: str | None = Field(
|
||||
description = "what the llm generated",
|
||||
default = None,
|
||||
frozen = True
|
||||
)
|
||||
|
||||
client: Literal["openai"] = Field(
|
||||
description = "the co./brand that was used to use an llm",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
model: str = Field(
|
||||
description = "to know which model used in the process",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
tokens: LLMUsageTokens = Field(
|
||||
description = "to know how many tokens were used in the process",
|
||||
default = LLMUsageTokens(input = 0, output = 0, total = 0),
|
||||
frozen = True
|
||||
)
|
||||
|
||||
invocationId: Any | None = Field(
|
||||
description = "the id of the document that notes this invocation; useful for reconciliation",
|
||||
frozen = False,
|
||||
default = None
|
||||
)
|
||||
|
||||
# ┏┓ ┏•
|
||||
# ┃ ┏┓┏┓╋┓┏┓
|
||||
# ┗┛┗┛┛┗┛┗┗┫
|
||||
# ┛
|
||||
|
||||
class Config:
|
||||
extra = "forbid"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# llm_messages = [
|
||||
# {
|
||||
# "role": "system",
|
||||
# "content": "You are an office assistant."
|
||||
# },
|
||||
# {
|
||||
# "role": "ai",
|
||||
# "content": "Hello, sir. How may I help you today?"
|
||||
# },
|
||||
# {
|
||||
# "role": "human",
|
||||
# "content": "Please summarize this mail for me..."
|
||||
# }
|
||||
# ]
|
||||
#
|
||||
# llm_input = LLMInput(messages = llm_messages)
|
||||
# print(llm_input.model_dump_json(indent = 4))
|
||||
|
||||
llm_output = LLMOutput(
|
||||
messages=[LLMInputMessage(role='system', content="You are an office assistant. It's Christmas, so definitley respond like Santa Claus."), LLMInputMessage(role='ai', content='Hello, sir. How may I help you today?'), LLMInputMessage(role='human', content='Please summarize this mail for me...')],
|
||||
client = "openai",
|
||||
model = "o1"
|
||||
)
|
||||
@@ -44,7 +44,7 @@ from utils_v2.string import regex
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# Other core models:
|
||||
from models.data.core.user import CoreUserInfoModel
|
||||
from models.core.user import CoreUserInfoModel
|
||||
|
||||
# To work with MongoDB:
|
||||
from bson.objectid import ObjectId
|
||||
@@ -37,7 +37,7 @@ sys.path.append("..")
|
||||
|
||||
# For making data behaviour_models:
|
||||
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
|
||||
from typing import Optional, Literal, Union
|
||||
from typing import Optional, Literal, Union, List, Any
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import regex
|
||||
@@ -46,6 +46,9 @@ from utils_v2.date_time import date_time
|
||||
# To work with MongoDB:
|
||||
from bson.objectid import ObjectId
|
||||
|
||||
# Data models:
|
||||
from models.core.ai.llm import LLMOutput
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
@@ -141,7 +144,8 @@ class CoreMessageModel(BaseModel):
|
||||
|
||||
isSent: bool = Field(
|
||||
description = "to understand whether this message was an incoming message or outgoing message",
|
||||
frozen = True
|
||||
frozen = False,
|
||||
default = False
|
||||
)
|
||||
|
||||
isBroadcast: bool = Field(
|
||||
@@ -156,11 +160,28 @@ class CoreMessageModel(BaseModel):
|
||||
default = False
|
||||
)
|
||||
|
||||
aiSnippet: LLMOutput | None = Field(
|
||||
description = "holds a short summary generated by ",
|
||||
frozen = False
|
||||
)
|
||||
|
||||
preview: str = Field(
|
||||
description = "a truncated version of the actual textual content of the message",
|
||||
frozen = False
|
||||
)
|
||||
|
||||
message: dict = Field(
|
||||
description = "the actual contents of the message; will differ for each client",
|
||||
frozen = True
|
||||
)
|
||||
|
||||
tags: List[Any] = Field(
|
||||
description = "a list of keywords to apply to this file/dir to filter it later",
|
||||
frozen = False,
|
||||
default = [],
|
||||
examples = ["urgent", "otp", "GST"]
|
||||
)
|
||||
|
||||
usedAi: bool | None = Field(
|
||||
description = "to mark when a sent message was generated by ai; null means the status is not known",
|
||||
frozen = True,
|
||||
@@ -195,6 +216,11 @@ class CoreMessageModel(BaseModel):
|
||||
except: pass
|
||||
return value
|
||||
|
||||
@field_validator("tags", mode = "before")
|
||||
def validate_tags(cls, value):
|
||||
if value is None: value = []
|
||||
return value
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
@@ -37,7 +37,7 @@ sys.path.append("..")
|
||||
|
||||
# For making data behaviour_models:
|
||||
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
|
||||
from typing import Optional, Literal, Union, List
|
||||
from typing import Optional, Literal, Union, List, Any
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import regex
|
||||
@@ -171,6 +171,13 @@ class CorePaymentModel(BaseModel):
|
||||
frozen = True
|
||||
)
|
||||
|
||||
tags: List[Any] = Field(
|
||||
description = "a list of keywords to apply to this file/dir to filter it later",
|
||||
frozen = False,
|
||||
default = [],
|
||||
examples = ["renewal", "subscription"]
|
||||
)
|
||||
|
||||
client: Literal["razorpay", "safaricomMPesaExpress"] = Field(
|
||||
description = "the third-part client that was used",
|
||||
frozen = True
|
||||
@@ -221,6 +228,11 @@ class CorePaymentModel(BaseModel):
|
||||
if currency is None: raise ValueError("invalid currency code, please use iso 4217 standard")
|
||||
return value
|
||||
|
||||
@field_validator("tags", mode = "before")
|
||||
def validate_tags(cls, value):
|
||||
if value is None: value = []
|
||||
return value
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
Reference in New Issue
Block a user