khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241210) Started working on files.

Browse Source
This commit is contained in:
khushal
2024-12-10 19:56:25 +05:30
parent 4c3d546065
commit 570a28ef38
7 changed files with 828 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
models/behaviour/file/__init__.py
View File
models/behaviour/file/file.py
+371
View File
@@ -0,0 +1,371 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 10th Dec., 2024
OBJECTIVE:
To define all file-management activities in one place.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
import io
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
from sqlalchemy.orm.collections import collection
sys.path.append(".")
sys.path.append("..")
# My async utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
# Base model:
from models.behaviour.base import BaseModel
# Data models:
from models.data.core.user import CoreUserInfoModel
from models.data.core.file import CoreFileInfoModel, CoreFileSharingModel, CoreFilePermissionsModel
# To work with MongoDB:
from bson import ObjectId
# To work with datatypes:
from typing import Any, Literal
# To make deep-copies:
import copy
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class FileManagementModel:
# Define class-level variables:
USER_COLLECTION = "_filesUser"
FILE_COLLECTION = "_filesInfo"
DIRS_COLLECTION = "_dirsInfo"
LOGS_COLLECTION = "_filesLogs"
# ┳┓ ┓•
# ┣┫┏┓┏┓┏┫┓┏┓┏┓
# ┛┗┗ ┗┻┗┻┗┛┗┗┫
# ┛
async def exists(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> bool:
"""
To check whether, or not, a particular file's record exists in the database.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: True if it exists, else False.
"""
# Run the query:
record = await mongo_conn.find_one(
collection = self.FILE_COLLECTION,
filter = {"_id": ObjectId(file_id)},
projection = {"_id": True, "user": True}
)
# Return the result:
if not record: return False
else: return True
async def info(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId
) -> CoreFileInfoModel | None:
"""
To get the information about this file.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: None if the file doesn't exist, else its summary.
"""
# Run the query:
record = await mongo_conn.find_one(
collection = self.FILE_COLLECTION,
filter = {"_id": ObjectId(file_id)}
)
# return the results:
if not record: return None
else: return CoreFileInfoModel(**record)
async def is_private(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> bool | None:
"""
To check whether, or not, a particular file is publicly readable.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: True if private, else False. None if it doesn't exist at all.
"""
# Run the query:
record = await mongo_conn.find_one(
collection = self.FILE_COLLECTION,
filter = {"_id": ObjectId(file_id)},
projection = {"_id": False, "isPrivate": True}
)
# Return the result:
if not record: return None
else: return record["isPrivate"]
async def is_public(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> bool | None:
"""
A wrapper around the is_private, method that returns the opposite value.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: True if public, else False. None if it doesn't exist at all.
"""
# Call the existing function:
is_private = await self.is_private(
mongo_conn = mongo_conn,
file_id = file_id
)
# Return the opposite result:
if is_private is None: return is_private
else: return not is_private
async def is_owner(
self,
mongo_conn: AsyncMongoStorage,
user_info: CoreUserInfoModel,
file_id: ObjectId | str,
) -> bool | None:
"""
To check if a specific user is the owner of a specific file.
:param mongo_conn: The instance of the database connection to perform this action.
:param user_info: The details of the user who needs to have permissions to this file.
:param file_id: The id of the file to check.
:return: True if owner, else False. None if something goes wrong.
"""
# Parse the user conditions by ignoring the null values:
user_conditions = {k: v for k, v in user_info.model_dump().items() if v is not None}
# Run the query:
record = await mongo_conn.find_one(
collection = self.FILE_COLLECTION,
filter = mongo_conn.dict_to_dot_notation({
"_id": ObjectId(file_id),
"user": user_conditions,
}),
projection = {
"user": True
}
)
# Return the result:
if record is None: return False
else: return True
async def has_permission(
self,
mongo_conn: AsyncMongoStorage,
user_info: CoreUserInfoModel,
file_id: ObjectId | str,
permission: Literal["read", "delete", "changePermissions"]
) -> bool | None:
"""
To check if a particular user has permissions to a given file.
:param mongo_conn: The instance of the database connection to perform this action.
:param user_info: The details of the user who needs to have permissions to this file.
:param file_id: The id of the file to check.
:param permission: The name of the permission that the said user must have on this file.
:return: True if the user has said permission, else False. None if something goes wrong.
"""
# Parse the user conditions by ignoring the null values:
user_conditions = {k: v for k, v in user_info.model_dump().items() if v is not None}
# # If the user is the owner,
# # he has all the permissions:
# is_owner = await self.is_owner(
# mongo_conn = mongo_conn,
# user_info = user_info,
# file_id = file_id
# )
# if is_owner: return True
# otherwise, we run the query to check for granted permissions:
record = await mongo_conn.find_one(
collection = self.FILE_COLLECTION,
filter = mongo_conn.dict_to_dot_notation({
"_id": ObjectId(file_id),
"sharedWith.user": user_conditions,
}),
projection = {
"sharedWith.$": True
}
)
# Return the result:
if record is None: return False
else: return record["sharedWith"][0]["permissions"][permission]
@staticmethod
async def download(
mongo_conn: AsyncMongoStorage,
file_id: ObjectId
) -> io.BytesIO | None:
"""
To quickly download small files. Do not use this for larger files because the file will be held in RAM first
and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to fetch.
:return: The file in a BytesIO buffer, or None if the file doesn't exist.
"""
# Get the file from the database:
buffer = io.BytesIO()
success = await mongo_conn.easy_download(
destination = buffer,
file_id = ObjectId(file_id),
raise_exception = True
)
buffer.seek(0)
# Return the result:
if not success: return None
else: return buffer
async def download_from_stream(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId
) -> Any:
"""
To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to fetch.
:return: YET TO BE IMPLEMENTED.
"""
pass
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
import time
async def main():
files_mongo = AsyncMongoStorage(
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
database_name = "converseStore",
max_connections = 10,
debug = True
)
await files_mongo.connect()
my_fs = FileManagementModel()
start_time = time.time()
tasks = [my_fs.has_permission(
mongo_conn = files_mongo,
user_info = CoreUserInfoModel(
userId = _,
# industry = "fashion"
),
file_id = "67583b64f35e2c7d3cab7955",
permission = "read"
) for _ in range(1_000)]
results = await asyncio.gather(*tasks)
for i, r in enumerate(results):
print(f"{i: >4}: {r}")
print(f"Done in {time.time() - start_time:.5f} seconds")
asyncio.run(main())
models/data/core/auth_token.py
+1
View File
@@ -197,6 +197,7 @@ class CoreAuthTokenModel(BaseModel):
if __name__ == "__main__":
from utils_v2.string import json
auth_token = CoreAuthTokenModel(
models/data/core/file.py
+284
View File
@@ -0,0 +1,284 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 10th Dec., 2024.
OBJECTIVE:
To define how information about files will be stored in the database.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
from typing import Optional, Literal, Union, Any, Dict, List
# My utils:
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with MongoDB:
from bson.objectid import ObjectId
# Other data models:
from models.data.core.user import CoreUserInfoModel
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class CoreFilePermissionsModel(BaseModel):
read: bool = Field(
description = "grants permissions to read/view this file",
frozen = True,
default = True
)
delete: bool = Field(
description = "grants permissions to delete this file entirely",
frozen = True,
default = False
)
changePermissions: bool = Field(
description = "grants permissions to modify the permissions of this file",
frozen = True,
default = False
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "allow"
arbitrary_types_allowed = True
# ---------------------------------------------------------------------------------------------------------------------
class CoreFileSharingModel(BaseModel):
user: CoreUserInfoModel = Field(
description = "to identify the user who has access to this file",
frozen = True
)
permissions: CoreFilePermissionsModel = Field(
description = "the permissions that the above mentioned user has to this file",
frozen = True
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "allow"
arbitrary_types_allowed = True
# ---------------------------------------------------------------------------------------------------------------------
class CoreFileInfoModel(BaseModel):
version: str = Field(
description = "a hint about the version no. of this message",
min_length = 1,
frozen = True,
default = "2.0.0"
)
user: CoreUserInfoModel = Field(
description = "to identify the user who owns this file",
frozen = True
)
filename: str = Field(
description = "the name of this file",
frozen = True
)
uploadTs: AwareDatetime = Field(
description = "the time (utc) at which this message was sent by the sender",
frozen = True
)
length: int = Field(
description = "to note when the user has marked this message as unread",
frozen = False,
default = False
)
hash: str = Field(
description = "a simple hash to verify the integrity of the uploaded data",
frozen = True
)
metadata: Dict[str, Any] = Field(
description = "any addition data about this file to filter it later",
frozen = False,
default = {}
)
tags: List[str] = Field(
description = "a list of keywords to apply to this file to filter it later",
frozen = False,
default = [],
examples = ["Bank Statement", "PDF", "bhopli@orange.com"]
)
parentId: ObjectId | None = Field(
description = "to identify the parent directory of this file; null means root directory",
frozen = False
)
isPrivate: bool = Field(
description = "whether, or not, this file is a private file",
frozen = True
)
sharedWith: List[CoreFileSharingModel] = Field(
description = "sharing settings; specially relevant when the file is private",
frozen = True,
default = []
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "allow"
arbitrary_types_allowed = True
# ┓┏ ┓• ┓ •
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
@field_validator("uploadTs", mode = "before")
def parse_date_time(cls, value):
return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC)
@field_validator("parentId", mode = "before")
def parse_oid(cls, value):
try: value = ObjectId(value)
except: pass
return value
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
from utils_v2.string import json
file_info = CoreFileInfoModel(
user = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = 2,
billingAccountId = 3,
departmentId = 4,
branchId = 5,
industry = "fashion"
),
filename = "Giga-Cat.png",
uploadTs = date_time.get_current_utc_date_time(as_string = False),
length = 1024,
hash = "abcdefgh",
metadata = {
"camera": "iPhone 1000 Pro Max XS"
},
tags = [
"image",
"cute",
"cat"
],
parentId = "67519cf3a7804fcbc6f12452",
isPrivate = True,
sharedWith = [
CoreFileSharingModel(
user = CoreUserInfoModel(
fullName = "Polki Muchhwaali",
userId = 6,
entityId = 7,
billingAccountId = 8,
departmentId = 9,
branchId = 10,
industry = "entertainment"
),
permissions = CoreFilePermissionsModel(
read = True,
delete = False,
changePermissions = False
)
)
]
)
print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str))
models/data/core/message.py
+21 -9
View File
@@ -91,12 +91,24 @@ class CoreMessageModel(BaseModel):
frozen = True
)
readTs: AwareDatetime = Field(
description = "the time (utc) at which this message was read and stored by your server",
syncTs: AwareDatetime = Field(
description = "the time (utc) at which this message was pulled and stored in your server",
frozen = True,
default_factory = lambda: date_time.get_current_utc_date_time(as_string = False)
)
readTs: AwareDatetime | None = Field(
description = "the time (utc) at which this message was read by the user",
frozen = True,
default = None
)
markedAsUnread: bool = Field(
description = "to note when the user has marked this message as unread",
frozen = False,
default = False
)
tokenId: ObjectId = Field(
description = "the id of the auth token that is associated with this message",
frozen = True
@@ -127,10 +139,9 @@ class CoreMessageModel(BaseModel):
default = None
)
isInward: bool = Field(
description = "to understand whether this message was an inward message or outward message",
frozen = True,
default = True
isSent: bool = Field(
description = "to understand whether this message was an incoming message or outgoing message",
frozen = True
)
isBroadcast: bool = Field(
@@ -145,7 +156,7 @@ class CoreMessageModel(BaseModel):
default = False
)
payload: dict = Field(
message: dict = Field(
description = "the actual contents of the message; will differ for each client",
frozen = True
)
@@ -163,7 +174,7 @@ class CoreMessageModel(BaseModel):
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
@field_validator("ts", "readTs", mode = "before")
@field_validator("ts", "syncTs", "readTs", mode = "before")
def parse_date_time(cls, value):
return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC)
@@ -191,7 +202,8 @@ if __name__ == "__main__":
client = "gmail",
clientMessageId = 123,
clientThreadId = 456,
payload = {
isSent = False,
message = {
"from": "bhopli@gmil.com",
"to": "hello@thecaoffice.com",
"message": "Hello, World!"
models/data/core/user_info.py → models/data/core/user.py
+33 -13
View File
@@ -89,37 +89,44 @@ class CoreUserInfoModel(BaseModel):
fullName: str | None = Field(
description = "the full name of the user as found in the database",
frozen = True,
default = None,
examples = ["Bhopli Narangi"]
)
userId: int | str | None = Field(
userId: int | str | None = Field(
description = "the id of the user as found in the database",
frozen = True
frozen = True,
default = None
)
entityId: int | str | None = Field(
entityId: int | str | None = Field(
description = "the id of the entity with which this user is associated",
frozen = True
frozen = True,
default = None
)
billingAccountId: int | str | None = Field(
billingAccountId: int | str | None = Field(
description = "the id of the billing account with which this user is associated",
frozen = True
frozen = True,
default = None
)
departmentId: int | str | None = Field(
departmentId: int | str | None = Field(
description = "the id of the dept. in which this user is working",
frozen = True
frozen = True,
default = None
)
branchId: int | str | None = Field(
branchId: int | str | None = Field(
description = "the id of the branch in which this user is working",
frozen = True
frozen = True,
default = None
)
industry: str | None = Field(
industry: str | None = Field(
description = "the name of the industry this user is working in",
frozen = True
frozen = True,
default = None
)
# ┏┓ ┏•
@@ -140,4 +147,17 @@ class CoreUserInfoModel(BaseModel):
if __name__ == "__main__":
pass
from utils_v2.string import json
user_info = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = 2,
billingAccountId = 3,
departmentId = 4,
branchId = 5,
industry = "fashion"
)
print("USER INFO MODEL:", json.to_string(user_info.model_dump(), default = str))