(20241212) dictionary key bug fix.

This commit is contained in:
2024-12-12 12:35:59 +05:30
parent 75d269c9a5
commit 70c817b3b8
5 changed files with 871 additions and 5 deletions
+444
View File
@@ -0,0 +1,444 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 12th Dec., 2024
OBJECTIVE:
To define all file-management activities in one place.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level:
import io
# My async utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
from utils_v2.system import files
from utils_v2.security.hash import Hasher
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
# Base model:
from models.behaviour.base import BaseModel
# Data models:
from models.data.core.user import CoreUserInfoModel
from models.data.core.file import CoreFileInfoModel, CoreFileAccessResponseModel
# To work with MongoDB:
from bson import ObjectId
# To work with datatypes:
from typing import Any, Literal, List
# To make deep copies:
import copy
# For debugging:
from icecream import IceCreamDebugger
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class FileManagementModel:
def __init__(
self,
debug = True,
debug_prefix = "File Objs. | ",
):
# Debugging:
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self._printer.disable()
def enable_debug(self):
self._printer.enable()
def disable_debug(self):
self._printer.disable()
# ┏┓ • ┓ ┏┓ •
# ┃┃┓┏┓┏┃┏ ┃┃┓┏┏┓┏┓┓┏┓┏
# ┗┻┗┻┗┗┛┗ ┗┻┗┻┗ ┛ ┗┗ ┛
async def exists(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> CoreFileAccessResponseModel:
"""
To check whether, or not, a particular file's record exists in the database.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: A structured response where the existence of the file is noted in the 'result' field.
"""
# Create a response:
response = CoreFileAccessResponseModel()
try:
# Run the query:
record = await mongo_conn.find_one_file(
filter = {"_id": ObjectId(file_id)},
projection = {"_id": True, "user": True},
raise_exception = True
)
# Note down the result:
if record:
response.data = True
response.success = True
response.message = "ok"
except Exception as exception:
self._printer(exception)
response.exception = exception
response.message = str(exception)
response.success = False
response.data = None
# Done here:
return response
async def info(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> CoreFileAccessResponseModel:
"""
To get the information about this file.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: A structured response where the info of the file is noted in the 'result' field.
"""
# Create a response:
response = CoreFileAccessResponseModel()
try:
# Run the query:
record = await mongo_conn.find_one_file(
filter = {"_id": ObjectId(file_id)},
raise_exception = True
)
# Note down the result:
response.success = True
if record:
response.data = CoreFileInfoModel(**record["metadata"])
response.message = "ok"
else:
response.message = "no such file object"
# If something goes wrong:
except Exception as exception:
self._printer(exception)
response.exception = exception
response.message = str(exception)
response.success = False
response.data = None
# Done here:
return response
async def is_private(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> bool | None:
"""
To check whether, or not, a particular file is publicly readable.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to check.
:return: True if private, else False. None if it doesn't exist at all.
"""
# Create a response:
response = CoreFileAccessResponseModel()
try:
# Run the query:
record = await mongo_conn.find_one_file(
filter = {"_id": ObjectId(file_id)},
projection = {"_id": False, "isPrivate": True},
raise_exception = True
)
# Note down the result:
if record:
response.data = record["metadata"]["isPrivate"]
response.success = True
response.message = "ok"
# If something goes wrong:
except Exception as exception:
self._printer(exception)
response.exception = exception
response.message = str(exception)
response.success = False
response.data = None
# Done here:
return response
# ┓ • •
# ┃ ┓┏╋┓┏┓┏┓
# ┗┛┗┛┗┗┛┗┗┫
# ┛
pass
# ┳┓ ┓•
# ┣┫┏┓┏┓┏┫┓┏┓┏┓
# ┛┗┗ ┗┻┗┻┗┛┗┗┫
# ┛
async def download_file(
self,
mongo_conn: AsyncMongoStorage,
file_id: ObjectId | str
) -> CoreFileAccessResponseModel:
"""
To quickly download small files. Do not use this for larger files because the file will be held in RAM first
and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory.
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file to fetch.
:return: The file in a BytesIO buffer in the 'data' field of the structured response.
"""
# Create a response:
response = CoreFileAccessResponseModel()
try:
# Get the file from the database:
buffer = io.BytesIO()
response.success = await mongo_conn.easy_download(
destination = buffer,
file_id = ObjectId(file_id),
raise_exception = True
)
buffer.seek(0)
# Note down the results:
response.message = (
"file fetched successfully" if response.success
else "file fetching failed"
)
response.data = buffer if response.success else None
# If something goes wrong:
except Exception as exception:
self._printer(exception)
response.exception = exception
response.message = str(exception)
response.success = False
response.data = None
# Done here:
return response
@staticmethod
async def get_file_download_stream(
mongo_conn: AsyncMongoStorage,
file_id: ObjectId
) -> Any:
"""
To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM. Once
the stream is created you can read from it, and terminate it like this:
READ: await stream.read(chunk_size)
CLOSE (without awaiting): stream.close()
:param mongo_conn: The instance of the database connection to perform this action.
:param file_id: The id of the file whose stream you would like to fetch.
:return: Returns the stream object that will allow more efficient downloads of files on the user's end.
"""
return await mongo_conn.get_download_stream(file_id = file_id)
# ┓ ┏ • •
# ┃┃┃┏┓┓╋┓┏┓┏┓
# ┗┻┛┛ ┗┗┗┛┗┗┫
# ┛
async def upload_file(
self,
mongo_conn: AsyncMongoStorage,
file_info: CoreFileInfoModel,
file_data: io.BytesIO | str,
chunk_size: int = None
):
# Create a response:
response = CoreFileAccessResponseModel()
try:
# Read the file's data into a BytesIO object:
if isinstance(file_data, str):
file_data = io.BytesIO(files.read_file(file_data, mode = "rb"))
file_data.seek(0)
# Hash the file's data:
hasher = Hasher()
hasher.update(file_data.getvalue())
file_info.hash = hasher.hexdigest()
# Save the file to the database:
file_data.seek(0)
response.success = await mongo_conn.easy_upload(
source = file_data,
file_name = file_info.filename,
file_metadata = file_info.model_dump(),
file_id = file_info.fileId,
chunk_size = chunk_size,
raise_exception = True
)
# Note down the results:
response.message = (
"file saved successfully" if response.success
else "file saving failed"
)
response.data = True if response.success else False
# If something goes wrong:
except Exception as exception:
self._printer(exception)
response.exception = exception
response.message = str(exception)
response.success = False
response.data = None
# Done here:
return response
async def upload_from_stream(self): pass
# ┳┓ ┓ •
# ┃┃┏┓┃┏┓╋┓┏┓┏┓
# ┻┛┗ ┗┗ ┗┗┛┗┗┫
# ┛
async def delete_file(self): pass
# ┏┓ • •
# ┃┃┏┓┏┓┏┳┓┓┏┏┓┏┓┏┓┏
# ┣┛┗ ┛ ┛┗┗┗┛┛┗┗┛┛┗┛
async def make_public(self): pass
async def make_private(self): pass
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
import time
async def main():
files_mongo = AsyncMongoStorage(
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
database_name = "converseStore",
max_connections = 10,
debug = True
)
await files_mongo.connect()
my_files = FileManagementModel()
user_bhopli = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = 2,
billingAccountId = 3,
departmentId = 4,
branchId = 5,
industry = "technology"
)
file_info = await my_files.info(
mongo_conn = files_mongo,
file_id = "67598d48c1bf89b25695f20b"
)
print("FILE INFO:", json.to_string(file_info.model_dump(), default = str))
asyncio.run(main())
-1
View File
@@ -139,7 +139,6 @@ class MailOAuthModel(BaseModel):
"syncFreq": auth_token.syncFreq "syncFreq": auth_token.syncFreq
}, },
"$setOnInsert": { "$setOnInsert": {
"version": auth_token.version,
"serviceType": auth_token.serviceType, "serviceType": auth_token.serviceType,
"client": auth_token.client, "client": auth_token.client,
"authType": auth_token.authType, "authType": auth_token.authType,
+266
View File
@@ -0,0 +1,266 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 12th Dec., 2024.
OBJECTIVE:
To define how information about files will be stored in the database.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
from typing import Optional, Literal, Union, Any, Dict, List
# My utils:
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with MongoDB:
from bson.objectid import ObjectId
# Other data models:
from models.data.core.user import CoreUserInfoModel
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class CoreFileInfoModel(BaseModel):
fileId: ObjectId = Field(
description = "the id of the document in mongodb that holds this information",
frozen = False,
default = None,
alias = "_id"
)
user: CoreUserInfoModel = Field(
description = "to identify the user who owns this file",
frozen = True
)
filename: str = Field(
description = "the name of this file",
frozen = True
)
uploadTs: AwareDatetime = Field(
description = "the time (utc) at which this file was uploaded",
frozen = True
)
length: int | None = Field(
description = "to note the size of the file in bytes",
frozen = False,
default = None
)
hash: str | None = Field(
description = "a simple hash to verify the integrity of the uploaded data; irrelevant for dirs",
frozen = True,
default = None
)
metadata: Dict[str, Any] = Field(
description = "any additional data about this file to filter it later",
frozen = False,
default = {}
)
tags: List[str] = Field(
description = "a list of keywords to apply to this file to filter it later",
frozen = False,
default = [],
examples = ["Bank Statement", "PDF", "bhopli@orange.com"]
)
isPrivate: bool = Field(
description = "whether this file is a private file, or publicly available",
frozen = True
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "allow"
arbitrary_types_allowed = True
def model_dump(self, *args, **kwargs):
return super().model_dump(*args, by_alias = True, **kwargs)
# ┓┏ ┓• ┓ •
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
@field_validator("uploadTs", mode = "before")
def parse_date_time(cls, value):
return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC)
@field_validator("metadata", mode = "before")
def validate_metadata(cls, value):
if value is None: value = {}
return value
@field_validator("tags", mode = "before")
def validate_tags(cls, value):
if value is None: value = []
return value
@field_validator("fileId", mode = "before")
def parse_oid(cls, value):
try:
if isinstance(value, str):
value = ObjectId(value)
except: pass
return value
# ┏┓ •
# ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
# ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
# ┛
@property
def summary(self):
return {
"user": self.user.model_dump(),
"name": self.name,
"createTs": self.createTs,
"parentId": self.parentId,
"isPrivate": self.isPrivate,
}
# ---------------------------------------------------------------------------------------------------------------------
class CoreFileAccessResponseModel(BaseModel):
success: bool = Field(
description = "whether, or not, the operation was successful",
frozen = False,
default = False
)
message: str | None = Field(
description = "useful to note the reason in case of an unsuccessful operation",
frozen = False,
default = "message not documented"
)
data: Any = Field(
description = "the result of the operation",
frozen = False,
default = None
)
exception: Exception | None = Field(
description = "to document any exceptions that came up",
frozen = False,
default = None
)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "allow"
arbitrary_types_allowed = True
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
from utils_v2.string import json
file_info = CoreFileInfoModel(
_id = "67519cf3a7804fcbc6f12452",
user = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = 2,
billingAccountId = 3,
departmentId = 4,
branchId = 5,
industry = "technology"
),
filename = "Graduation Certificate.png",
uploadTs = date_time.get_current_utc_date_time(as_string = False),
length = 1024,
hash = "abcdefgh12345678",
metadata = {
"camera": "iPhone 1000 Pro Max XS"
},
tags = [
"important"
],
parentId = None,
isPrivate = True
)
print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str))
+95 -1
View File
@@ -291,6 +291,62 @@ class CoreFileObjectInfoModel(BaseModel):
"isPrivate": self.isPrivate, "isPrivate": self.isPrivate,
} }
# ┏┓ ┏┓
# ┃ ┓┏┏╋┏┓┏┳┓ ┣ ┓┏┏┓┏┏
# ┗┛┗┻┛┗┗┛┛┗┗ ┻ ┗┻┛┗┗┛
def is_owner(
self,
user: CoreUserInfoModel,
ignore_null: bool = False
):
"""
To check if a given user is the owner of this file object.
:param user: The instance of the model that defines the user that needs to be checked.
:param ignore_null: Whether, or not, to consider null values in the user to check.
:return: True if the given user is the owner, else False.
"""
if self.user.matches(user, ignore_null = ignore_null): return True
else: return False
def has_permission(
self,
user: CoreUserInfoModel,
permission: Literal["read", "write", "delete", "changePermissions"],
ignore_null: bool = False
) -> bool:
"""
To check if a given user is the owner of this file object.
:param user: The instance of the model that defines the user that needs to be checked.
:param permission: The permission to check.
:param ignore_null: Whether, or not, to consider null values in the user to check.
:return: True if the given user is the owner, else False.
"""
# The owner of the file object always has all permissions:
if self.is_owner(user, ignore_null = ignore_null): return True
# Now we check if the given non-owner user has the requested permission:
for sharing_data in self.sharedWith:
# We match the users.
# If they don't match, we move to the next user:
if not sharing_data.user.matches(
user,
ignore_null = ignore_null
): continue
# If we found a matching user,
# we check for the permission:
if getattr(sharing_data.permissions, permission, False):
return True
# If not condition hit, we don't have the permission:
return False
# --------------------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------
@@ -376,7 +432,7 @@ if __name__ == "__main__":
billingAccountId = 8, billingAccountId = 8,
departmentId = 9, departmentId = 9,
branchId = 10, branchId = 10,
industry = "entertainment" industry = "finance"
), ),
permissionType = "explicit", permissionType = "explicit",
inheritedFromId = None, inheritedFromId = None,
@@ -391,3 +447,41 @@ if __name__ == "__main__":
) )
print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str)) print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str))
user_bhopli = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = 2,
billingAccountId = 3,
departmentId = 4,
branchId = 5,
industry = "technology"
)
user_bhopli_partial = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = None,
billingAccountId = None,
departmentId = 4,
branchId = None,
industry = "technology"
)
user_polki = CoreUserInfoModel(
fullName = "Polki Muchhwaali",
userId = 6,
entityId = 7,
billingAccountId = 8,
departmentId = 9,
branchId = 10,
industry = "finance"
)
print("IS OWNER:", file_info.is_owner(user_bhopli))
print("IS OWNER:", file_info.is_owner(user_bhopli_partial, ignore_null = False))
print("IS OWNER:", file_info.is_owner(user_bhopli_partial, ignore_null = True))
print("HAS PERM:", file_info.has_permission(user_bhopli_partial, permission = "read", ignore_null = False))
print("HAS PERM:", file_info.has_permission(user_bhopli_partial, permission = "read", ignore_null = True))
print("HAS PERM:", file_info.has_permission(user_polki, permission = "read", ignore_null = True))
print("HAS PERM:", file_info.has_permission(user_polki, permission = "delete", ignore_null = True))
+66 -3
View File
@@ -131,6 +131,46 @@ class CoreUserInfoModel(BaseModel):
extra = "ignore" extra = "ignore"
arbitrary_types_allowed = True arbitrary_types_allowed = True
# ┏┓ ┏┓
# ┃ ┓┏┏╋┏┓┏┳┓ ┣ ┓┏┏┓┏┏
# ┗┛┗┻┛┗┗┛┛┗┗ ┻ ┗┻┛┗┗┛
def matches(
self,
other_user: 'CoreUserInfoModel',
ignore_null: bool = False
) -> bool:
"""
To check if another instance of this user model matches this instance of the user model.
:param other_user: The other user to check. That user's value must match this user's values to return a positive
result. Whether, or not, null values are matched will be determined by the next param.
:param ignore_null: If set to True, null values IN THE OTHER USER will not be compared during the matching.
:return: True if they match, else False.
"""
# Start by assuming success:
are_matching = True
# Iterate through the required items:
for k, v in other_user.model_dump().items():
# Do not consider fields that are nulls if asked to ignore them:
if ignore_null and v is None: continue
# Extract the corresponding value from the other user,
# and test it for being equal:
_v = getattr(self, k, None)
if (
(not isinstance(v, type(_v))) or
(v != _v)
):
are_matching = False
break
# Done here:
return are_matching
# ***************************************************************************************************************** # *****************************************************************************************************************
# ***** **** # ***** ****
@@ -143,14 +183,37 @@ if __name__ == "__main__":
from utils_v2.string import json from utils_v2.string import json
user_info = CoreUserInfoModel( user_bhopli = CoreUserInfoModel(
fullName = "Bhopli Narangi", fullName = "Bhopli Narangi",
userId = 1, userId = 1,
entityId = 2, entityId = 2,
billingAccountId = 3, billingAccountId = 3,
departmentId = 4, departmentId = 4,
branchId = 5, branchId = 5,
industry = "fashion" industry = "technology"
) )
print("USER INFO MODEL:", json.to_string(user_info.model_dump(), default = str)) user_bhopli_partial = CoreUserInfoModel(
fullName = "Bhopli Narangi",
userId = 1,
entityId = None,
billingAccountId = None,
departmentId = 4,
branchId = None,
industry = "technology"
)
user_polki = CoreUserInfoModel(
fullName = "Polki Muchhwaali",
userId = 6,
entityId = 7,
billingAccountId = 8,
departmentId = 9,
branchId = 10,
industry = "finance"
)
print(user_bhopli.matches(user_bhopli_partial, ignore_null = False))
print(user_bhopli.matches(user_bhopli_partial, ignore_null = True))
print(user_bhopli.matches(user_polki, ignore_null = False))
print(user_bhopli.matches(user_polki, ignore_null = True))