From 70c817b3b83c4fd40a366984e25c43b2fb53e8df Mon Sep 17 00:00:00 2001 From: khushal Date: Thu, 12 Dec 2024 12:35:59 +0530 Subject: [PATCH] (20241212) dictionary key bug fix. --- models/behaviour/file/file.py | 444 ++++++++++++++++++++++++++++++ models/behaviour/mail/oauth_v3.py | 1 - models/data/core/file.py | 266 ++++++++++++++++++ models/data/core/file_object.py | 96 ++++++- models/data/core/user.py | 69 ++++- 5 files changed, 871 insertions(+), 5 deletions(-) create mode 100644 models/behaviour/file/file.py create mode 100644 models/data/core/file.py diff --git a/models/behaviour/file/file.py b/models/behaviour/file/file.py new file mode 100644 index 0000000..3d54849 --- /dev/null +++ b/models/behaviour/file/file.py @@ -0,0 +1,444 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Dec., 2024 + + OBJECTIVE: + + To define all file-management activities in one place. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level: +import io +# My async utils: +from utils_v2.string import json +from utils_v2.date_time import date_time +from utils_v2.system import files +from utils_v2.security.hash import Hasher +from utils_v2.database.async_mysql_v2 import AsyncMySQL +from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage + +# Base model: +from models.behaviour.base import BaseModel + +# Data models: +from models.data.core.user import CoreUserInfoModel +from models.data.core.file import CoreFileInfoModel, CoreFileAccessResponseModel + +# To work with MongoDB: +from bson import ObjectId + +# To work with datatypes: +from typing import Any, Literal, List + +# To make deep copies: +import copy + +# For debugging: +from icecream import IceCreamDebugger + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class FileManagementModel: + + def __init__( + self, + debug = True, + debug_prefix = "File Objs. | ", + ): + + # Debugging: + self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) + if not debug: self._printer.disable() + + def enable_debug(self): + self._printer.enable() + + def disable_debug(self): + self._printer.disable() + + # ┏┓ • ┓ ┏┓ • + # ┃┃┓┏┓┏┃┏ ┃┃┓┏┏┓┏┓┓┏┓┏ + # ┗┻┗┻┗┗┛┗ ┗┻┗┻┗ ┛ ┗┗ ┛ + + async def exists( + self, + mongo_conn: AsyncMongoStorage, + file_id: ObjectId | str + ) -> CoreFileAccessResponseModel: + + """ + To check whether, or not, a particular file's record exists in the database. + :param mongo_conn: The instance of the database connection to perform this action. + :param file_id: The id of the file to check. + :return: A structured response where the existence of the file is noted in the 'result' field. + """ + + # Create a response: + response = CoreFileAccessResponseModel() + + try: + + # Run the query: + record = await mongo_conn.find_one_file( + filter = {"_id": ObjectId(file_id)}, + projection = {"_id": True, "user": True}, + raise_exception = True + ) + + # Note down the result: + if record: + response.data = True + response.success = True + response.message = "ok" + + except Exception as exception: + self._printer(exception) + response.exception = exception + response.message = str(exception) + response.success = False + response.data = None + + # Done here: + return response + + async def info( + self, + mongo_conn: AsyncMongoStorage, + file_id: ObjectId | str + ) -> CoreFileAccessResponseModel: + + """ + To get the information about this file. + :param mongo_conn: The instance of the database connection to perform this action. + :param file_id: The id of the file to check. + :return: A structured response where the info of the file is noted in the 'result' field. + """ + + # Create a response: + response = CoreFileAccessResponseModel() + + try: + + # Run the query: + record = await mongo_conn.find_one_file( + filter = {"_id": ObjectId(file_id)}, + raise_exception = True + ) + + # Note down the result: + response.success = True + if record: + response.data = CoreFileInfoModel(**record["metadata"]) + response.message = "ok" + else: + response.message = "no such file object" + + # If something goes wrong: + except Exception as exception: + self._printer(exception) + response.exception = exception + response.message = str(exception) + response.success = False + response.data = None + + # Done here: + return response + + async def is_private( + self, + mongo_conn: AsyncMongoStorage, + file_id: ObjectId | str + ) -> bool | None: + + """ + To check whether, or not, a particular file is publicly readable. + :param mongo_conn: The instance of the database connection to perform this action. + :param file_id: The id of the file to check. + :return: True if private, else False. None if it doesn't exist at all. + """ + + # Create a response: + response = CoreFileAccessResponseModel() + + try: + + # Run the query: + record = await mongo_conn.find_one_file( + filter = {"_id": ObjectId(file_id)}, + projection = {"_id": False, "isPrivate": True}, + raise_exception = True + ) + + # Note down the result: + if record: + response.data = record["metadata"]["isPrivate"] + response.success = True + response.message = "ok" + + # If something goes wrong: + except Exception as exception: + self._printer(exception) + response.exception = exception + response.message = str(exception) + response.success = False + response.data = None + + # Done here: + return response + + # ┓ • • + # ┃ ┓┏╋┓┏┓┏┓ + # ┗┛┗┛┗┗┛┗┗┫ + # ┛ + + pass + + # ┳┓ ┓• + # ┣┫┏┓┏┓┏┫┓┏┓┏┓ + # ┛┗┗ ┗┻┗┻┗┛┗┗┫ + # ┛ + + async def download_file( + self, + mongo_conn: AsyncMongoStorage, + file_id: ObjectId | str + ) -> CoreFileAccessResponseModel: + + """ + To quickly download small files. Do not use this for larger files because the file will be held in RAM first + and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory. + :param mongo_conn: The instance of the database connection to perform this action. + :param file_id: The id of the file to fetch. + :return: The file in a BytesIO buffer in the 'data' field of the structured response. + """ + + # Create a response: + response = CoreFileAccessResponseModel() + + try: + + # Get the file from the database: + buffer = io.BytesIO() + response.success = await mongo_conn.easy_download( + destination = buffer, + file_id = ObjectId(file_id), + raise_exception = True + ) + buffer.seek(0) + + # Note down the results: + response.message = ( + "file fetched successfully" if response.success + else "file fetching failed" + ) + response.data = buffer if response.success else None + + # If something goes wrong: + except Exception as exception: + self._printer(exception) + response.exception = exception + response.message = str(exception) + response.success = False + response.data = None + + # Done here: + return response + + @staticmethod + async def get_file_download_stream( + mongo_conn: AsyncMongoStorage, + file_id: ObjectId + ) -> Any: + + """ + To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM. Once + the stream is created you can read from it, and terminate it like this: + READ: await stream.read(chunk_size) + CLOSE (without awaiting): stream.close() + :param mongo_conn: The instance of the database connection to perform this action. + :param file_id: The id of the file whose stream you would like to fetch. + :return: Returns the stream object that will allow more efficient downloads of files on the user's end. + """ + + return await mongo_conn.get_download_stream(file_id = file_id) + + # ┓ ┏ • • + # ┃┃┃┏┓┓╋┓┏┓┏┓ + # ┗┻┛┛ ┗┗┗┛┗┗┫ + # ┛ + + async def upload_file( + self, + mongo_conn: AsyncMongoStorage, + file_info: CoreFileInfoModel, + file_data: io.BytesIO | str, + chunk_size: int = None + ): + + # Create a response: + response = CoreFileAccessResponseModel() + + try: + + # Read the file's data into a BytesIO object: + if isinstance(file_data, str): + file_data = io.BytesIO(files.read_file(file_data, mode = "rb")) + file_data.seek(0) + + # Hash the file's data: + hasher = Hasher() + hasher.update(file_data.getvalue()) + file_info.hash = hasher.hexdigest() + + # Save the file to the database: + file_data.seek(0) + response.success = await mongo_conn.easy_upload( + source = file_data, + file_name = file_info.filename, + file_metadata = file_info.model_dump(), + file_id = file_info.fileId, + chunk_size = chunk_size, + raise_exception = True + ) + + # Note down the results: + response.message = ( + "file saved successfully" if response.success + else "file saving failed" + ) + response.data = True if response.success else False + + # If something goes wrong: + except Exception as exception: + self._printer(exception) + response.exception = exception + response.message = str(exception) + response.success = False + response.data = None + + # Done here: + return response + + async def upload_from_stream(self): pass + + # ┳┓ ┓ • + # ┃┃┏┓┃┏┓╋┓┏┓┏┓ + # ┻┛┗ ┗┗ ┗┗┛┗┗┫ + # ┛ + + async def delete_file(self): pass + + # ┏┓ • • + # ┃┃┏┓┏┓┏┳┓┓┏┏┓┏┓┏┓┏ + # ┣┛┗ ┛ ┛┗┗┗┛┛┗┗┛┛┗┛ + + async def make_public(self): pass + + async def make_private(self): pass + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + import asyncio + import time + + async def main(): + + files_mongo = AsyncMongoStorage( + connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external", + database_name = "converseStore", + max_connections = 10, + debug = True + ) + await files_mongo.connect() + my_files = FileManagementModel() + + user_bhopli = CoreUserInfoModel( + fullName = "Bhopli Narangi", + userId = 1, + entityId = 2, + billingAccountId = 3, + departmentId = 4, + branchId = 5, + industry = "technology" + ) + + file_info = await my_files.info( + mongo_conn = files_mongo, + file_id = "67598d48c1bf89b25695f20b" + ) + print("FILE INFO:", json.to_string(file_info.model_dump(), default = str)) + + + asyncio.run(main()) diff --git a/models/behaviour/mail/oauth_v3.py b/models/behaviour/mail/oauth_v3.py index a55643c..5a8af3d 100644 --- a/models/behaviour/mail/oauth_v3.py +++ b/models/behaviour/mail/oauth_v3.py @@ -139,7 +139,6 @@ class MailOAuthModel(BaseModel): "syncFreq": auth_token.syncFreq }, "$setOnInsert": { - "version": auth_token.version, "serviceType": auth_token.serviceType, "client": auth_token.client, "authType": auth_token.authType, diff --git a/models/data/core/file.py b/models/data/core/file.py new file mode 100644 index 0000000..71620a3 --- /dev/null +++ b/models/data/core/file.py @@ -0,0 +1,266 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Dec., 2024. + + OBJECTIVE: + + To define how information about files will be stored in the database. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For making data behaviour_models: +from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime +from typing import Optional, Literal, Union, Any, Dict, List + +# My utils: +from utils_v2.string import regex +from utils_v2.date_time import date_time + +# To work with MongoDB: +from bson.objectid import ObjectId + +# Other data models: +from models.data.core.user import CoreUserInfoModel + +# To work with date and time: +import datetime + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class CoreFileInfoModel(BaseModel): + + fileId: ObjectId = Field( + description = "the id of the document in mongodb that holds this information", + frozen = False, + default = None, + alias = "_id" + ) + + user: CoreUserInfoModel = Field( + description = "to identify the user who owns this file", + frozen = True + ) + + filename: str = Field( + description = "the name of this file", + frozen = True + ) + + uploadTs: AwareDatetime = Field( + description = "the time (utc) at which this file was uploaded", + frozen = True + ) + + length: int | None = Field( + description = "to note the size of the file in bytes", + frozen = False, + default = None + ) + + hash: str | None = Field( + description = "a simple hash to verify the integrity of the uploaded data; irrelevant for dirs", + frozen = True, + default = None + ) + + metadata: Dict[str, Any] = Field( + description = "any additional data about this file to filter it later", + frozen = False, + default = {} + ) + + tags: List[str] = Field( + description = "a list of keywords to apply to this file to filter it later", + frozen = False, + default = [], + examples = ["Bank Statement", "PDF", "bhopli@orange.com"] + ) + + isPrivate: bool = Field( + description = "whether this file is a private file, or publicly available", + frozen = True + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "allow" + arbitrary_types_allowed = True + + def model_dump(self, *args, **kwargs): + return super().model_dump(*args, by_alias = True, **kwargs) + + # ┓┏ ┓• ┓ • + # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓ + # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗ + + @field_validator("uploadTs", mode = "before") + def parse_date_time(cls, value): + return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC) + + @field_validator("metadata", mode = "before") + def validate_metadata(cls, value): + if value is None: value = {} + return value + + @field_validator("tags", mode = "before") + def validate_tags(cls, value): + if value is None: value = [] + return value + + @field_validator("fileId", mode = "before") + def parse_oid(cls, value): + try: + if isinstance(value, str): + value = ObjectId(value) + except: pass + return value + + # ┏┓ • + # ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏ + # ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛ + # ┛ + + @property + def summary(self): + return { + "user": self.user.model_dump(), + "name": self.name, + "createTs": self.createTs, + "parentId": self.parentId, + "isPrivate": self.isPrivate, + } + + +# --------------------------------------------------------------------------------------------------------------------- + + +class CoreFileAccessResponseModel(BaseModel): + + success: bool = Field( + description = "whether, or not, the operation was successful", + frozen = False, + default = False + ) + + message: str | None = Field( + description = "useful to note the reason in case of an unsuccessful operation", + frozen = False, + default = "message not documented" + ) + + data: Any = Field( + description = "the result of the operation", + frozen = False, + default = None + ) + + exception: Exception | None = Field( + description = "to document any exceptions that came up", + frozen = False, + default = None + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "allow" + arbitrary_types_allowed = True + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + from utils_v2.string import json + + file_info = CoreFileInfoModel( + _id = "67519cf3a7804fcbc6f12452", + user = CoreUserInfoModel( + fullName = "Bhopli Narangi", + userId = 1, + entityId = 2, + billingAccountId = 3, + departmentId = 4, + branchId = 5, + industry = "technology" + ), + filename = "Graduation Certificate.png", + uploadTs = date_time.get_current_utc_date_time(as_string = False), + length = 1024, + hash = "abcdefgh12345678", + metadata = { + "camera": "iPhone 1000 Pro Max XS" + }, + tags = [ + "important" + ], + parentId = None, + isPrivate = True + ) + + print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str)) \ No newline at end of file diff --git a/models/data/core/file_object.py b/models/data/core/file_object.py index d6dce56..f52db8e 100644 --- a/models/data/core/file_object.py +++ b/models/data/core/file_object.py @@ -291,6 +291,62 @@ class CoreFileObjectInfoModel(BaseModel): "isPrivate": self.isPrivate, } + # ┏┓ ┏┓ + # ┃ ┓┏┏╋┏┓┏┳┓ ┣ ┓┏┏┓┏┏ + # ┗┛┗┻┛┗┗┛┛┗┗ ┻ ┗┻┛┗┗┛ + + def is_owner( + self, + user: CoreUserInfoModel, + ignore_null: bool = False + ): + + """ + To check if a given user is the owner of this file object. + :param user: The instance of the model that defines the user that needs to be checked. + :param ignore_null: Whether, or not, to consider null values in the user to check. + :return: True if the given user is the owner, else False. + """ + + if self.user.matches(user, ignore_null = ignore_null): return True + else: return False + + def has_permission( + self, + user: CoreUserInfoModel, + permission: Literal["read", "write", "delete", "changePermissions"], + ignore_null: bool = False + ) -> bool: + + """ + To check if a given user is the owner of this file object. + :param user: The instance of the model that defines the user that needs to be checked. + :param permission: The permission to check. + :param ignore_null: Whether, or not, to consider null values in the user to check. + :return: True if the given user is the owner, else False. + """ + + # The owner of the file object always has all permissions: + if self.is_owner(user, ignore_null = ignore_null): return True + + # Now we check if the given non-owner user has the requested permission: + for sharing_data in self.sharedWith: + + # We match the users. + # If they don't match, we move to the next user: + if not sharing_data.user.matches( + user, + ignore_null = ignore_null + ): continue + + # If we found a matching user, + # we check for the permission: + if getattr(sharing_data.permissions, permission, False): + return True + + # If not condition hit, we don't have the permission: + return False + # --------------------------------------------------------------------------------------------------------------------- @@ -376,7 +432,7 @@ if __name__ == "__main__": billingAccountId = 8, departmentId = 9, branchId = 10, - industry = "entertainment" + industry = "finance" ), permissionType = "explicit", inheritedFromId = None, @@ -391,3 +447,41 @@ if __name__ == "__main__": ) print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str)) + + user_bhopli = CoreUserInfoModel( + fullName = "Bhopli Narangi", + userId = 1, + entityId = 2, + billingAccountId = 3, + departmentId = 4, + branchId = 5, + industry = "technology" + ) + + user_bhopli_partial = CoreUserInfoModel( + fullName = "Bhopli Narangi", + userId = 1, + entityId = None, + billingAccountId = None, + departmentId = 4, + branchId = None, + industry = "technology" + ) + + user_polki = CoreUserInfoModel( + fullName = "Polki Muchhwaali", + userId = 6, + entityId = 7, + billingAccountId = 8, + departmentId = 9, + branchId = 10, + industry = "finance" + ) + + print("IS OWNER:", file_info.is_owner(user_bhopli)) + print("IS OWNER:", file_info.is_owner(user_bhopli_partial, ignore_null = False)) + print("IS OWNER:", file_info.is_owner(user_bhopli_partial, ignore_null = True)) + print("HAS PERM:", file_info.has_permission(user_bhopli_partial, permission = "read", ignore_null = False)) + print("HAS PERM:", file_info.has_permission(user_bhopli_partial, permission = "read", ignore_null = True)) + print("HAS PERM:", file_info.has_permission(user_polki, permission = "read", ignore_null = True)) + print("HAS PERM:", file_info.has_permission(user_polki, permission = "delete", ignore_null = True)) diff --git a/models/data/core/user.py b/models/data/core/user.py index b9b8a7c..b0f26d4 100644 --- a/models/data/core/user.py +++ b/models/data/core/user.py @@ -131,6 +131,46 @@ class CoreUserInfoModel(BaseModel): extra = "ignore" arbitrary_types_allowed = True + # ┏┓ ┏┓ + # ┃ ┓┏┏╋┏┓┏┳┓ ┣ ┓┏┏┓┏┏ + # ┗┛┗┻┛┗┗┛┛┗┗ ┻ ┗┻┛┗┗┛ + + def matches( + self, + other_user: 'CoreUserInfoModel', + ignore_null: bool = False + ) -> bool: + + """ + To check if another instance of this user model matches this instance of the user model. + :param other_user: The other user to check. That user's value must match this user's values to return a positive + result. Whether, or not, null values are matched will be determined by the next param. + :param ignore_null: If set to True, null values IN THE OTHER USER will not be compared during the matching. + :return: True if they match, else False. + """ + + # Start by assuming success: + are_matching = True + + # Iterate through the required items: + for k, v in other_user.model_dump().items(): + + # Do not consider fields that are nulls if asked to ignore them: + if ignore_null and v is None: continue + + # Extract the corresponding value from the other user, + # and test it for being equal: + _v = getattr(self, k, None) + if ( + (not isinstance(v, type(_v))) or + (v != _v) + ): + are_matching = False + break + + # Done here: + return are_matching + # ***************************************************************************************************************** # ***** **** @@ -143,14 +183,37 @@ if __name__ == "__main__": from utils_v2.string import json - user_info = CoreUserInfoModel( + user_bhopli = CoreUserInfoModel( fullName = "Bhopli Narangi", userId = 1, entityId = 2, billingAccountId = 3, departmentId = 4, branchId = 5, - industry = "fashion" + industry = "technology" ) - print("USER INFO MODEL:", json.to_string(user_info.model_dump(), default = str)) + user_bhopli_partial = CoreUserInfoModel( + fullName = "Bhopli Narangi", + userId = 1, + entityId = None, + billingAccountId = None, + departmentId = 4, + branchId = None, + industry = "technology" + ) + + user_polki = CoreUserInfoModel( + fullName = "Polki Muchhwaali", + userId = 6, + entityId = 7, + billingAccountId = 8, + departmentId = 9, + branchId = 10, + industry = "finance" + ) + + print(user_bhopli.matches(user_bhopli_partial, ignore_null = False)) + print(user_bhopli.matches(user_bhopli_partial, ignore_null = True)) + print(user_bhopli.matches(user_polki, ignore_null = False)) + print(user_bhopli.matches(user_polki, ignore_null = True))