""" AUTHOR: Khushal P Soonderji DATE: Tuesday, 10th Dec., 2024 OBJECTIVE: To define all file-management activities in one place. REFERENCES: N/A DOWNLOADS: N/A """ import io # ***************************************************************************************************************** # ***** **** # *** IMPORT *** # ***** **** # ***************************************************************************************************************** # To make sibling directories accessible for imports: import sys sys.path.append(".") sys.path.append("..") # My async utils: from utils_v2.string import json from utils_v2.date_time import date_time from utils_v2.system import files from utils_v2.security.hash import Hasher from utils_v2.database.async_mysql_v2 import AsyncMySQL from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage # Base model: from models.behaviour.base import BaseModel # Data models: from models.data.core.user import CoreUserInfoModel from models.data.core.file_object import ( CoreFileObjectInfoModel, CoreFileObjectSharingModel, CoreFileObjectPermissionsModel, CoreFileObjectAccessResponseModel ) # To work with MongoDB: from bson import ObjectId # To work with datatypes: from typing import Any, Literal, List # To make deep copies: import copy # For debugging: from icecream import IceCreamDebugger # ***************************************************************************************************************** # ***** **** # *** MACROS / ONE-TIME INIT *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** VARIABLES *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** FUNCTIONS *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** CLASSES *** # ***** **** # ***************************************************************************************************************** class FileObjectManagementModel: # Define class-level variables: FILE_OBJECTS_COLLECTION = "_fileObjects" def __init__( self, debug = True, debug_prefix = "File Objs. | ", ): # Debugging: self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True) if not debug: self._printer.disable() def enable_debug(self): self._printer.enable() def disable_debug(self): self._printer.disable() # ┓┏ ┓ # ┣┫┏┓┃┏┓┏┓┏┓┏ # ┛┗┗ ┗┣┛┗ ┛ ┛ # ┛ @staticmethod def users_match( user_p: CoreUserInfoModel, user_r: CoreUserInfoModel, ignore_null: bool = True ) -> bool: """ To match if a user that is requesting a resource is the same as the user known to have access to the resource. :param user_p: One of the dicts to check. :param user_r: The other dict to check. :param ignore_null: Whether to consider only non-null values, or all values. :return: True if they match, else False. """ # Start by assuming success: are_matching = True # Iterate through the required items: for rk, rv in user_r.model_dump().items(): # Do not consider fields that are nulls if asked to ignore them: if ignore_null and rv is None: continue # Extract the corresponding value from the other user, # and test it for being equal: pv = getattr(user_p, rk, None) if ( (not isinstance(rv, type(pv))) or (rv != pv) ): are_matching = False break # Done here: return are_matching # ┏┓ • ┓ ┏┓ • # ┃┃┓┏┓┏┃┏ ┃┃┓┏┏┓┏┓┓┏┓┏ # ┗┻┗┻┗┗┛┗ ┗┻┗┻┗ ┛ ┗┗ ┛ async def exists( self, mongo_conn: AsyncMongoStorage, file_object_id: ObjectId | str ) -> CoreFileObjectAccessResponseModel: """ To check whether, or not, a particular file's record exists in the database. :param mongo_conn: The instance of the database connection to perform this action. :param file_object_id: The id of the file to check. :return: A structured response where the existence of the file is noted in the 'result' field. """ # Create a response: response = CoreFileObjectAccessResponseModel() try: # Run the query: record = await mongo_conn.find_one( collection = self.FILE_OBJECTS_COLLECTION, filter = {"_id": ObjectId(file_object_id)}, projection = {"_id": True, "user": True, "isDir": True}, raise_exception = True ) # Note down the result: if record: response.result = True response.success = True response.message = "ok" except Exception as exception: response.exception = exception response.message = str(exception) response.success = False response.result = None # Done here: return response async def info( self, mongo_conn: AsyncMongoStorage, file_object_id: ObjectId | str ) -> CoreFileObjectAccessResponseModel: """ To get the information about this file. :param mongo_conn: The instance of the database connection to perform this action. :param file_object_id: The id of the file to check. :return: A structured response where the info of the file is noted in the 'result' field. """ # Create a response: response = CoreFileObjectAccessResponseModel() try: # Run the query: record = await mongo_conn.find_one( collection = self.FILE_OBJECTS_COLLECTION, filter = {"_id": ObjectId(file_object_id)}, raise_exception = True ) # Note down the result: response.success = True if record: response.result = CoreFileObjectInfoModel(**record) response.message = "ok" else: response.message = "no such file object" # If something goes wrong: except Exception as exception: response.exception = exception response.message = str(exception) response.success = False response.result = None # Done here: return response async def is_private( self, mongo_conn: AsyncMongoStorage, file_object_id: ObjectId | str ) -> bool | None: """ To check whether, or not, a particular file is publicly readable. :param mongo_conn: The instance of the database connection to perform this action. :param file_object_id: The id of the file to check. :return: True if private, else False. None if it doesn't exist at all. """ # Create a response: response = CoreFileObjectAccessResponseModel() try: # Run the query: record = await mongo_conn.find_one( collection = self.FILE_OBJECTS_COLLECTION, filter = {"_id": ObjectId(file_object_id)}, projection = {"_id": False, "isPrivate": True}, raise_exception = True ) # Note down the result: if record: response.result = record["isPrivate"] response.success = True response.message = "ok" # If something goes wrong: except Exception as exception: response.exception = exception response.message = str(exception) response.success = False response.result = None # Done here: return response def is_owner( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, file_object_info: CoreFileObjectInfoModel, ignore_null: bool = True ) -> CoreFileObjectAccessResponseModel: """ To check if a specific user is the owner of a specific file. :param mongo_conn: The instance of the database connection to perform this action. :param user_info: The details of the user who needs to have permissions to this file. :param file_object_info: The information about the file/dir. Fetch it from the 'info' method. :param ignore_null: Whether to consider only non-null values, or all values. :return: True if owner, else False. None if something goes wrong. """ # Create a response: response = CoreFileObjectAccessResponseModel() try: # Test for a match: if self.users_match( user_p = file_object_info.user, user_r = user_info, ignore_null = ignore_null ): response.result = True response.message = "user is the owner of this resource" else: response.result = False response.message = "user is not the owner of this resource" response.success = True # If something goes wrong: except Exception as exception: response.exception = exception response.message = str(exception) response.success = False response.result = None # Done here: return response def has_permission( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, file_object_info: CoreFileObjectInfoModel, permission: Literal["read", "write", "delete", "changePermissions"], ignore_null: bool = True ) -> CoreFileObjectAccessResponseModel: """ To check if a particular user has permissions to a given file obj. You may pass either an instance of the file's info, or you may send the file's id to check. If you pass just the file's id, a database call will be needed. :param mongo_conn: The instance of the database connection to perform this action. :param user_info: The details of the user who needs to have permissions to this file. :param permission: The name of the permission that the said user must have on this file. :param file_object_info: The information about the file/dir. Fetch it from the 'info' method. :param ignore_null: Whether to consider only non-null values, or all values. :return: True if the user has said permission, else False. None if something goes wrong. """ # If this is a public file/dir, # and the permission requested is 'read': if (not file_object_info.isPrivate) and permission == "read": return CoreFileObjectAccessResponseModel( success = True, message = "this resource is publicly available", result = True, exception = None ) # The owner always has all permissions: response = self.is_owner( mongo_conn = mongo_conn, user_info = user_info, file_object_info = file_object_info, ignore_null = ignore_null ) if not response.success: return response if response.result is True: return response # Note down the failure of the ownership test: response.message = "this user does not have the requested permission over this resource" # Since The person requesting this is not the owner, # we check with the sharing details: for sharing_data in file_object_info.sharedWith: # We match the users. # If they don't match, we move to the next user: if not self.users_match( user_p = sharing_data.user, user_r = user_info, ignore_null = ignore_null ): continue # If we found a matching user, # we check for the permission: if getattr(sharing_data.permissions, permission, False): response.result = True response.success = True response.message = "this user has the requested permission over this resource" break # Done here: return response # ┓ • • # ┃ ┓┏╋┓┏┓┏┓ # ┗┛┗┛┗┗┛┗┗┫ # ┛ async def list_owned_dirs( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, limit: int = 50, skip: int = 0, ) -> List[CoreFileObjectInfoModel] | None: """ To list all the dirs that are owned by the described user. :param mongo_conn: The connection instance to use to make the check. :param user_info: The information about the user that we must match. :param limit: How many max. records to fetch. :param skip: How many initial records to skip. Useful for pagination. :return: The list of files owned by the user (can be empty), or None if something goes wrong. """ # Run the query: records = await mongo_conn.find_many( collection = self.FILE_OBJECTS_COLLECTION, filter = mongo_conn.dict_to_dot_notation({ "user": {k: v for k, v in user_info.model_dump().items() if v is not None}, "isDir": True }), limit = limit, skip = skip ) # If something went wrong, we receive null for the records. # We pass that null on: if records is None: return None # Otherwise, we format and return the records: return [CoreFileObjectInfoModel(**record) for record in records] async def list_shared_dirs( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, limit: int = 50, skip: int = 0, ) -> List[CoreFileObjectInfoModel] | None: """ To list all the dirs that have been shared with the described user. :param mongo_conn: The connection instance to use to make the check. :param user_info: The information about the user that we must match. :param limit: How many max. records to fetch. :param skip: How many initial records to skip. Useful for pagination. :return: The list of files owned by the user (can be empty), or None if something goes wrong. """ # Run the query: records = await mongo_conn.find_many( collection = self.FILE_OBJECTS_COLLECTION, filter = mongo_conn.dict_to_dot_notation({ "sharedWith.user": {k: v for k, v in user_info.model_dump().items() if v is not None}, "isDir": True }), limit = limit, skip = skip ) # If something went wrong, we receive null for the records. # We pass that null on: if records is None: return None # Otherwise, we format and return the records: return [CoreFileObjectInfoModel(**record) for record in records] async def list_owned_files( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, limit: int = 50, skip: int = 0, ) -> List[CoreFileObjectInfoModel] | None: """ To list all the files that are owned by the described user. :param mongo_conn: The connection instance to use to make the check. :param user_info: The information about the user that we must match. :param limit: How many max. records to fetch. :param skip: How many initial records to skip. Useful for pagination. :return: The list of files owned by the user (can be empty), or None if something goes wrong. """ # Run the query: records = await mongo_conn.find_many( collection = self.FILE_OBJECTS_COLLECTION, filter = mongo_conn.dict_to_dot_notation({ "user": {k: v for k, v in user_info.model_dump().items() if v is not None}, "isDir": False }), limit = limit, skip = skip ) # If something went wrong, we receive null for the records. # We pass that null on: if records is None: return None # Otherwise, we format and return the records: return [CoreFileObjectInfoModel(**record) for record in records] async def list_shared_files( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, limit: int = 50, skip: int = 0, ) -> List[CoreFileObjectInfoModel] | None: """ To list all the files that have been shared with the described user. :param mongo_conn: The connection instance to use to make the check. :param user_info: The information about the user that we must match. :param limit: How many max. records to fetch. :param skip: How many initial records to skip. Useful for pagination. :return: The list of files owned by the user (can be empty), or None if something goes wrong. """ # Run the query: records = await mongo_conn.find_many( collection = self.FILE_OBJECTS_COLLECTION, filter = mongo_conn.dict_to_dot_notation({ "sharedWith.user": {k: v for k, v in user_info.model_dump().items() if v is not None}, "isDir": False }), limit = limit, skip = skip ) # If something went wrong, we receive null for the records. # We pass that null on: if records is None: return None # Otherwise, we format and return the records: return [CoreFileObjectInfoModel(**record) for record in records] # ┳┓ ┓• # ┣┫┏┓┏┓┏┫┓┏┓┏┓ # ┛┗┗ ┗┻┗┻┗┛┗┗┫ # ┛ @staticmethod async def download_file( mongo_conn: AsyncMongoStorage, file_id: ObjectId ) -> io.BytesIO | None: """ To quickly download small files. Do not use this for larger files because the file will be held in RAM first and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory. WARNING: Check for permissions before using this method. :param mongo_conn: The instance of the database connection to perform this action. :param file_id: The id of the file to fetch. :return: The file in a BytesIO buffer, or None if the file doesn't exist. """ # Get the file from the database: buffer = io.BytesIO() success = await mongo_conn.easy_download( destination = buffer, file_id = ObjectId(file_id), raise_exception = True ) buffer.seek(0) # Return the result: if not success: return None else: return buffer @staticmethod async def get_file_download_stream( mongo_conn: AsyncMongoStorage, file_id: ObjectId ) -> Any: """ To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM. Once the stream is created you can read from it, and terminate it like this: READ: await stream.read(chunk_size) CLOSE (without awaiting): stream.close() WARNING: Check for permissions before using this method. :param mongo_conn: The instance of the database connection to perform this action. :param file_id: The id of the file whose stream you would like to fetch. :return: Returns the stream object that will allow more efficient downloads of files on the user's end. """ return await mongo_conn.get_download_stream(file_id = file_id) # ┓ ┏ • • # ┃┃┃┏┓┓╋┓┏┓┏┓ # ┗┻┛┛ ┗┗┗┛┗┗┫ # ┛ async def make_dir( self, mongo_conn: AsyncMongoStorage, user_info: CoreUserInfoModel, dir_name: str, dir_metadata: dict = None, dir_tags: list = None, parent_id: ObjectId | str = None ) -> bool: # Create an instance of the directory's model: dir_model = CoreFileObjectInfoModel( _id = mongo_conn.generate_id(), user = user_info, isDir = True, name = dir_name, createTs = date_time.get_current_utc_date_time(as_string = False), metadata = dir_metadata, tags = dir_tags, parentId = parent_id, isPrivate = True ) print("DIRECTORY:", json.to_string(dir_model.model_dump(), default = str)) # Insert this document into the database: inserted_id = await mongo_conn.insert_one( collection = self.FILE_OBJECTS_COLLECTION, document = dir_model.model_dump() ) # Done here: return True if inserted_id else False async def upload_file( self, mongo_conn: AsyncMongoStorage, file_info: CoreFileObjectInfoModel, file_data: io.BytesIO | str ): # Start by assuming failure: file_uploaded = False # Start a session: async with await (await mongo_conn.client).start_session() as session: # Define the transaction options: options = { # "read_concern": {"level": "snapshot"}, # ... Optional: ensures consistent reads. # "write_concern": {"w": "majority"}, # ...... Ensures writes are acknowledged. # "read_preference": "primary", # ............ Specify where to read from (e.g., primary). } # Start the transaction: async with session.start_transaction(**options): try: # Check that we indeed have a file that we are uploading: if file_info.isDir: raise ValueError("cannot data to upload directory object") # Generate an ObjectId: file_info.fileObjectId = mongo_conn.generate_id(as_str = False) # Read the file's data into a BytesIO object: if isinstance(file_data, str): file_data = io.BytesIO(files.read_file(file_data, mode = "rb")) file_data.seek(0) # Hash the file's data: hasher = Hasher() hasher.update(file_data.getvalue()) file_info.hash = hasher.hexdigest() # Now we upload the actual content of the file with the same id: success = await mongo_conn.easy_upload( source = file_data, file_name = file_info.name, file_metadata = None, file_id = file_info.fileObjectId, session = session, raise_exception = True ) # Safety check to ensure that the data was written: if not success: raise ValueError("file object's bytes weren't uploaded") # Write the file object's info model now with the same id: inserted_id = await mongo_conn.insert_one( collection = self.FILE_OBJECTS_COLLECTION, document = file_info.model_dump(), session = session, raise_exception = True ) # Safety check to ensure that the info was written: if inserted_id is None: raise ValueError("file object's info wasn't inserted") # If we've reached this far: file_uploaded = True # If something goes wrong: except Exception as exception: session.abort_transaction() file_uploaded = False self._printer(exception) # Done here: return file_uploaded async def upload_from_stream(self): pass # ┳┓ ┓ • # ┃┃┏┓┃┏┓╋┓┏┓┏┓ # ┻┛┗ ┗┗ ┗┗┛┗┗┫ # ┛ async def delete_dir(self): pass async def delete_file(self): pass # ┏┓ • • # ┃┃┏┓┏┓┏┳┓┓┏┏┓┏┓┏┓┏ # ┣┛┗ ┛ ┛┗┗┗┛┛┗┗┛┛┗┛ async def update_permissions(self): pass async def make_public(self): pass async def make_private(self): pass # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** # ***** **** # ***************************************************************************************************************** if __name__ == "__main__": import asyncio import time async def main(): files_mongo = AsyncMongoStorage( connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external", database_name = "converseStore", max_connections = 10, debug = True ) await files_mongo.connect() my_fs = FileObjectManagementModel() user_bhopli = CoreUserInfoModel( fullName = "Bhopli Narangi", userId = 1, entityId = 2, billingAccountId = 3, departmentId = 4, branchId = 5, industry = "technology" ) user_polki = CoreUserInfoModel( fullName = "Polki Muchhwaali", userId = 6, entityId = 7, billingAccountId = 8, departmentId = 9, branchId = 10, # industry = "finance" ) file_info = await my_fs.info( mongo_conn = files_mongo, file_object_id = "67598d48c1bf89b25695f20b" ) print("SUCCESS:", file_info.success) print("MESSAGE:", file_info.message) print("FILE INFO:", json.to_string(file_info.result.model_dump(), default = str)) is_owner = my_fs.has_permission( mongo_conn = files_mongo, user_info = user_polki, file_object_info = file_info.result, permission = "write" ) print("HAS PERMISSION:", is_owner.model_dump_json(indent = 4)) asyncio.run(main())