445 lines
15 KiB
Python
445 lines
15 KiB
Python
"""
|
|
|
|
AUTHOR:
|
|
|
|
Khushal P Soonderji
|
|
|
|
DATE:
|
|
|
|
Thursday, 12th Dec., 2024
|
|
|
|
OBJECTIVE:
|
|
|
|
To define all file-management activities in one place.
|
|
|
|
REFERENCES:
|
|
|
|
N/A
|
|
|
|
DOWNLOADS:
|
|
|
|
N/A
|
|
|
|
"""
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** IMPORT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# To make sibling directories accessible for imports:
|
|
import sys
|
|
sys.path.append(".")
|
|
sys.path.append("..")
|
|
|
|
# System-level:
|
|
import io
|
|
# My async utils:
|
|
from utils_v2.string import json
|
|
from utils_v2.date_time import date_time
|
|
from utils_v2.system import files
|
|
from utils_v2.security.hash import Hasher
|
|
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
|
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
|
|
|
|
# Base model:
|
|
from models.behaviour.base import BaseModel
|
|
|
|
# Data models:
|
|
from models.data.core.user import CoreUserInfoModel
|
|
from models.data.core.file import CoreFileInfoModel, CoreFileAccessResponseModel
|
|
|
|
# To work with MongoDB:
|
|
from bson import ObjectId
|
|
|
|
# To work with datatypes:
|
|
from typing import Any, Literal, List
|
|
|
|
# To make deep copies:
|
|
import copy
|
|
|
|
# For debugging:
|
|
from icecream import IceCreamDebugger
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MACROS / ONE-TIME INIT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** VARIABLES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** FUNCTIONS ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** CLASSES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
class FileManagementModel:
|
|
|
|
def __init__(
|
|
self,
|
|
debug = True,
|
|
debug_prefix = "File Objs. | ",
|
|
):
|
|
|
|
# Debugging:
|
|
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
|
|
if not debug: self._printer.disable()
|
|
|
|
def enable_debug(self):
|
|
self._printer.enable()
|
|
|
|
def disable_debug(self):
|
|
self._printer.disable()
|
|
|
|
# ┏┓ • ┓ ┏┓ •
|
|
# ┃┃┓┏┓┏┃┏ ┃┃┓┏┏┓┏┓┓┏┓┏
|
|
# ┗┻┗┻┗┗┛┗ ┗┻┗┻┗ ┛ ┗┗ ┛
|
|
|
|
async def exists(
|
|
self,
|
|
mongo_conn: AsyncMongoStorage,
|
|
file_id: ObjectId | str
|
|
) -> CoreFileAccessResponseModel:
|
|
|
|
"""
|
|
To check whether, or not, a particular file's record exists in the database.
|
|
:param mongo_conn: The instance of the database connection to perform this action.
|
|
:param file_id: The id of the file to check.
|
|
:return: A structured response where the existence of the file is noted in the 'result' field.
|
|
"""
|
|
|
|
# Create a response:
|
|
response = CoreFileAccessResponseModel()
|
|
|
|
try:
|
|
|
|
# Run the query:
|
|
record = await mongo_conn.find_one_file(
|
|
filter = {"_id": ObjectId(file_id)},
|
|
projection = {"_id": True, "user": True},
|
|
raise_exception = True
|
|
)
|
|
|
|
# Note down the result:
|
|
if record:
|
|
response.data = True
|
|
response.success = True
|
|
response.message = "ok"
|
|
|
|
except Exception as exception:
|
|
self._printer(exception)
|
|
response.exception = exception
|
|
response.message = str(exception)
|
|
response.success = False
|
|
response.data = None
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
async def info(
|
|
self,
|
|
mongo_conn: AsyncMongoStorage,
|
|
file_id: ObjectId | str
|
|
) -> CoreFileAccessResponseModel:
|
|
|
|
"""
|
|
To get the information about this file.
|
|
:param mongo_conn: The instance of the database connection to perform this action.
|
|
:param file_id: The id of the file to check.
|
|
:return: A structured response where the info of the file is noted in the 'result' field.
|
|
"""
|
|
|
|
# Create a response:
|
|
response = CoreFileAccessResponseModel()
|
|
|
|
try:
|
|
|
|
# Run the query:
|
|
record = await mongo_conn.find_one_file(
|
|
filter = {"_id": ObjectId(file_id)},
|
|
raise_exception = True
|
|
)
|
|
|
|
# Note down the result:
|
|
response.success = True
|
|
if record:
|
|
response.data = CoreFileInfoModel(**record["metadata"])
|
|
response.message = "ok"
|
|
else:
|
|
response.message = "no such file object"
|
|
|
|
# If something goes wrong:
|
|
except Exception as exception:
|
|
self._printer(exception)
|
|
response.exception = exception
|
|
response.message = str(exception)
|
|
response.success = False
|
|
response.data = None
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
async def is_private(
|
|
self,
|
|
mongo_conn: AsyncMongoStorage,
|
|
file_id: ObjectId | str
|
|
) -> bool | None:
|
|
|
|
"""
|
|
To check whether, or not, a particular file is publicly readable.
|
|
:param mongo_conn: The instance of the database connection to perform this action.
|
|
:param file_id: The id of the file to check.
|
|
:return: True if private, else False. None if it doesn't exist at all.
|
|
"""
|
|
|
|
# Create a response:
|
|
response = CoreFileAccessResponseModel()
|
|
|
|
try:
|
|
|
|
# Run the query:
|
|
record = await mongo_conn.find_one_file(
|
|
filter = {"_id": ObjectId(file_id)},
|
|
projection = {"_id": False, "isPrivate": True},
|
|
raise_exception = True
|
|
)
|
|
|
|
# Note down the result:
|
|
if record:
|
|
response.data = record["metadata"]["isPrivate"]
|
|
response.success = True
|
|
response.message = "ok"
|
|
|
|
# If something goes wrong:
|
|
except Exception as exception:
|
|
self._printer(exception)
|
|
response.exception = exception
|
|
response.message = str(exception)
|
|
response.success = False
|
|
response.data = None
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
# ┓ • •
|
|
# ┃ ┓┏╋┓┏┓┏┓
|
|
# ┗┛┗┛┗┗┛┗┗┫
|
|
# ┛
|
|
|
|
pass
|
|
|
|
# ┳┓ ┓•
|
|
# ┣┫┏┓┏┓┏┫┓┏┓┏┓
|
|
# ┛┗┗ ┗┻┗┻┗┛┗┗┫
|
|
# ┛
|
|
|
|
async def download_file(
|
|
self,
|
|
mongo_conn: AsyncMongoStorage,
|
|
file_id: ObjectId | str
|
|
) -> CoreFileAccessResponseModel:
|
|
|
|
"""
|
|
To quickly download small files. Do not use this for larger files because the file will be held in RAM first
|
|
and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory.
|
|
:param mongo_conn: The instance of the database connection to perform this action.
|
|
:param file_id: The id of the file to fetch.
|
|
:return: The file in a BytesIO buffer in the 'data' field of the structured response.
|
|
"""
|
|
|
|
# Create a response:
|
|
response = CoreFileAccessResponseModel()
|
|
|
|
try:
|
|
|
|
# Get the file from the database:
|
|
buffer = io.BytesIO()
|
|
response.success = await mongo_conn.easy_download(
|
|
destination = buffer,
|
|
file_id = ObjectId(file_id),
|
|
raise_exception = True
|
|
)
|
|
buffer.seek(0)
|
|
|
|
# Note down the results:
|
|
response.message = (
|
|
"file fetched successfully" if response.success
|
|
else "file fetching failed"
|
|
)
|
|
response.data = buffer if response.success else None
|
|
|
|
# If something goes wrong:
|
|
except Exception as exception:
|
|
self._printer(exception)
|
|
response.exception = exception
|
|
response.message = str(exception)
|
|
response.success = False
|
|
response.data = None
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
@staticmethod
|
|
async def get_file_download_stream(
|
|
mongo_conn: AsyncMongoStorage,
|
|
file_id: ObjectId
|
|
) -> Any:
|
|
|
|
"""
|
|
To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM. Once
|
|
the stream is created you can read from it, and terminate it like this:
|
|
READ: await stream.read(chunk_size)
|
|
CLOSE (without awaiting): stream.close()
|
|
:param mongo_conn: The instance of the database connection to perform this action.
|
|
:param file_id: The id of the file whose stream you would like to fetch.
|
|
:return: Returns the stream object that will allow more efficient downloads of files on the user's end.
|
|
"""
|
|
|
|
return await mongo_conn.get_download_stream(file_id = file_id)
|
|
|
|
# ┓ ┏ • •
|
|
# ┃┃┃┏┓┓╋┓┏┓┏┓
|
|
# ┗┻┛┛ ┗┗┗┛┗┗┫
|
|
# ┛
|
|
|
|
async def upload_file(
|
|
self,
|
|
mongo_conn: AsyncMongoStorage,
|
|
file_info: CoreFileInfoModel,
|
|
file_data: io.BytesIO | str,
|
|
chunk_size: int = None
|
|
):
|
|
|
|
# Create a response:
|
|
response = CoreFileAccessResponseModel()
|
|
|
|
try:
|
|
|
|
# Read the file's data into a BytesIO object:
|
|
if isinstance(file_data, str):
|
|
file_data = io.BytesIO(files.read_file(file_data, mode = "rb"))
|
|
file_data.seek(0)
|
|
|
|
# Hash the file's data:
|
|
hasher = Hasher()
|
|
hasher.update(file_data.getvalue())
|
|
file_info.hash = hasher.hexdigest()
|
|
|
|
# Save the file to the database:
|
|
file_data.seek(0)
|
|
response.success = await mongo_conn.easy_upload(
|
|
source = file_data,
|
|
file_name = file_info.filename,
|
|
file_metadata = file_info.model_dump(),
|
|
file_id = file_info.fileId,
|
|
chunk_size = chunk_size,
|
|
raise_exception = True
|
|
)
|
|
|
|
# Note down the results:
|
|
response.message = (
|
|
"file saved successfully" if response.success
|
|
else "file saving failed"
|
|
)
|
|
response.data = True if response.success else False
|
|
|
|
# If something goes wrong:
|
|
except Exception as exception:
|
|
self._printer(exception)
|
|
response.exception = exception
|
|
response.message = str(exception)
|
|
response.success = False
|
|
response.data = None
|
|
|
|
# Done here:
|
|
return response
|
|
|
|
async def upload_from_stream(self): pass
|
|
|
|
# ┳┓ ┓ •
|
|
# ┃┃┏┓┃┏┓╋┓┏┓┏┓
|
|
# ┻┛┗ ┗┗ ┗┗┛┗┗┫
|
|
# ┛
|
|
|
|
async def delete_file(self): pass
|
|
|
|
# ┏┓ • •
|
|
# ┃┃┏┓┏┓┏┳┓┓┏┏┓┏┓┏┓┏
|
|
# ┣┛┗ ┛ ┛┗┗┗┛┛┗┗┛┛┗┛
|
|
|
|
async def make_public(self): pass
|
|
|
|
async def make_private(self): pass
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MAIN PROGRAM ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
import asyncio
|
|
import time
|
|
|
|
async def main():
|
|
|
|
files_mongo = AsyncMongoStorage(
|
|
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
|
|
database_name = "converseStore",
|
|
max_connections = 10,
|
|
debug = True
|
|
)
|
|
await files_mongo.connect()
|
|
my_files = FileManagementModel()
|
|
|
|
user_bhopli = CoreUserInfoModel(
|
|
fullName = "Bhopli Narangi",
|
|
userId = 1,
|
|
entityId = 2,
|
|
billingAccountId = 3,
|
|
departmentId = 4,
|
|
branchId = 5,
|
|
industry = "technology"
|
|
)
|
|
|
|
file_info = await my_files.info(
|
|
mongo_conn = files_mongo,
|
|
file_id = "67598d48c1bf89b25695f20b"
|
|
)
|
|
print("FILE INFO:", json.to_string(file_info.model_dump(), default = str))
|
|
|
|
|
|
asyncio.run(main())
|