(20241210) Started working on files.
This commit is contained in:
@@ -0,0 +1,371 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
AUTHOR:
|
||||||
|
|
||||||
|
Khushal P Soonderji
|
||||||
|
|
||||||
|
DATE:
|
||||||
|
|
||||||
|
Tuesday, 10th Dec., 2024
|
||||||
|
|
||||||
|
OBJECTIVE:
|
||||||
|
|
||||||
|
To define all file-management activities in one place.
|
||||||
|
|
||||||
|
REFERENCES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
DOWNLOADS:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
"""
|
||||||
|
import io
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** IMPORT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# To make sibling directories accessible for imports:
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from sqlalchemy.orm.collections import collection
|
||||||
|
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
# My async utils:
|
||||||
|
from utils_v2.string import json
|
||||||
|
from utils_v2.date_time import date_time
|
||||||
|
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||||
|
from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage
|
||||||
|
|
||||||
|
# Base model:
|
||||||
|
from models.behaviour.base import BaseModel
|
||||||
|
|
||||||
|
# Data models:
|
||||||
|
from models.data.core.user import CoreUserInfoModel
|
||||||
|
from models.data.core.file import CoreFileInfoModel, CoreFileSharingModel, CoreFilePermissionsModel
|
||||||
|
|
||||||
|
# To work with MongoDB:
|
||||||
|
from bson import ObjectId
|
||||||
|
|
||||||
|
# To work with datatypes:
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
# To make deep-copies:
|
||||||
|
import copy
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MACROS / ONE-TIME INIT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** VARIABLES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** FUNCTIONS ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** CLASSES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
class FileManagementModel:
|
||||||
|
|
||||||
|
# Define class-level variables:
|
||||||
|
USER_COLLECTION = "_filesUser"
|
||||||
|
FILE_COLLECTION = "_filesInfo"
|
||||||
|
DIRS_COLLECTION = "_dirsInfo"
|
||||||
|
LOGS_COLLECTION = "_filesLogs"
|
||||||
|
|
||||||
|
# ┳┓ ┓•
|
||||||
|
# ┣┫┏┓┏┓┏┫┓┏┓┏┓
|
||||||
|
# ┛┗┗ ┗┻┗┻┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
async def exists(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
file_id: ObjectId | str
|
||||||
|
) -> bool:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To check whether, or not, a particular file's record exists in the database.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param file_id: The id of the file to check.
|
||||||
|
:return: True if it exists, else False.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Run the query:
|
||||||
|
record = await mongo_conn.find_one(
|
||||||
|
collection = self.FILE_COLLECTION,
|
||||||
|
filter = {"_id": ObjectId(file_id)},
|
||||||
|
projection = {"_id": True, "user": True}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Return the result:
|
||||||
|
if not record: return False
|
||||||
|
else: return True
|
||||||
|
|
||||||
|
async def info(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
file_id: ObjectId
|
||||||
|
) -> CoreFileInfoModel | None:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To get the information about this file.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param file_id: The id of the file to check.
|
||||||
|
:return: None if the file doesn't exist, else its summary.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Run the query:
|
||||||
|
record = await mongo_conn.find_one(
|
||||||
|
collection = self.FILE_COLLECTION,
|
||||||
|
filter = {"_id": ObjectId(file_id)}
|
||||||
|
)
|
||||||
|
|
||||||
|
# return the results:
|
||||||
|
if not record: return None
|
||||||
|
else: return CoreFileInfoModel(**record)
|
||||||
|
|
||||||
|
async def is_private(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
file_id: ObjectId | str
|
||||||
|
) -> bool | None:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To check whether, or not, a particular file is publicly readable.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param file_id: The id of the file to check.
|
||||||
|
:return: True if private, else False. None if it doesn't exist at all.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Run the query:
|
||||||
|
record = await mongo_conn.find_one(
|
||||||
|
collection = self.FILE_COLLECTION,
|
||||||
|
filter = {"_id": ObjectId(file_id)},
|
||||||
|
projection = {"_id": False, "isPrivate": True}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Return the result:
|
||||||
|
if not record: return None
|
||||||
|
else: return record["isPrivate"]
|
||||||
|
|
||||||
|
async def is_public(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
file_id: ObjectId | str
|
||||||
|
) -> bool | None:
|
||||||
|
|
||||||
|
"""
|
||||||
|
A wrapper around the is_private, method that returns the opposite value.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param file_id: The id of the file to check.
|
||||||
|
:return: True if public, else False. None if it doesn't exist at all.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Call the existing function:
|
||||||
|
is_private = await self.is_private(
|
||||||
|
mongo_conn = mongo_conn,
|
||||||
|
file_id = file_id
|
||||||
|
)
|
||||||
|
|
||||||
|
# Return the opposite result:
|
||||||
|
if is_private is None: return is_private
|
||||||
|
else: return not is_private
|
||||||
|
|
||||||
|
async def is_owner(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
user_info: CoreUserInfoModel,
|
||||||
|
file_id: ObjectId | str,
|
||||||
|
) -> bool | None:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To check if a specific user is the owner of a specific file.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param user_info: The details of the user who needs to have permissions to this file.
|
||||||
|
:param file_id: The id of the file to check.
|
||||||
|
:return: True if owner, else False. None if something goes wrong.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Parse the user conditions by ignoring the null values:
|
||||||
|
user_conditions = {k: v for k, v in user_info.model_dump().items() if v is not None}
|
||||||
|
|
||||||
|
# Run the query:
|
||||||
|
record = await mongo_conn.find_one(
|
||||||
|
collection = self.FILE_COLLECTION,
|
||||||
|
filter = mongo_conn.dict_to_dot_notation({
|
||||||
|
"_id": ObjectId(file_id),
|
||||||
|
"user": user_conditions,
|
||||||
|
}),
|
||||||
|
projection = {
|
||||||
|
"user": True
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Return the result:
|
||||||
|
if record is None: return False
|
||||||
|
else: return True
|
||||||
|
|
||||||
|
async def has_permission(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
user_info: CoreUserInfoModel,
|
||||||
|
file_id: ObjectId | str,
|
||||||
|
permission: Literal["read", "delete", "changePermissions"]
|
||||||
|
) -> bool | None:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To check if a particular user has permissions to a given file.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param user_info: The details of the user who needs to have permissions to this file.
|
||||||
|
:param file_id: The id of the file to check.
|
||||||
|
:param permission: The name of the permission that the said user must have on this file.
|
||||||
|
:return: True if the user has said permission, else False. None if something goes wrong.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Parse the user conditions by ignoring the null values:
|
||||||
|
user_conditions = {k: v for k, v in user_info.model_dump().items() if v is not None}
|
||||||
|
|
||||||
|
# # If the user is the owner,
|
||||||
|
# # he has all the permissions:
|
||||||
|
# is_owner = await self.is_owner(
|
||||||
|
# mongo_conn = mongo_conn,
|
||||||
|
# user_info = user_info,
|
||||||
|
# file_id = file_id
|
||||||
|
# )
|
||||||
|
# if is_owner: return True
|
||||||
|
|
||||||
|
# otherwise, we run the query to check for granted permissions:
|
||||||
|
record = await mongo_conn.find_one(
|
||||||
|
collection = self.FILE_COLLECTION,
|
||||||
|
filter = mongo_conn.dict_to_dot_notation({
|
||||||
|
"_id": ObjectId(file_id),
|
||||||
|
"sharedWith.user": user_conditions,
|
||||||
|
}),
|
||||||
|
projection = {
|
||||||
|
"sharedWith.$": True
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Return the result:
|
||||||
|
if record is None: return False
|
||||||
|
else: return record["sharedWith"][0]["permissions"][permission]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def download(
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
file_id: ObjectId
|
||||||
|
) -> io.BytesIO | None:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To quickly download small files. Do not use this for larger files because the file will be held in RAM first
|
||||||
|
and any large file will end up filling RAM fast. It's okay for smaller files that won't block up the memory.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param file_id: The id of the file to fetch.
|
||||||
|
:return: The file in a BytesIO buffer, or None if the file doesn't exist.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Get the file from the database:
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
success = await mongo_conn.easy_download(
|
||||||
|
destination = buffer,
|
||||||
|
file_id = ObjectId(file_id),
|
||||||
|
raise_exception = True
|
||||||
|
)
|
||||||
|
buffer.seek(0)
|
||||||
|
|
||||||
|
# Return the result:
|
||||||
|
if not success: return None
|
||||||
|
else: return buffer
|
||||||
|
|
||||||
|
async def download_from_stream(
|
||||||
|
self,
|
||||||
|
mongo_conn: AsyncMongoStorage,
|
||||||
|
file_id: ObjectId
|
||||||
|
) -> Any:
|
||||||
|
|
||||||
|
"""
|
||||||
|
To download any file as a stream. Better than the simple 'download' method because it doesn't block RAM.
|
||||||
|
:param mongo_conn: The instance of the database connection to perform this action.
|
||||||
|
:param file_id: The id of the file to fetch.
|
||||||
|
:return: YET TO BE IMPLEMENTED.
|
||||||
|
"""
|
||||||
|
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MAIN PROGRAM ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import time
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
|
||||||
|
files_mongo = AsyncMongoStorage(
|
||||||
|
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
|
||||||
|
database_name = "converseStore",
|
||||||
|
max_connections = 10,
|
||||||
|
debug = True
|
||||||
|
)
|
||||||
|
await files_mongo.connect()
|
||||||
|
|
||||||
|
my_fs = FileManagementModel()
|
||||||
|
|
||||||
|
start_time = time.time()
|
||||||
|
|
||||||
|
tasks = [my_fs.has_permission(
|
||||||
|
mongo_conn = files_mongo,
|
||||||
|
user_info = CoreUserInfoModel(
|
||||||
|
userId = _,
|
||||||
|
# industry = "fashion"
|
||||||
|
),
|
||||||
|
file_id = "67583b64f35e2c7d3cab7955",
|
||||||
|
permission = "read"
|
||||||
|
) for _ in range(1_000)]
|
||||||
|
results = await asyncio.gather(*tasks)
|
||||||
|
for i, r in enumerate(results):
|
||||||
|
print(f"{i: >4}: {r}")
|
||||||
|
|
||||||
|
print(f"Done in {time.time() - start_time:.5f} seconds")
|
||||||
|
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
@@ -197,6 +197,7 @@ class CoreAuthTokenModel(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
from utils_v2.string import json
|
from utils_v2.string import json
|
||||||
|
|
||||||
auth_token = CoreAuthTokenModel(
|
auth_token = CoreAuthTokenModel(
|
||||||
|
|||||||
@@ -0,0 +1,284 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
AUTHOR:
|
||||||
|
|
||||||
|
Khushal P Soonderji
|
||||||
|
|
||||||
|
DATE:
|
||||||
|
|
||||||
|
Tuesday, 10th Dec., 2024.
|
||||||
|
|
||||||
|
OBJECTIVE:
|
||||||
|
|
||||||
|
To define how information about files will be stored in the database.
|
||||||
|
|
||||||
|
REFERENCES:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
DOWNLOADS:
|
||||||
|
|
||||||
|
N/A
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** IMPORT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# To make sibling directories accessible for imports:
|
||||||
|
import sys
|
||||||
|
sys.path.append(".")
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
# For making data behaviour_models:
|
||||||
|
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
|
||||||
|
from typing import Optional, Literal, Union, Any, Dict, List
|
||||||
|
|
||||||
|
# My utils:
|
||||||
|
from utils_v2.string import regex
|
||||||
|
from utils_v2.date_time import date_time
|
||||||
|
|
||||||
|
# To work with MongoDB:
|
||||||
|
from bson.objectid import ObjectId
|
||||||
|
|
||||||
|
# Other data models:
|
||||||
|
from models.data.core.user import CoreUserInfoModel
|
||||||
|
|
||||||
|
# To work with date and time:
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MACROS / ONE-TIME INIT ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** VARIABLES ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
# --- Nothing Yet
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** FUNCTIONS ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
class CoreFilePermissionsModel(BaseModel):
|
||||||
|
|
||||||
|
read: bool = Field(
|
||||||
|
description = "grants permissions to read/view this file",
|
||||||
|
frozen = True,
|
||||||
|
default = True
|
||||||
|
)
|
||||||
|
|
||||||
|
delete: bool = Field(
|
||||||
|
description = "grants permissions to delete this file entirely",
|
||||||
|
frozen = True,
|
||||||
|
default = False
|
||||||
|
)
|
||||||
|
|
||||||
|
changePermissions: bool = Field(
|
||||||
|
description = "grants permissions to modify the permissions of this file",
|
||||||
|
frozen = True,
|
||||||
|
default = False
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "allow"
|
||||||
|
arbitrary_types_allowed = True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class CoreFileSharingModel(BaseModel):
|
||||||
|
|
||||||
|
user: CoreUserInfoModel = Field(
|
||||||
|
description = "to identify the user who has access to this file",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
permissions: CoreFilePermissionsModel = Field(
|
||||||
|
description = "the permissions that the above mentioned user has to this file",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "allow"
|
||||||
|
arbitrary_types_allowed = True
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class CoreFileInfoModel(BaseModel):
|
||||||
|
|
||||||
|
version: str = Field(
|
||||||
|
description = "a hint about the version no. of this message",
|
||||||
|
min_length = 1,
|
||||||
|
frozen = True,
|
||||||
|
default = "2.0.0"
|
||||||
|
)
|
||||||
|
|
||||||
|
user: CoreUserInfoModel = Field(
|
||||||
|
description = "to identify the user who owns this file",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
filename: str = Field(
|
||||||
|
description = "the name of this file",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
uploadTs: AwareDatetime = Field(
|
||||||
|
description = "the time (utc) at which this message was sent by the sender",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
length: int = Field(
|
||||||
|
description = "to note when the user has marked this message as unread",
|
||||||
|
frozen = False,
|
||||||
|
default = False
|
||||||
|
)
|
||||||
|
|
||||||
|
hash: str = Field(
|
||||||
|
description = "a simple hash to verify the integrity of the uploaded data",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
metadata: Dict[str, Any] = Field(
|
||||||
|
description = "any addition data about this file to filter it later",
|
||||||
|
frozen = False,
|
||||||
|
default = {}
|
||||||
|
)
|
||||||
|
|
||||||
|
tags: List[str] = Field(
|
||||||
|
description = "a list of keywords to apply to this file to filter it later",
|
||||||
|
frozen = False,
|
||||||
|
default = [],
|
||||||
|
examples = ["Bank Statement", "PDF", "bhopli@orange.com"]
|
||||||
|
)
|
||||||
|
|
||||||
|
parentId: ObjectId | None = Field(
|
||||||
|
description = "to identify the parent directory of this file; null means root directory",
|
||||||
|
frozen = False
|
||||||
|
)
|
||||||
|
|
||||||
|
isPrivate: bool = Field(
|
||||||
|
description = "whether, or not, this file is a private file",
|
||||||
|
frozen = True
|
||||||
|
)
|
||||||
|
|
||||||
|
sharedWith: List[CoreFileSharingModel] = Field(
|
||||||
|
description = "sharing settings; specially relevant when the file is private",
|
||||||
|
frozen = True,
|
||||||
|
default = []
|
||||||
|
)
|
||||||
|
|
||||||
|
# ┏┓ ┏•
|
||||||
|
# ┃ ┏┓┏┓╋┓┏┓
|
||||||
|
# ┗┛┗┛┛┗┛┗┗┫
|
||||||
|
# ┛
|
||||||
|
|
||||||
|
class Config:
|
||||||
|
extra = "allow"
|
||||||
|
arbitrary_types_allowed = True
|
||||||
|
|
||||||
|
# ┓┏ ┓• ┓ •
|
||||||
|
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
|
||||||
|
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
|
||||||
|
|
||||||
|
@field_validator("uploadTs", mode = "before")
|
||||||
|
def parse_date_time(cls, value):
|
||||||
|
return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC)
|
||||||
|
|
||||||
|
@field_validator("parentId", mode = "before")
|
||||||
|
def parse_oid(cls, value):
|
||||||
|
try: value = ObjectId(value)
|
||||||
|
except: pass
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
# ***** ****
|
||||||
|
# *** MAIN PROGRAM ***
|
||||||
|
# ***** ****
|
||||||
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
from utils_v2.string import json
|
||||||
|
|
||||||
|
file_info = CoreFileInfoModel(
|
||||||
|
user = CoreUserInfoModel(
|
||||||
|
fullName = "Bhopli Narangi",
|
||||||
|
userId = 1,
|
||||||
|
entityId = 2,
|
||||||
|
billingAccountId = 3,
|
||||||
|
departmentId = 4,
|
||||||
|
branchId = 5,
|
||||||
|
industry = "fashion"
|
||||||
|
),
|
||||||
|
filename = "Giga-Cat.png",
|
||||||
|
uploadTs = date_time.get_current_utc_date_time(as_string = False),
|
||||||
|
length = 1024,
|
||||||
|
hash = "abcdefgh",
|
||||||
|
metadata = {
|
||||||
|
"camera": "iPhone 1000 Pro Max XS"
|
||||||
|
},
|
||||||
|
tags = [
|
||||||
|
"image",
|
||||||
|
"cute",
|
||||||
|
"cat"
|
||||||
|
],
|
||||||
|
parentId = "67519cf3a7804fcbc6f12452",
|
||||||
|
isPrivate = True,
|
||||||
|
sharedWith = [
|
||||||
|
CoreFileSharingModel(
|
||||||
|
user = CoreUserInfoModel(
|
||||||
|
fullName = "Polki Muchhwaali",
|
||||||
|
userId = 6,
|
||||||
|
entityId = 7,
|
||||||
|
billingAccountId = 8,
|
||||||
|
departmentId = 9,
|
||||||
|
branchId = 10,
|
||||||
|
industry = "entertainment"
|
||||||
|
),
|
||||||
|
permissions = CoreFilePermissionsModel(
|
||||||
|
read = True,
|
||||||
|
delete = False,
|
||||||
|
changePermissions = False
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
print("FILE INFO MODEL:", json.to_string(file_info.model_dump(), default = str))
|
||||||
@@ -91,12 +91,24 @@ class CoreMessageModel(BaseModel):
|
|||||||
frozen = True
|
frozen = True
|
||||||
)
|
)
|
||||||
|
|
||||||
readTs: AwareDatetime = Field(
|
syncTs: AwareDatetime = Field(
|
||||||
description = "the time (utc) at which this message was read and stored by your server",
|
description = "the time (utc) at which this message was pulled and stored in your server",
|
||||||
frozen = True,
|
frozen = True,
|
||||||
default_factory = lambda: date_time.get_current_utc_date_time(as_string = False)
|
default_factory = lambda: date_time.get_current_utc_date_time(as_string = False)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
readTs: AwareDatetime | None = Field(
|
||||||
|
description = "the time (utc) at which this message was read by the user",
|
||||||
|
frozen = True,
|
||||||
|
default = None
|
||||||
|
)
|
||||||
|
|
||||||
|
markedAsUnread: bool = Field(
|
||||||
|
description = "to note when the user has marked this message as unread",
|
||||||
|
frozen = False,
|
||||||
|
default = False
|
||||||
|
)
|
||||||
|
|
||||||
tokenId: ObjectId = Field(
|
tokenId: ObjectId = Field(
|
||||||
description = "the id of the auth token that is associated with this message",
|
description = "the id of the auth token that is associated with this message",
|
||||||
frozen = True
|
frozen = True
|
||||||
@@ -127,10 +139,9 @@ class CoreMessageModel(BaseModel):
|
|||||||
default = None
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
isInward: bool = Field(
|
isSent: bool = Field(
|
||||||
description = "to understand whether this message was an inward message or outward message",
|
description = "to understand whether this message was an incoming message or outgoing message",
|
||||||
frozen = True,
|
frozen = True
|
||||||
default = True
|
|
||||||
)
|
)
|
||||||
|
|
||||||
isBroadcast: bool = Field(
|
isBroadcast: bool = Field(
|
||||||
@@ -145,7 +156,7 @@ class CoreMessageModel(BaseModel):
|
|||||||
default = False
|
default = False
|
||||||
)
|
)
|
||||||
|
|
||||||
payload: dict = Field(
|
message: dict = Field(
|
||||||
description = "the actual contents of the message; will differ for each client",
|
description = "the actual contents of the message; will differ for each client",
|
||||||
frozen = True
|
frozen = True
|
||||||
)
|
)
|
||||||
@@ -163,7 +174,7 @@ class CoreMessageModel(BaseModel):
|
|||||||
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
|
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
|
||||||
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
|
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
|
||||||
|
|
||||||
@field_validator("ts", "readTs", mode = "before")
|
@field_validator("ts", "syncTs", "readTs", mode = "before")
|
||||||
def parse_date_time(cls, value):
|
def parse_date_time(cls, value):
|
||||||
return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC)
|
return date_time.parse_date_time(input_value = value, timezone = date_time.TIMEZONE_UTC)
|
||||||
|
|
||||||
@@ -191,7 +202,8 @@ if __name__ == "__main__":
|
|||||||
client = "gmail",
|
client = "gmail",
|
||||||
clientMessageId = 123,
|
clientMessageId = 123,
|
||||||
clientThreadId = 456,
|
clientThreadId = 456,
|
||||||
payload = {
|
isSent = False,
|
||||||
|
message = {
|
||||||
"from": "bhopli@gmil.com",
|
"from": "bhopli@gmil.com",
|
||||||
"to": "hello@thecaoffice.com",
|
"to": "hello@thecaoffice.com",
|
||||||
"message": "Hello, World!"
|
"message": "Hello, World!"
|
||||||
|
|||||||
@@ -89,37 +89,44 @@ class CoreUserInfoModel(BaseModel):
|
|||||||
fullName: str | None = Field(
|
fullName: str | None = Field(
|
||||||
description = "the full name of the user as found in the database",
|
description = "the full name of the user as found in the database",
|
||||||
frozen = True,
|
frozen = True,
|
||||||
|
default = None,
|
||||||
examples = ["Bhopli Narangi"]
|
examples = ["Bhopli Narangi"]
|
||||||
)
|
)
|
||||||
|
|
||||||
userId: int | str | None = Field(
|
userId: int | str | None = Field(
|
||||||
description = "the id of the user as found in the database",
|
description = "the id of the user as found in the database",
|
||||||
frozen = True
|
frozen = True,
|
||||||
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
entityId: int | str | None = Field(
|
entityId: int | str | None = Field(
|
||||||
description = "the id of the entity with which this user is associated",
|
description = "the id of the entity with which this user is associated",
|
||||||
frozen = True
|
frozen = True,
|
||||||
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
billingAccountId: int | str | None = Field(
|
billingAccountId: int | str | None = Field(
|
||||||
description = "the id of the billing account with which this user is associated",
|
description = "the id of the billing account with which this user is associated",
|
||||||
frozen = True
|
frozen = True,
|
||||||
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
departmentId: int | str | None = Field(
|
departmentId: int | str | None = Field(
|
||||||
description = "the id of the dept. in which this user is working",
|
description = "the id of the dept. in which this user is working",
|
||||||
frozen = True
|
frozen = True,
|
||||||
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
branchId: int | str | None = Field(
|
branchId: int | str | None = Field(
|
||||||
description = "the id of the branch in which this user is working",
|
description = "the id of the branch in which this user is working",
|
||||||
frozen = True
|
frozen = True,
|
||||||
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
industry: str | None = Field(
|
industry: str | None = Field(
|
||||||
description = "the name of the industry this user is working in",
|
description = "the name of the industry this user is working in",
|
||||||
frozen = True
|
frozen = True,
|
||||||
|
default = None
|
||||||
)
|
)
|
||||||
|
|
||||||
# ┏┓ ┏•
|
# ┏┓ ┏•
|
||||||
@@ -140,4 +147,17 @@ class CoreUserInfoModel(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
pass
|
|
||||||
|
from utils_v2.string import json
|
||||||
|
|
||||||
|
user_info = CoreUserInfoModel(
|
||||||
|
fullName = "Bhopli Narangi",
|
||||||
|
userId = 1,
|
||||||
|
entityId = 2,
|
||||||
|
billingAccountId = 3,
|
||||||
|
departmentId = 4,
|
||||||
|
branchId = 5,
|
||||||
|
industry = "fashion"
|
||||||
|
)
|
||||||
|
|
||||||
|
print("USER INFO MODEL:", json.to_string(user_info.model_dump(), default = str))
|
||||||
+118
-35
@@ -49,17 +49,23 @@ from utils_v2.date_time import date_time
|
|||||||
|
|
||||||
# To work with mails:
|
# To work with mails:
|
||||||
import email
|
import email
|
||||||
|
from email.message import Message
|
||||||
|
from email.utils import parsedate_tz
|
||||||
|
from email.utils import parseaddr
|
||||||
|
|
||||||
# To parse the HTML content in the mail:
|
# To parse the HTML content in the mail:
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
# To work with datatypes:
|
# To work with datatypes:
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict, List, Literal
|
||||||
|
|
||||||
# To work with various encodings:
|
# To work with various encodings:
|
||||||
import base64
|
import base64
|
||||||
import quopri
|
import quopri
|
||||||
|
|
||||||
|
# To work with date and time:
|
||||||
|
import datetime
|
||||||
|
import pytz
|
||||||
import time
|
import time
|
||||||
|
|
||||||
|
|
||||||
@@ -90,12 +96,93 @@ import time
|
|||||||
# *****************************************************************************************************************
|
# *****************************************************************************************************************
|
||||||
|
|
||||||
|
|
||||||
|
def parse_addr(addr_header: str) -> List[Dict[str, str]]:
|
||||||
|
|
||||||
|
# If the field is null, we return null:
|
||||||
|
if addr_header is None: return []
|
||||||
|
|
||||||
|
# Create an empty variable that will hold the results:
|
||||||
|
addrs = []
|
||||||
|
|
||||||
|
# Iterate through the addresses and parse them:
|
||||||
|
for a in addr_header.split(","):
|
||||||
|
n, e = parseaddr(a.strip())
|
||||||
|
addrs.append({
|
||||||
|
"name": n.strip() or e.strip(),
|
||||||
|
"email": e.strip()
|
||||||
|
})
|
||||||
|
|
||||||
|
# Done here:
|
||||||
|
return addrs
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def parse_date(date_header: str) -> datetime.datetime | None:
|
||||||
|
|
||||||
|
# Try to parse the date header:
|
||||||
|
date_tuple = parsedate_tz(date_header)
|
||||||
|
|
||||||
|
# If the date header was parsed successfully, we assemble
|
||||||
|
# the parts to get an aware object in UTC timezone:
|
||||||
|
if date_tuple:
|
||||||
|
dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60)))
|
||||||
|
dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC)
|
||||||
|
return dt
|
||||||
|
|
||||||
|
# In case of an invalid date header:
|
||||||
|
else: return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def decode_payload(
|
||||||
|
raw_payload: str | bytes,
|
||||||
|
content_main_type: str,
|
||||||
|
content_charset: str | None,
|
||||||
|
content_transfer_encoding: Literal[None, "base64", "quoted-printable"]
|
||||||
|
) -> str | bytes:
|
||||||
|
|
||||||
|
# Start by assuming nothing needs to be done:
|
||||||
|
payload = raw_payload
|
||||||
|
|
||||||
|
# We decode various kinds of parts:
|
||||||
|
match content_transfer_encoding:
|
||||||
|
|
||||||
|
# This is just unencoded plaintext:
|
||||||
|
case None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Typically see with attachments:
|
||||||
|
case "base64":
|
||||||
|
charset = content_charset or "utf-8"
|
||||||
|
payload = raw_payload
|
||||||
|
payload = base64.b64decode(payload)
|
||||||
|
if content_main_type == "text": payload = payload.decode(charset)
|
||||||
|
|
||||||
|
# Typically seen with HTML parts:
|
||||||
|
case "quoted-printable":
|
||||||
|
charset = content_charset or "utf-8"
|
||||||
|
payload = raw_payload.encode(charset)
|
||||||
|
payload = quopri.decodestring(payload)
|
||||||
|
if content_main_type == "text": payload = payload.decode(charset)
|
||||||
|
|
||||||
|
# Done here:
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
def parse_part(
|
def parse_part(
|
||||||
# part: email.message.Message
|
part: Message | List[Message]
|
||||||
part
|
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
|
|
||||||
|
# Start by extracting basic details:
|
||||||
part_json = {
|
part_json = {
|
||||||
|
"boundary": part.get_boundary(),
|
||||||
"contentType": part.get_content_type(),
|
"contentType": part.get_content_type(),
|
||||||
"contentMainType": part.get_content_maintype(),
|
"contentMainType": part.get_content_maintype(),
|
||||||
"contentSubType": part.get_content_subtype(),
|
"contentSubType": part.get_content_subtype(),
|
||||||
@@ -103,31 +190,19 @@ def parse_part(
|
|||||||
"contentTransferEncoding": part.get("Content-Transfer-Encoding"),
|
"contentTransferEncoding": part.get("Content-Transfer-Encoding"),
|
||||||
"contentDisposition": part.get_content_disposition(),
|
"contentDisposition": part.get_content_disposition(),
|
||||||
"filename": part.get_filename(),
|
"filename": part.get_filename(),
|
||||||
"contentId": part.get("Content-ID"),
|
"contentId": part.get("Content-ID")
|
||||||
"payload": part.get_payload(decode = False)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# We decode various kinds of parts:
|
# Process the payload of this part:
|
||||||
match part_json["contentTransferEncoding"]:
|
if part_json["contentMainType"] == "multipart":
|
||||||
|
part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)]
|
||||||
# This is just unencoded plaintext:
|
else:
|
||||||
case None: pass
|
part_json["payload"] = decode_payload(
|
||||||
|
raw_payload = part.get_payload(decode = False),
|
||||||
# Typically see with attachments:
|
content_main_type = part_json["contentMainType"],
|
||||||
case "base64":
|
content_charset = part_json["contentCharset"],
|
||||||
charset = part_json["contentCharset"] or "utf-8"
|
content_transfer_encoding = part_json["contentTransferEncoding"]
|
||||||
payload = part_json["payload"]
|
)
|
||||||
payload = base64.b64decode(payload)
|
|
||||||
part_json["payload"] = payload
|
|
||||||
|
|
||||||
# Typically seen with HTML parts:
|
|
||||||
case "quoted-printable":
|
|
||||||
charset = part_json["contentCharset"] or "utf-8"
|
|
||||||
payload = part_json["payload"].encode(charset)
|
|
||||||
payload = quopri.decodestring(payload)
|
|
||||||
part_json["payload"] = payload.decode(charset)
|
|
||||||
|
|
||||||
# print("PARSED PART:", json.to_string(part_json, default = str))
|
|
||||||
|
|
||||||
# Done here;
|
# Done here;
|
||||||
return part_json
|
return part_json
|
||||||
@@ -152,18 +227,26 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
|
|||||||
if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
|
if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
|
||||||
else: parsed_mail = email.message_from_bytes(raw_mail)
|
else: parsed_mail = email.message_from_bytes(raw_mail)
|
||||||
|
|
||||||
# Make variables:
|
# Extract the most basic details:
|
||||||
parts = []
|
mail_json = {
|
||||||
attachments = []
|
"ts": parse_date(parsed_mail["Date"]),
|
||||||
|
"headers": {k: v for k, v in parsed_mail.items()},
|
||||||
|
"from": parse_addr(parsed_mail["From"]),
|
||||||
|
"to": parse_addr(parsed_mail["To"]),
|
||||||
|
"cc": parse_addr(parsed_mail["Cc"]),
|
||||||
|
"bcc": parse_addr(parsed_mail["Bcc"]),
|
||||||
|
"subject": parsed_mail["Subject"],
|
||||||
|
"payload": None
|
||||||
|
}
|
||||||
|
|
||||||
# Iterate through each part of the mail for multipart mails:
|
# Iterate through each part of the mail for multipart mails:
|
||||||
if parsed_mail.is_multipart():
|
if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail)
|
||||||
for part in parsed_mail.walk():
|
|
||||||
part_json = parse_part(part)
|
|
||||||
if part_json["contentDisposition"] in ["inline", "attachment"]: attachments.append()
|
|
||||||
|
|
||||||
# When the mails are not multipart, just plaintext:
|
# When the mails are not multipart, just plaintext:
|
||||||
else: print("PLAINTEXT PART:", parsed_mail.get_payload())
|
else: mail_json["payload"] = parsed_mail.get_payload()
|
||||||
|
|
||||||
|
# Done here:
|
||||||
|
return mail_json
|
||||||
|
|
||||||
|
|
||||||
# *****************************************************************************************************************
|
# *****************************************************************************************************************
|
||||||
@@ -180,4 +263,4 @@ if __name__ == "__main__":
|
|||||||
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
|
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
|
||||||
parse_results = parse(mail_string_raw)
|
parse_results = parse(mail_string_raw)
|
||||||
|
|
||||||
# print(json.to_string(parse_results, default = str))
|
print(json.to_string(parse_results, default = str))
|
||||||
|
|||||||
Reference in New Issue
Block a user