diff --git a/api/blueprints/mail/sync_v2.py b/api/blueprints/mail/sync_v2.py index 0363155..2c1ad3e 100644 --- a/api/blueprints/mail/sync_v2.py +++ b/api/blueprints/mail/sync_v2.py @@ -142,7 +142,7 @@ async def sync_mails( """ # Try to sync the mails: - return await current_app.mail_api_model.sync( + return await current_app.mail_controller.sync( db_conn = current_app.sql_writer, mongo_conn = current_app.data_mongo, user_info = user_info, diff --git a/api/main.py b/api/main.py index a1b1544..b7ca673 100644 --- a/api/main.py +++ b/api/main.py @@ -94,7 +94,7 @@ from icecream import IceCreamDebugger # All the blueprints: from api.blueprints.mail.oauth_request import mail_oauth_bp from api.blueprints.mail.oauth_callback import mail_callback_bp -# from api.blueprints.mail.sync_v2 import mail_sync_bp +from api.blueprints.mail.sync_v2 import mail_sync_bp # from api.blueprints.mail.list import mail_list_bp # from api.blueprints.mail.retrieve import mail_retrieve_bp # from api.blueprints.sms.auth import sms_auth_bp @@ -133,7 +133,7 @@ app = Quart(__name__, template_folder = r"../views") app = cors(app) app.register_blueprint(mail_oauth_bp, url_prefix = f"/{MODULE_BASE}/mail") app.register_blueprint(mail_callback_bp, url_prefix = f"/{MODULE_BASE}/mail") -# app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail") +app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail") # app.register_blueprint(mail_list_bp, url_prefix = f"/{MODULE_BASE}/mail") # app.register_blueprint(mail_retrieve_bp, url_prefix = f"/{MODULE_BASE}/mail") # app.register_blueprint(sms_auth_bp, url_prefix = f"/{MODULE_BASE}/sms") diff --git a/controllers/api/mail.py b/controllers/api/mail.py index 2e466e6..be04de5 100644 --- a/controllers/api/mail.py +++ b/controllers/api/mail.py @@ -213,6 +213,26 @@ class MailController: ) ) + def drop_attachments( + self, + payload: dict + ): + + # If the part is some sort of file: + if payload["contentMainType"] not in ["multipart", "text"]: + payload["payload"] = None + payload["payloadId"] = None + payload["payloadUrl"] = None + + # If the payload is of multipart type, + # we use recursion to look inside it: + elif payload["contentMainType"] == "multipart": + for part in payload["payload"]: + self.drop_attachments(part) + + # Done here: + return payload + # ┏┓┏┓ ┓ ┏┓ ┏┓ # ┃┃┣┫┓┏╋┣┓┏┛ ┃┫ # ┗┛┛┗┗┻┗┛┗┗━•┗┛ @@ -300,7 +320,7 @@ class MailController: # If we've not been forced to re-sync the mail message, # we first check if the mail already exists in our database: if not force_sync: - mail_record = await current_app.core_message_controller.get_previews( + mail_records = await current_app.core_message_controller.get_previews( mongo_conn = mongo_conn, token_ids = [ObjectId(token_id)], limit = 1, @@ -312,9 +332,12 @@ class MailController: "clientMessageId": message_id } ) - if mail_record: + if mail_records: sync_result.success = True - sync_result.message = f"gmail message '{message_id}' already sync'd on '{mail_record['readTs']} (UTC)'" + sync_result.message = ( + f"gmail message '{message_id}' already " + f"sync'd on '{mail_records[0].syncTs} (UTC)'" + ) return sync_result # Now that we know that we have to fetch the mail from GMail: @@ -330,7 +353,7 @@ class MailController: return sync_result # HANDLE ATTACHMENTS HERE: - pass + client_response.data["payload"] = self.drop_attachments(client_response.data["payload"]) # Now we structure the message into the model: mail_message = CoreMessageModel( @@ -341,7 +364,8 @@ class MailController: client = auth_token.client, clientMessageId = message_id, clientThreadId = client_response.data["threadId"], - payload = client_response.data + preview = client_response.data["subject"], + message = client_response.data ) # Give a quick indicator of whether this mail is an inbox mail or sent mail: @@ -351,16 +375,17 @@ class MailController: else: mail_message.isSent = True # Invoke the LLM: - mail_message.aiSnippet = await self.summarize_mail_with_ai( + ai_snippet = await self.summarize_mail_with_ai( mongo_conn = mongo_conn, user_info = user_info, llm = llm, message = mail_message ) + mail_message.aiSnippet = ai_snippet.summary # Done here: - print("ONE MAIL:", json.to_string(mail_message.model_dump(), default = str)) sync_result.success = True + sync_result.mailMessage = mail_message return sync_result async def __sync_many_gmail( @@ -415,7 +440,7 @@ class MailController: query = query_string ) if not client_response.success: - sync_results["message"] = f"gmail: {client_response.message}" + sync_results.message = f"gmail: {client_response.message}" return sync_results messages_list = client_response.data["messages"] @@ -442,22 +467,27 @@ class MailController: for result in individual_sync_results: if result.success: sync_results.successCount += 1 else: sync_results.failureCount += 1 - if result.mailMessage: mongo_operations.append(ReplaceOne( - filter = { - "tokenId": token_id, - "serviceType": auth_token.serviceType, - "client": auth_token.client, - "clientMessageId": result.mailMessage.clientMessageId - }, - replacement = result.mailMessage.model_dump(), - upsert = True - )) + if result.mailMessage: + replacement_json = result.mailMessage.model_dump() + replacement_json.pop("_id", None) + mongo_operations.append( + ReplaceOne( + filter = { + "tokenId": ObjectId(token_id), + "serviceType": auth_token.serviceType, + "client": auth_token.client, + "clientMessageId": result.mailMessage.clientMessageId + }, + replacement = replacement_json, + upsert = True + ) + ) # Make the bulk insert operation: if mongo_operations: sync_count = await current_app.core_message_controller.bulk_write( mongo_conn = mongo_conn, - requests = mongo_operations + mongo_operations = mongo_operations ) # Apply the labels to the read messages: @@ -496,7 +526,7 @@ class MailController: # ┻ ┗ ┗┗┛┗ ┻ ┗┛┛┗┗ ┛┗┛ # We first load the authorization tokens: - auth_token = await current_app.mail_oauth_model.get_token( + auth_token = await self.get_token( mongo_conn = mongo_conn, token_id = token_id, ) @@ -556,5 +586,5 @@ if __name__ == "__main__": # raw_mail_json = json.from_file(file_options[1]) # print("FROM FILE:", json.to_string(raw_mail_json["payload"])) # print("\n\n---------\n\n") - # mail_model = MailAPIModel() - # print(mail_model.extract_plaintext_parts(raw_mail_json["payload"])) + # mail_controller = MailController() + # print(json.to_string(mail_controller.drop_attachments(raw_mail_json["payload"]))) diff --git a/controllers/core/ai/llm.py b/controllers/core/ai/llm.py index 8fd772d..30b466d 100644 --- a/controllers/core/ai/llm.py +++ b/controllers/core/ai/llm.py @@ -143,8 +143,6 @@ class LLMController(BaseModel): # Invoke the AI, and format the response: llm_response = await self.__llm.ainvoke(prompt) - print("LLM RESPONSE:", llm_response) - print("INPUT MESSAGES:", llm_input.messages) llm_response = LLMOutput( messages = llm_input.messages, output = llm_response.content, diff --git a/controllers/core/message.py b/controllers/core/message.py index dfa9151..20bf8fa 100644 --- a/controllers/core/message.py +++ b/controllers/core/message.py @@ -144,7 +144,8 @@ class MessageController(BaseModel): return await mongo_conn.bulk_write( collection = self.MESSAGES_COLLECTION, - requests = mongo_operations + requests = mongo_operations, + raise_exception = True ) # ┏┓┳┓┳┳┳┓ ┳┓ • @@ -222,6 +223,7 @@ class MessageController(BaseModel): projection = { "_id": True, "ts": True, + "tokenId": True, "markedAsUnread": True, "serviceType": True, "client": True, @@ -232,7 +234,6 @@ class MessageController(BaseModel): "sentSuccessfully": True, "aiSnippet": True, "preview": True, - "message": {}, "tags": True, "usedAi": True }, @@ -240,6 +241,7 @@ class MessageController(BaseModel): ) # Convert the fetched records to instances of the data model and return: + for record in records: record["message"] = {} return [CoreMessageModel(**record) for record in records] async def get_messages( diff --git a/models/api/mail/sync.py b/models/api/mail/sync.py index 7f26a6c..d6902c1 100644 --- a/models/api/mail/sync.py +++ b/models/api/mail/sync.py @@ -36,7 +36,7 @@ sys.path.append(".") sys.path.append("..") # For making data behaviour_models: -from pydantic import BaseModel, Field, field_validator, PastDatetime +from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime from typing import Optional, Literal # My utils: @@ -117,13 +117,13 @@ class MailSyncRequestData(BaseModel): frozen = True ) - startDate: PastDatetime = Field( + startDate: AwareDatetime = Field( description = "the starting date from which the user wants to sync their mail", default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1), frozen = True ) - endDate: PastDatetime = Field( + endDate: AwareDatetime = Field( description = "the ending date till which the user wants to sync their mail", default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1), frozen = True diff --git a/models/core/ai/llm.py b/models/core/ai/llm.py index 0989246..cc43755 100644 --- a/models/core/ai/llm.py +++ b/models/core/ai/llm.py @@ -220,6 +220,19 @@ class LLMOutput(BaseModel): class Config: extra = "forbid" + # ┏┓ • + # ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏ + # ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛ + # ┛ + + @property + def summary(self): + return { + "output": self.output, + "tokens": self.tokens.model_dump(), + "invocationId": self.invocationId + } + # ***************************************************************************************************************** # ***** **** diff --git a/models/core/message.py b/models/core/message.py index 93904ea..e1a2aaa 100644 --- a/models/core/message.py +++ b/models/core/message.py @@ -160,9 +160,10 @@ class CoreMessageModel(BaseModel): default = False ) - aiSnippet: LLMOutput | None = Field( + aiSnippet: LLMOutput | dict | None = Field( description = "holds a short summary generated by ", - frozen = False + frozen = False, + default = None ) preview: str = Field( diff --git a/utils_v2/mail/mail_parser.py b/utils_v2/mail/mail_parser.py index 975df24..75b7db3 100644 --- a/utils_v2/mail/mail_parser.py +++ b/utils_v2/mail/mail_parser.py @@ -6,7 +6,7 @@ DATE: - Tuesday, 26th Nov., 2024 + Tuesday, 10th Dec., 2024 OBJECTIVE: @@ -18,6 +18,7 @@ 1. GitHub: https://github.com/SpamScope/mail-parser 2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322 3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045 + 4. StackOverflow: https://stackoverflow.com/questions/17874360/python-how-to-parse-the-body-from-a-raw-email-given-that-raw-email-does-not DOWNLOADS: @@ -47,18 +48,26 @@ from utils_v2.string import regex from utils_v2.date_time import date_time # To work with mails: -import mailparser +import email +from email.message import Message +from email.utils import parsedate_tz +from email.utils import parseaddr # To parse the HTML content in the mail: from bs4 import BeautifulSoup # To work with datatypes: -from typing import Any, Dict +from typing import Any, Dict, List, Literal # To work with various encodings: import base64 import quopri +# To work with date and time: +import datetime +import pytz +import time + # ***************************************************************************************************************** # ***** **** @@ -87,46 +96,116 @@ import quopri # ***************************************************************************************************************** -def from_quoted_printable(text: str) -> str: +def parse_addr(addr_header: str) -> List[Dict[str, str]]: - decoded_text = "" - decoded_bytes = quopri.decodestring(text) - for encoding in ["utf-8", "utf-16", "utf-32", "latin1"]: - try: text = decoded_bytes.decode(encoding) - except: text = "" - if text.find("From") >= 0: - decoded_text = text - break - return decoded_text + # If the field is null, we return null: + if addr_header is None: return [] + + # Create an empty variable that will hold the results: + addrs = [] + + # Iterate through the addresses and parse them: + for a in addr_header.split(","): + n, e = parseaddr(a.strip()) + addrs.append({ + "name": n.strip() or e.strip(), + "email": e.strip() + }) + + # Done here: + return addrs # --------------------------------------------------------------------------------------------------------------------- -def find_in_raw_mail( - raw_mail: str, - text: str -) -> int: +def parse_date(date_header: str) -> datetime.datetime | None: - # We first treat it as un-encoded text: - offset = raw_mail.find(text) - if offset >= 0: return offset + # Try to parse the date header: + date_tuple = parsedate_tz(date_header) - # Then we try Base64 encoding: - offset = raw_mail.find(base64.b64encode(text.encode("utf-8")).decode("utf-8")) - if offset >= 0: return offset + # If the date header was parsed successfully, we assemble + # the parts to get an aware object in UTC timezone: + if date_tuple: + dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60))) + dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC) + return dt - # then we try Quoted-Printable encoding: - offset = from_quoted_printable(raw_mail).find(text) - print("MAIL:") - print(raw_mail) - print("\n\n\n---\n\n\n") - print("TEXT:") - print(quopri.encodestring(text.encode("utf-8")).decode("utf-8")) - if offset >= 0: return offset + # In case of an invalid date header: + else: return None - # Done here, even if nothing worked: - return offset + +# --------------------------------------------------------------------------------------------------------------------- + + +def decode_payload( + raw_payload: str | bytes, + content_main_type: str, + content_charset: str | None, + content_transfer_encoding: Literal[None, "base64", "quoted-printable"] +) -> str | bytes: + + # Start by assuming nothing needs to be done: + payload = raw_payload + + # We decode various kinds of parts: + match content_transfer_encoding: + + # This is just unencoded plaintext: + case None: + pass + + # Typically see with attachments: + case "base64": + charset = content_charset or "utf-8" + payload = raw_payload + payload = base64.b64decode(payload) + if content_main_type == "text": payload = payload.decode(charset) + + # Typically seen with HTML parts: + case "quoted-printable": + charset = content_charset or "utf-8" + payload = raw_payload.encode(charset) + payload = quopri.decodestring(payload) + if content_main_type == "text": payload = payload.decode(charset) + + # Done here: + return payload + + +# --------------------------------------------------------------------------------------------------------------------- + + +def parse_part( + part: Message | List[Message] +) -> Dict[str, Any]: + + # Start by extracting basic details: + part_json = { + "boundary": part.get_boundary(), + "contentType": part.get_content_type(), + "contentMainType": part.get_content_maintype(), + "contentSubType": part.get_content_subtype(), + "contentCharset": part.get_content_charset(), + "contentTransferEncoding": part.get("Content-Transfer-Encoding"), + "contentDisposition": part.get_content_disposition(), + "filename": part.get_filename(), + "contentId": part.get("Content-ID") + } + + # Process the payload of this part: + if part_json["contentMainType"] == "multipart": + part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)] + else: + part_json["payload"] = decode_payload( + raw_payload = part.get_payload(decode = False), + content_main_type = part_json["contentMainType"], + content_charset = part_json["contentCharset"], + content_transfer_encoding = part_json["contentTransferEncoding"] + ) + + # Done here; + return part_json # --------------------------------------------------------------------------------------------------------------------- @@ -145,78 +224,30 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]: """ # Parse the raw format: - if isinstance(raw_mail, str): parsed_mail = mailparser.parse_from_string(raw_mail) - else: parsed_mail = mailparser.parse_from_bytes(raw_mail) + if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail) + else: parsed_mail = email.message_from_bytes(raw_mail) - # print(parsed_mail.mail_json) - # return - - # Format the attachments: - message_attachments = [ - { - "filename": attachment["filename"], - "type": attachment["mail_content_type"], - "cid": regex.find_first(text = attachment["content-id"], pattern = r"(?<=<).*(?=>)"), - "rawCid": attachment["content-id"], - "contentDisposition": (cd := attachment["content-disposition"]), - "isInline": True if cd.lower().find("inline") >= 0 else False, - "charset": attachment["charset"], - "contentTransferEncoding": attachment["content_transfer_encoding"], - "payload": attachment["payload"] - } for attachment in parsed_mail.attachments - ] - - # print("PRINTING PARTS") - # print("LIBRARY PARTS:", type(parsed_mail)) - - # # Figure out which entity (text and HTML) came in which sequence. - # # The library doesn't give us any sequence info so we do some custom string processing here to figure out the order - # # in which to render the contents of the page. - # parts = [ - # # { - # # "partNo": None, - # # "offset": max( - # # parsed_mail.message_as_string.find(t), - # # parsed_mail.message_as_string.find(base64.b64encode(t.encode()).decode()) - # # ), - # # "type": "text/plain", - # # "data": t - # # } for t in parsed_mail.text_plain - # ] - # parts = parts + [ - # { - # "partNo": None, - # "offset": find_in_raw_mail(raw_mail = parsed_mail.message_as_string, text = h), - # "type": "text/html", - # "data": h - # } for h in parsed_mail.text_html - # ] - # parts = sorted(parts, key = lambda x: x["offset"]) - # for i, p in enumerate(parts): p["partNo"] = i - - # Get the unformatted text from everything in the mail: - unformatted_text = [] - for p in parsed_mail.text_html: - html_parser = BeautifulSoup(p, "html.parser") - unformatted_text.append(html_parser.get_text()) - - # Put everything together: - return { - "ts": date_time.to_timezone(parsed_mail.date, timezone = date_time.TIMEZONE_UTC), - "headers": parsed_mail.headers, - "from": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers["From"]], - "to": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers["To"]], - "cc": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers.get("Cc", [])], - "bcc": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers.get("Bcc", [])], - "subject": parsed_mail.headers["Subject"], - "text": parsed_mail.text_plain, - "html": parsed_mail.text_html, - # "parts": parts, - "unformattedText": "\n".join(unformatted_text), - "attachments": message_attachments, - "isInbox": None + # Extract the most basic details: + mail_json = { + "ts": parse_date(parsed_mail["Date"]), + "headers": {k: v for k, v in parsed_mail.items()}, + "from": parse_addr(parsed_mail["From"]), + "to": parse_addr(parsed_mail["To"]), + "cc": parse_addr(parsed_mail["Cc"]), + "bcc": parse_addr(parsed_mail["Bcc"]), + "subject": parsed_mail["Subject"], + "payload": None } + # Iterate through each part of the mail for multipart mails: + if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail) + + # When the mails are not multipart, just plaintext: + else: mail_json["payload"] = parsed_mail.get_payload() + + # Done here: + return mail_json + # ***************************************************************************************************************** # ***** **** @@ -229,7 +260,7 @@ if __name__ == "__main__": from utils_v2.system import files - mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail.txt") + mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt") parse_results = parse(mail_string_raw) print(json.to_string(parse_results, default = str)) diff --git a/utils_v2/mail/mail_parser_v2.py b/utils_v2/mail/mail_parser_v2.py deleted file mode 100644 index 75b7db3..0000000 --- a/utils_v2/mail/mail_parser_v2.py +++ /dev/null @@ -1,266 +0,0 @@ -""" - - AUTHOR: - - Khushal P Soonderji - - DATE: - - Tuesday, 10th Dec., 2024 - - OBJECTIVE: - - To parse raw mail bodies and give a structure that is suitable for storing in No-SQL databases like MongoDB. The - raw mail's text is expected to be compliant with standard defined in RFC 5322, RFC 2045, and maybe a few more. - - REFERENCES: - - 1. GitHub: https://github.com/SpamScope/mail-parser - 2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322 - 3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045 - 4. StackOverflow: https://stackoverflow.com/questions/17874360/python-how-to-parse-the-body-from-a-raw-email-given-that-raw-email-does-not - - DOWNLOADS: - - N/A - -""" - - -# ***************************************************************************************************************** -# ***** **** -# *** IMPORT *** -# ***** **** -# ***************************************************************************************************************** - - -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") - -# System-level activities: -import io - -# My utils: -from utils_v2.string import json -from utils_v2.string import regex -from utils_v2.date_time import date_time - -# To work with mails: -import email -from email.message import Message -from email.utils import parsedate_tz -from email.utils import parseaddr - -# To parse the HTML content in the mail: -from bs4 import BeautifulSoup - -# To work with datatypes: -from typing import Any, Dict, List, Literal - -# To work with various encodings: -import base64 -import quopri - -# To work with date and time: -import datetime -import pytz -import time - - -# ***************************************************************************************************************** -# ***** **** -# *** MACROS / ONE-TIME INIT *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** VARIABLES *** -# ***** **** -# ***************************************************************************************************************** - - -# --- Nothing Yet - - -# ***************************************************************************************************************** -# ***** **** -# *** FUNCTIONS *** -# ***** **** -# ***************************************************************************************************************** - - -def parse_addr(addr_header: str) -> List[Dict[str, str]]: - - # If the field is null, we return null: - if addr_header is None: return [] - - # Create an empty variable that will hold the results: - addrs = [] - - # Iterate through the addresses and parse them: - for a in addr_header.split(","): - n, e = parseaddr(a.strip()) - addrs.append({ - "name": n.strip() or e.strip(), - "email": e.strip() - }) - - # Done here: - return addrs - - -# --------------------------------------------------------------------------------------------------------------------- - - -def parse_date(date_header: str) -> datetime.datetime | None: - - # Try to parse the date header: - date_tuple = parsedate_tz(date_header) - - # If the date header was parsed successfully, we assemble - # the parts to get an aware object in UTC timezone: - if date_tuple: - dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60))) - dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC) - return dt - - # In case of an invalid date header: - else: return None - - -# --------------------------------------------------------------------------------------------------------------------- - - -def decode_payload( - raw_payload: str | bytes, - content_main_type: str, - content_charset: str | None, - content_transfer_encoding: Literal[None, "base64", "quoted-printable"] -) -> str | bytes: - - # Start by assuming nothing needs to be done: - payload = raw_payload - - # We decode various kinds of parts: - match content_transfer_encoding: - - # This is just unencoded plaintext: - case None: - pass - - # Typically see with attachments: - case "base64": - charset = content_charset or "utf-8" - payload = raw_payload - payload = base64.b64decode(payload) - if content_main_type == "text": payload = payload.decode(charset) - - # Typically seen with HTML parts: - case "quoted-printable": - charset = content_charset or "utf-8" - payload = raw_payload.encode(charset) - payload = quopri.decodestring(payload) - if content_main_type == "text": payload = payload.decode(charset) - - # Done here: - return payload - - -# --------------------------------------------------------------------------------------------------------------------- - - -def parse_part( - part: Message | List[Message] -) -> Dict[str, Any]: - - # Start by extracting basic details: - part_json = { - "boundary": part.get_boundary(), - "contentType": part.get_content_type(), - "contentMainType": part.get_content_maintype(), - "contentSubType": part.get_content_subtype(), - "contentCharset": part.get_content_charset(), - "contentTransferEncoding": part.get("Content-Transfer-Encoding"), - "contentDisposition": part.get_content_disposition(), - "filename": part.get_filename(), - "contentId": part.get("Content-ID") - } - - # Process the payload of this part: - if part_json["contentMainType"] == "multipart": - part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)] - else: - part_json["payload"] = decode_payload( - raw_payload = part.get_payload(decode = False), - content_main_type = part_json["contentMainType"], - content_charset = part_json["contentCharset"], - content_transfer_encoding = part_json["contentTransferEncoding"] - ) - - # Done here; - return part_json - - -# --------------------------------------------------------------------------------------------------------------------- - - -def parse(raw_mail: str | bytes) -> Dict[str, Any]: - - """ - To parse the raw mail text to a usable JSON that can even be stored on a No-SQL database like MongoDB. - DOCUMENTATION: - 1. GitHub: https://github.com/SpamScope/mail-parser - 2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322 - 3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045 - :param raw_mail: The raw mail body that adheres to RFC 5322 and RFC 2045 (among others). - :return: The parsed JSON format (dict) of the mail. - """ - - # Parse the raw format: - if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail) - else: parsed_mail = email.message_from_bytes(raw_mail) - - # Extract the most basic details: - mail_json = { - "ts": parse_date(parsed_mail["Date"]), - "headers": {k: v for k, v in parsed_mail.items()}, - "from": parse_addr(parsed_mail["From"]), - "to": parse_addr(parsed_mail["To"]), - "cc": parse_addr(parsed_mail["Cc"]), - "bcc": parse_addr(parsed_mail["Bcc"]), - "subject": parsed_mail["Subject"], - "payload": None - } - - # Iterate through each part of the mail for multipart mails: - if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail) - - # When the mails are not multipart, just plaintext: - else: mail_json["payload"] = parsed_mail.get_payload() - - # Done here: - return mail_json - - -# ***************************************************************************************************************** -# ***** **** -# *** MAIN PROGRAM *** -# ***** **** -# ***************************************************************************************************************** - - -if __name__ == "__main__": - - from utils_v2.system import files - - mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt") - parse_results = parse(mail_string_raw) - - print(json.to_string(parse_results, default = str))