khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20241212) Day-end push.

Browse Source
This commit is contained in:
khushal
2024-12-12 20:27:00 +05:30
parent e5d5c4c86d
commit e9a106dd91
10 changed files with 212 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/blueprints/mail/sync_v2.py
+1 -1
View File
@@ -142,7 +142,7 @@ async def sync_mails(
"""
# Try to sync the mails:
return await current_app.mail_api_model.sync(
return await current_app.mail_controller.sync(
db_conn = current_app.sql_writer,
mongo_conn = current_app.data_mongo,
user_info = user_info,
api/main.py
+2 -2
View File
@@ -94,7 +94,7 @@ from icecream import IceCreamDebugger
# All the blueprints:
from api.blueprints.mail.oauth_request import mail_oauth_bp
from api.blueprints.mail.oauth_callback import mail_callback_bp
# from api.blueprints.mail.sync_v2 import mail_sync_bp
from api.blueprints.mail.sync_v2 import mail_sync_bp
# from api.blueprints.mail.list import mail_list_bp
# from api.blueprints.mail.retrieve import mail_retrieve_bp
# from api.blueprints.sms.auth import sms_auth_bp
@@ -133,7 +133,7 @@ app = Quart(__name__, template_folder = r"../views")
app = cors(app)
app.register_blueprint(mail_oauth_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_callback_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
app.register_blueprint(mail_sync_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(mail_list_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(mail_retrieve_bp, url_prefix = f"/{MODULE_BASE}/mail")
# app.register_blueprint(sms_auth_bp, url_prefix = f"/{MODULE_BASE}/sms")
controllers/api/mail.py
+52 -22
View File
@@ -213,6 +213,26 @@ class MailController:
)
)
def drop_attachments(
self,
payload: dict
):
# If the part is some sort of file:
if payload["contentMainType"] not in ["multipart", "text"]:
payload["payload"] = None
payload["payloadId"] = None
payload["payloadUrl"] = None
# If the payload is of multipart type,
# we use recursion to look inside it:
elif payload["contentMainType"] == "multipart":
for part in payload["payload"]:
self.drop_attachments(part)
# Done here:
return payload
# ┏┓┏┓ ┓ ┏┓ ┏┓
# ┃┃┣┫┓┏╋┣┓┏┛ ┃┫
# ┗┛┛┗┗┻┗┛┗┗━•┗┛
@@ -300,7 +320,7 @@ class MailController:
# If we've not been forced to re-sync the mail message,
# we first check if the mail already exists in our database:
if not force_sync:
mail_record = await current_app.core_message_controller.get_previews(
mail_records = await current_app.core_message_controller.get_previews(
mongo_conn = mongo_conn,
token_ids = [ObjectId(token_id)],
limit = 1,
@@ -312,9 +332,12 @@ class MailController:
"clientMessageId": message_id
}
)
if mail_record:
if mail_records:
sync_result.success = True
sync_result.message = f"gmail message '{message_id}' already sync'd on '{mail_record['readTs']} (UTC)'"
sync_result.message = (
f"gmail message '{message_id}' already "
f"sync'd on '{mail_records[0].syncTs} (UTC)'"
)
return sync_result
# Now that we know that we have to fetch the mail from GMail:
@@ -330,7 +353,7 @@ class MailController:
return sync_result
# HANDLE ATTACHMENTS HERE:
pass
client_response.data["payload"] = self.drop_attachments(client_response.data["payload"])
# Now we structure the message into the model:
mail_message = CoreMessageModel(
@@ -341,7 +364,8 @@ class MailController:
client = auth_token.client,
clientMessageId = message_id,
clientThreadId = client_response.data["threadId"],
payload = client_response.data
preview = client_response.data["subject"],
message = client_response.data
)
# Give a quick indicator of whether this mail is an inbox mail or sent mail:
@@ -351,16 +375,17 @@ class MailController:
else: mail_message.isSent = True
# Invoke the LLM:
mail_message.aiSnippet = await self.summarize_mail_with_ai(
ai_snippet = await self.summarize_mail_with_ai(
mongo_conn = mongo_conn,
user_info = user_info,
llm = llm,
message = mail_message
)
mail_message.aiSnippet = ai_snippet.summary
# Done here:
print("ONE MAIL:", json.to_string(mail_message.model_dump(), default = str))
sync_result.success = True
sync_result.mailMessage = mail_message
return sync_result
async def __sync_many_gmail(
@@ -415,7 +440,7 @@ class MailController:
query = query_string
)
if not client_response.success:
sync_results["message"] = f"gmail: {client_response.message}"
sync_results.message = f"gmail: {client_response.message}"
return sync_results
messages_list = client_response.data["messages"]
@@ -442,22 +467,27 @@ class MailController:
for result in individual_sync_results:
if result.success: sync_results.successCount += 1
else: sync_results.failureCount += 1
if result.mailMessage: mongo_operations.append(ReplaceOne(
filter = {
"tokenId": token_id,
"serviceType": auth_token.serviceType,
"client": auth_token.client,
"clientMessageId": result.mailMessage.clientMessageId
},
replacement = result.mailMessage.model_dump(),
upsert = True
))
if result.mailMessage:
replacement_json = result.mailMessage.model_dump()
replacement_json.pop("_id", None)
mongo_operations.append(
ReplaceOne(
filter = {
"tokenId": ObjectId(token_id),
"serviceType": auth_token.serviceType,
"client": auth_token.client,
"clientMessageId": result.mailMessage.clientMessageId
},
replacement = replacement_json,
upsert = True
)
)
# Make the bulk insert operation:
if mongo_operations:
sync_count = await current_app.core_message_controller.bulk_write(
mongo_conn = mongo_conn,
requests = mongo_operations
mongo_operations = mongo_operations
)
# Apply the labels to the read messages:
@@ -496,7 +526,7 @@ class MailController:
# ┻ ┗ ┗┗┛┗ ┻ ┗┛┛┗┗ ┛┗┛
# We first load the authorization tokens:
auth_token = await current_app.mail_oauth_model.get_token(
auth_token = await self.get_token(
mongo_conn = mongo_conn,
token_id = token_id,
)
@@ -556,5 +586,5 @@ if __name__ == "__main__":
# raw_mail_json = json.from_file(file_options[1])
# print("FROM FILE:", json.to_string(raw_mail_json["payload"]))
# print("\n\n---------\n\n")
# mail_model = MailAPIModel()
# print(mail_model.extract_plaintext_parts(raw_mail_json["payload"]))
# mail_controller = MailController()
# print(json.to_string(mail_controller.drop_attachments(raw_mail_json["payload"])))
controllers/core/ai/llm.py
-2
View File
@@ -143,8 +143,6 @@ class LLMController(BaseModel):
# Invoke the AI, and format the response:
llm_response = await self.__llm.ainvoke(prompt)
print("LLM RESPONSE:", llm_response)
print("INPUT MESSAGES:", llm_input.messages)
llm_response = LLMOutput(
messages = llm_input.messages,
output = llm_response.content,
controllers/core/message.py
+4 -2
View File
@@ -144,7 +144,8 @@ class MessageController(BaseModel):
return await mongo_conn.bulk_write(
collection = self.MESSAGES_COLLECTION,
requests = mongo_operations
requests = mongo_operations,
raise_exception = True
)
# ┏┓┳┓┳┳┳┓ ┳┓ •
@@ -222,6 +223,7 @@ class MessageController(BaseModel):
projection = {
"_id": True,
"ts": True,
"tokenId": True,
"markedAsUnread": True,
"serviceType": True,
"client": True,
@@ -232,7 +234,6 @@ class MessageController(BaseModel):
"sentSuccessfully": True,
"aiSnippet": True,
"preview": True,
"message": {},
"tags": True,
"usedAi": True
},
@@ -240,6 +241,7 @@ class MessageController(BaseModel):
)
# Convert the fetched records to instances of the data model and return:
for record in records: record["message"] = {}
return [CoreMessageModel(**record) for record in records]
async def get_messages(
models/api/mail/sync.py
+3 -3
View File
@@ -36,7 +36,7 @@ sys.path.append(".")
sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator, PastDatetime
from pydantic import BaseModel, Field, field_validator, PastDatetime, AwareDatetime
from typing import Optional, Literal
# My utils:
@@ -117,13 +117,13 @@ class MailSyncRequestData(BaseModel):
frozen = True
)
startDate: PastDatetime = Field(
startDate: AwareDatetime = Field(
description = "the starting date from which the user wants to sync their mail",
default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(days = 1),
frozen = True
)
endDate: PastDatetime = Field(
endDate: AwareDatetime = Field(
description = "the ending date till which the user wants to sync their mail",
default_factory = lambda: date_time.get_current_utc_date_time() - datetime.timedelta(seconds = 1),
frozen = True
models/core/ai/llm.py
+13
View File
@@ -220,6 +220,19 @@ class LLMOutput(BaseModel):
class Config:
extra = "forbid"
# ┏┓ •
# ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
# ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
# ┛
@property
def summary(self):
return {
"output": self.output,
"tokens": self.tokens.model_dump(),
"invocationId": self.invocationId
}
# *****************************************************************************************************************
# ***** ****
models/core/message.py
+3 -2
View File
@@ -160,9 +160,10 @@ class CoreMessageModel(BaseModel):
default = False
)
aiSnippet: LLMOutput | None = Field(
aiSnippet: LLMOutput | dict | None = Field(
description = "holds a short summary generated by ",
frozen = False
frozen = False,
default = None
)
preview: str = Field(
utils_v2/mail/mail_parser.py
+134 -103
View File
@@ -6,7 +6,7 @@
DATE:
Tuesday, 26th Nov., 2024
Tuesday, 10th Dec., 2024
OBJECTIVE:
@@ -18,6 +18,7 @@
1. GitHub: https://github.com/SpamScope/mail-parser
2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
4. StackOverflow: https://stackoverflow.com/questions/17874360/python-how-to-parse-the-body-from-a-raw-email-given-that-raw-email-does-not
DOWNLOADS:
@@ -47,18 +48,26 @@ from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with mails:
import mailparser
import email
from email.message import Message
from email.utils import parsedate_tz
from email.utils import parseaddr
# To parse the HTML content in the mail:
from bs4 import BeautifulSoup
# To work with datatypes:
from typing import Any, Dict
from typing import Any, Dict, List, Literal
# To work with various encodings:
import base64
import quopri
# To work with date and time:
import datetime
import pytz
import time
# *****************************************************************************************************************
# ***** ****
@@ -87,46 +96,116 @@ import quopri
# *****************************************************************************************************************
def from_quoted_printable(text: str) -> str:
def parse_addr(addr_header: str) -> List[Dict[str, str]]:
decoded_text = ""
decoded_bytes = quopri.decodestring(text)
for encoding in ["utf-8", "utf-16", "utf-32", "latin1"]:
try: text = decoded_bytes.decode(encoding)
except: text = ""
if text.find("From") >= 0:
decoded_text = text
break
return decoded_text
# If the field is null, we return null:
if addr_header is None: return []
# Create an empty variable that will hold the results:
addrs = []
# Iterate through the addresses and parse them:
for a in addr_header.split(","):
n, e = parseaddr(a.strip())
addrs.append({
"name": n.strip() or e.strip(),
"email": e.strip()
})
# Done here:
return addrs
# ---------------------------------------------------------------------------------------------------------------------
def find_in_raw_mail(
raw_mail: str,
text: str
) -> int:
def parse_date(date_header: str) -> datetime.datetime | None:
# We first treat it as un-encoded text:
offset = raw_mail.find(text)
if offset >= 0: return offset
# Try to parse the date header:
date_tuple = parsedate_tz(date_header)
# Then we try Base64 encoding:
offset = raw_mail.find(base64.b64encode(text.encode("utf-8")).decode("utf-8"))
if offset >= 0: return offset
# If the date header was parsed successfully, we assemble
# the parts to get an aware object in UTC timezone:
if date_tuple:
dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60)))
dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC)
return dt
# then we try Quoted-Printable encoding:
offset = from_quoted_printable(raw_mail).find(text)
print("MAIL:")
print(raw_mail)
print("\n\n\n---\n\n\n")
print("TEXT:")
print(quopri.encodestring(text.encode("utf-8")).decode("utf-8"))
if offset >= 0: return offset
# In case of an invalid date header:
else: return None
# Done here, even if nothing worked:
return offset
# ---------------------------------------------------------------------------------------------------------------------
def decode_payload(
raw_payload: str | bytes,
content_main_type: str,
content_charset: str | None,
content_transfer_encoding: Literal[None, "base64", "quoted-printable"]
) -> str | bytes:
# Start by assuming nothing needs to be done:
payload = raw_payload
# We decode various kinds of parts:
match content_transfer_encoding:
# This is just unencoded plaintext:
case None:
pass
# Typically see with attachments:
case "base64":
charset = content_charset or "utf-8"
payload = raw_payload
payload = base64.b64decode(payload)
if content_main_type == "text": payload = payload.decode(charset)
# Typically seen with HTML parts:
case "quoted-printable":
charset = content_charset or "utf-8"
payload = raw_payload.encode(charset)
payload = quopri.decodestring(payload)
if content_main_type == "text": payload = payload.decode(charset)
# Done here:
return payload
# ---------------------------------------------------------------------------------------------------------------------
def parse_part(
part: Message | List[Message]
) -> Dict[str, Any]:
# Start by extracting basic details:
part_json = {
"boundary": part.get_boundary(),
"contentType": part.get_content_type(),
"contentMainType": part.get_content_maintype(),
"contentSubType": part.get_content_subtype(),
"contentCharset": part.get_content_charset(),
"contentTransferEncoding": part.get("Content-Transfer-Encoding"),
"contentDisposition": part.get_content_disposition(),
"filename": part.get_filename(),
"contentId": part.get("Content-ID")
}
# Process the payload of this part:
if part_json["contentMainType"] == "multipart":
part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)]
else:
part_json["payload"] = decode_payload(
raw_payload = part.get_payload(decode = False),
content_main_type = part_json["contentMainType"],
content_charset = part_json["contentCharset"],
content_transfer_encoding = part_json["contentTransferEncoding"]
)
# Done here;
return part_json
# ---------------------------------------------------------------------------------------------------------------------
@@ -145,78 +224,30 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
"""
# Parse the raw format:
if isinstance(raw_mail, str): parsed_mail = mailparser.parse_from_string(raw_mail)
else: parsed_mail = mailparser.parse_from_bytes(raw_mail)
if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
else: parsed_mail = email.message_from_bytes(raw_mail)
# print(parsed_mail.mail_json)
# return
# Format the attachments:
message_attachments = [
{
"filename": attachment["filename"],
"type": attachment["mail_content_type"],
"cid": regex.find_first(text = attachment["content-id"], pattern = r"(?<=<).*(?=>)"),
"rawCid": attachment["content-id"],
"contentDisposition": (cd := attachment["content-disposition"]),
"isInline": True if cd.lower().find("inline") >= 0 else False,
"charset": attachment["charset"],
"contentTransferEncoding": attachment["content_transfer_encoding"],
"payload": attachment["payload"]
} for attachment in parsed_mail.attachments
]
# print("PRINTING PARTS")
# print("LIBRARY PARTS:", type(parsed_mail))
# # Figure out which entity (text and HTML) came in which sequence.
# # The library doesn't give us any sequence info so we do some custom string processing here to figure out the order
# # in which to render the contents of the page.
# parts = [
# # {
# # "partNo": None,
# # "offset": max(
# # parsed_mail.message_as_string.find(t),
# # parsed_mail.message_as_string.find(base64.b64encode(t.encode()).decode())
# # ),
# # "type": "text/plain",
# # "data": t
# # } for t in parsed_mail.text_plain
# ]
# parts = parts + [
# {
# "partNo": None,
# "offset": find_in_raw_mail(raw_mail = parsed_mail.message_as_string, text = h),
# "type": "text/html",
# "data": h
# } for h in parsed_mail.text_html
# ]
# parts = sorted(parts, key = lambda x: x["offset"])
# for i, p in enumerate(parts): p["partNo"] = i
# Get the unformatted text from everything in the mail:
unformatted_text = []
for p in parsed_mail.text_html:
html_parser = BeautifulSoup(p, "html.parser")
unformatted_text.append(html_parser.get_text())
# Put everything together:
return {
"ts": date_time.to_timezone(parsed_mail.date, timezone = date_time.TIMEZONE_UTC),
"headers": parsed_mail.headers,
"from": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers["From"]],
"to": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers["To"]],
"cc": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers.get("Cc", [])],
"bcc": [{"name": _[0] or _[1], "email": _[1]} for _ in parsed_mail.headers.get("Bcc", [])],
"subject": parsed_mail.headers["Subject"],
"text": parsed_mail.text_plain,
"html": parsed_mail.text_html,
# "parts": parts,
"unformattedText": "\n".join(unformatted_text),
"attachments": message_attachments,
"isInbox": None
# Extract the most basic details:
mail_json = {
"ts": parse_date(parsed_mail["Date"]),
"headers": {k: v for k, v in parsed_mail.items()},
"from": parse_addr(parsed_mail["From"]),
"to": parse_addr(parsed_mail["To"]),
"cc": parse_addr(parsed_mail["Cc"]),
"bcc": parse_addr(parsed_mail["Bcc"]),
"subject": parsed_mail["Subject"],
"payload": None
}
# Iterate through each part of the mail for multipart mails:
if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail)
# When the mails are not multipart, just plaintext:
else: mail_json["payload"] = parsed_mail.get_payload()
# Done here:
return mail_json
# *****************************************************************************************************************
# ***** ****
@@ -229,7 +260,7 @@ if __name__ == "__main__":
from utils_v2.system import files
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail.txt")
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
parse_results = parse(mail_string_raw)
print(json.to_string(parse_results, default = str))
utils_v2/mail/mail_parser_v2.py
-266
View File
@@ -1,266 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 10th Dec., 2024
OBJECTIVE:
To parse raw mail bodies and give a structure that is suitable for storing in No-SQL databases like MongoDB. The
raw mail's text is expected to be compliant with standard defined in RFC 5322, RFC 2045, and maybe a few more.
REFERENCES:
1. GitHub: https://github.com/SpamScope/mail-parser
2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
4. StackOverflow: https://stackoverflow.com/questions/17874360/python-how-to-parse-the-body-from-a-raw-email-given-that-raw-email-does-not
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with mails:
import email
from email.message import Message
from email.utils import parsedate_tz
from email.utils import parseaddr
# To parse the HTML content in the mail:
from bs4 import BeautifulSoup
# To work with datatypes:
from typing import Any, Dict, List, Literal
# To work with various encodings:
import base64
import quopri
# To work with date and time:
import datetime
import pytz
import time
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def parse_addr(addr_header: str) -> List[Dict[str, str]]:
# If the field is null, we return null:
if addr_header is None: return []
# Create an empty variable that will hold the results:
addrs = []
# Iterate through the addresses and parse them:
for a in addr_header.split(","):
n, e = parseaddr(a.strip())
addrs.append({
"name": n.strip() or e.strip(),
"email": e.strip()
})
# Done here:
return addrs
# ---------------------------------------------------------------------------------------------------------------------
def parse_date(date_header: str) -> datetime.datetime | None:
# Try to parse the date header:
date_tuple = parsedate_tz(date_header)
# If the date header was parsed successfully, we assemble
# the parts to get an aware object in UTC timezone:
if date_tuple:
dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60)))
dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC)
return dt
# In case of an invalid date header:
else: return None
# ---------------------------------------------------------------------------------------------------------------------
def decode_payload(
raw_payload: str | bytes,
content_main_type: str,
content_charset: str | None,
content_transfer_encoding: Literal[None, "base64", "quoted-printable"]
) -> str | bytes:
# Start by assuming nothing needs to be done:
payload = raw_payload
# We decode various kinds of parts:
match content_transfer_encoding:
# This is just unencoded plaintext:
case None:
pass
# Typically see with attachments:
case "base64":
charset = content_charset or "utf-8"
payload = raw_payload
payload = base64.b64decode(payload)
if content_main_type == "text": payload = payload.decode(charset)
# Typically seen with HTML parts:
case "quoted-printable":
charset = content_charset or "utf-8"
payload = raw_payload.encode(charset)
payload = quopri.decodestring(payload)
if content_main_type == "text": payload = payload.decode(charset)
# Done here:
return payload
# ---------------------------------------------------------------------------------------------------------------------
def parse_part(
part: Message | List[Message]
) -> Dict[str, Any]:
# Start by extracting basic details:
part_json = {
"boundary": part.get_boundary(),
"contentType": part.get_content_type(),
"contentMainType": part.get_content_maintype(),
"contentSubType": part.get_content_subtype(),
"contentCharset": part.get_content_charset(),
"contentTransferEncoding": part.get("Content-Transfer-Encoding"),
"contentDisposition": part.get_content_disposition(),
"filename": part.get_filename(),
"contentId": part.get("Content-ID")
}
# Process the payload of this part:
if part_json["contentMainType"] == "multipart":
part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)]
else:
part_json["payload"] = decode_payload(
raw_payload = part.get_payload(decode = False),
content_main_type = part_json["contentMainType"],
content_charset = part_json["contentCharset"],
content_transfer_encoding = part_json["contentTransferEncoding"]
)
# Done here;
return part_json
# ---------------------------------------------------------------------------------------------------------------------
def parse(raw_mail: str | bytes) -> Dict[str, Any]:
"""
To parse the raw mail text to a usable JSON that can even be stored on a No-SQL database like MongoDB.
DOCUMENTATION:
1. GitHub: https://github.com/SpamScope/mail-parser
2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
:param raw_mail: The raw mail body that adheres to RFC 5322 and RFC 2045 (among others).
:return: The parsed JSON format (dict) of the mail.
"""
# Parse the raw format:
if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
else: parsed_mail = email.message_from_bytes(raw_mail)
# Extract the most basic details:
mail_json = {
"ts": parse_date(parsed_mail["Date"]),
"headers": {k: v for k, v in parsed_mail.items()},
"from": parse_addr(parsed_mail["From"]),
"to": parse_addr(parsed_mail["To"]),
"cc": parse_addr(parsed_mail["Cc"]),
"bcc": parse_addr(parsed_mail["Bcc"]),
"subject": parsed_mail["Subject"],
"payload": None
}
# Iterate through each part of the mail for multipart mails:
if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail)
# When the mails are not multipart, just plaintext:
else: mail_json["payload"] = parsed_mail.get_payload()
# Done here:
return mail_json
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
from utils_v2.system import files
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
parse_results = parse(mail_string_raw)
print(json.to_string(parse_results, default = str))