(20241210) Started working on files.
This commit is contained in:
+118
-35
@@ -49,17 +49,23 @@ from utils_v2.date_time import date_time
|
||||
|
||||
# To work with mails:
|
||||
import email
|
||||
from email.message import Message
|
||||
from email.utils import parsedate_tz
|
||||
from email.utils import parseaddr
|
||||
|
||||
# To parse the HTML content in the mail:
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, Dict
|
||||
from typing import Any, Dict, List, Literal
|
||||
|
||||
# To work with various encodings:
|
||||
import base64
|
||||
import quopri
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
import pytz
|
||||
import time
|
||||
|
||||
|
||||
@@ -90,12 +96,93 @@ import time
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
def parse_addr(addr_header: str) -> List[Dict[str, str]]:
|
||||
|
||||
# If the field is null, we return null:
|
||||
if addr_header is None: return []
|
||||
|
||||
# Create an empty variable that will hold the results:
|
||||
addrs = []
|
||||
|
||||
# Iterate through the addresses and parse them:
|
||||
for a in addr_header.split(","):
|
||||
n, e = parseaddr(a.strip())
|
||||
addrs.append({
|
||||
"name": n.strip() or e.strip(),
|
||||
"email": e.strip()
|
||||
})
|
||||
|
||||
# Done here:
|
||||
return addrs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_date(date_header: str) -> datetime.datetime | None:
|
||||
|
||||
# Try to parse the date header:
|
||||
date_tuple = parsedate_tz(date_header)
|
||||
|
||||
# If the date header was parsed successfully, we assemble
|
||||
# the parts to get an aware object in UTC timezone:
|
||||
if date_tuple:
|
||||
dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60)))
|
||||
dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC)
|
||||
return dt
|
||||
|
||||
# In case of an invalid date header:
|
||||
else: return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def decode_payload(
|
||||
raw_payload: str | bytes,
|
||||
content_main_type: str,
|
||||
content_charset: str | None,
|
||||
content_transfer_encoding: Literal[None, "base64", "quoted-printable"]
|
||||
) -> str | bytes:
|
||||
|
||||
# Start by assuming nothing needs to be done:
|
||||
payload = raw_payload
|
||||
|
||||
# We decode various kinds of parts:
|
||||
match content_transfer_encoding:
|
||||
|
||||
# This is just unencoded plaintext:
|
||||
case None:
|
||||
pass
|
||||
|
||||
# Typically see with attachments:
|
||||
case "base64":
|
||||
charset = content_charset or "utf-8"
|
||||
payload = raw_payload
|
||||
payload = base64.b64decode(payload)
|
||||
if content_main_type == "text": payload = payload.decode(charset)
|
||||
|
||||
# Typically seen with HTML parts:
|
||||
case "quoted-printable":
|
||||
charset = content_charset or "utf-8"
|
||||
payload = raw_payload.encode(charset)
|
||||
payload = quopri.decodestring(payload)
|
||||
if content_main_type == "text": payload = payload.decode(charset)
|
||||
|
||||
# Done here:
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_part(
|
||||
# part: email.message.Message
|
||||
part
|
||||
part: Message | List[Message]
|
||||
) -> Dict[str, Any]:
|
||||
|
||||
# Start by extracting basic details:
|
||||
part_json = {
|
||||
"boundary": part.get_boundary(),
|
||||
"contentType": part.get_content_type(),
|
||||
"contentMainType": part.get_content_maintype(),
|
||||
"contentSubType": part.get_content_subtype(),
|
||||
@@ -103,31 +190,19 @@ def parse_part(
|
||||
"contentTransferEncoding": part.get("Content-Transfer-Encoding"),
|
||||
"contentDisposition": part.get_content_disposition(),
|
||||
"filename": part.get_filename(),
|
||||
"contentId": part.get("Content-ID"),
|
||||
"payload": part.get_payload(decode = False)
|
||||
"contentId": part.get("Content-ID")
|
||||
}
|
||||
|
||||
# We decode various kinds of parts:
|
||||
match part_json["contentTransferEncoding"]:
|
||||
|
||||
# This is just unencoded plaintext:
|
||||
case None: pass
|
||||
|
||||
# Typically see with attachments:
|
||||
case "base64":
|
||||
charset = part_json["contentCharset"] or "utf-8"
|
||||
payload = part_json["payload"]
|
||||
payload = base64.b64decode(payload)
|
||||
part_json["payload"] = payload
|
||||
|
||||
# Typically seen with HTML parts:
|
||||
case "quoted-printable":
|
||||
charset = part_json["contentCharset"] or "utf-8"
|
||||
payload = part_json["payload"].encode(charset)
|
||||
payload = quopri.decodestring(payload)
|
||||
part_json["payload"] = payload.decode(charset)
|
||||
|
||||
# print("PARSED PART:", json.to_string(part_json, default = str))
|
||||
# Process the payload of this part:
|
||||
if part_json["contentMainType"] == "multipart":
|
||||
part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)]
|
||||
else:
|
||||
part_json["payload"] = decode_payload(
|
||||
raw_payload = part.get_payload(decode = False),
|
||||
content_main_type = part_json["contentMainType"],
|
||||
content_charset = part_json["contentCharset"],
|
||||
content_transfer_encoding = part_json["contentTransferEncoding"]
|
||||
)
|
||||
|
||||
# Done here;
|
||||
return part_json
|
||||
@@ -152,18 +227,26 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]:
|
||||
if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
|
||||
else: parsed_mail = email.message_from_bytes(raw_mail)
|
||||
|
||||
# Make variables:
|
||||
parts = []
|
||||
attachments = []
|
||||
# Extract the most basic details:
|
||||
mail_json = {
|
||||
"ts": parse_date(parsed_mail["Date"]),
|
||||
"headers": {k: v for k, v in parsed_mail.items()},
|
||||
"from": parse_addr(parsed_mail["From"]),
|
||||
"to": parse_addr(parsed_mail["To"]),
|
||||
"cc": parse_addr(parsed_mail["Cc"]),
|
||||
"bcc": parse_addr(parsed_mail["Bcc"]),
|
||||
"subject": parsed_mail["Subject"],
|
||||
"payload": None
|
||||
}
|
||||
|
||||
# Iterate through each part of the mail for multipart mails:
|
||||
if parsed_mail.is_multipart():
|
||||
for part in parsed_mail.walk():
|
||||
part_json = parse_part(part)
|
||||
if part_json["contentDisposition"] in ["inline", "attachment"]: attachments.append()
|
||||
if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail)
|
||||
|
||||
# When the mails are not multipart, just plaintext:
|
||||
else: print("PLAINTEXT PART:", parsed_mail.get_payload())
|
||||
else: mail_json["payload"] = parsed_mail.get_payload()
|
||||
|
||||
# Done here:
|
||||
return mail_json
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -180,4 +263,4 @@ if __name__ == "__main__":
|
||||
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
|
||||
parse_results = parse(mail_string_raw)
|
||||
|
||||
# print(json.to_string(parse_results, default = str))
|
||||
print(json.to_string(parse_results, default = str))
|
||||
|
||||
Reference in New Issue
Block a user