diff --git a/models/behaviour/mail/retrieve.py b/models/behaviour/mail/retrieve.py index 414ed0e..c90a76f 100644 --- a/models/behaviour/mail/retrieve.py +++ b/models/behaviour/mail/retrieve.py @@ -116,6 +116,8 @@ class MailRetrieveModel(BaseModel): "_id": True, "serviceType": True, "client": True, + "ts": True, + "readTs": True, "payload.ts": True, "payload.readTs": True, "payload.from": True, diff --git a/utils_v2/mail/mail_parser_v2.py b/utils_v2/mail/mail_parser_v2.py index d378b7d..89e6022 100644 --- a/utils_v2/mail/mail_parser_v2.py +++ b/utils_v2/mail/mail_parser_v2.py @@ -100,15 +100,36 @@ def parse_part( "contentMainType": part.get_content_maintype(), "contentSubType": part.get_content_subtype(), "contentCharset": part.get_content_charset(), + "contentTransferEncoding": part.get("Content-Transfer-Encoding"), "contentDisposition": part.get_content_disposition(), "filename": part.get_filename(), - "payload": part.get_payload(decode = True) + "contentId": part.get("Content-ID"), + "payload": part.get_payload(decode = False) } - if part_json["contentCharset"] is not None: - part_json["payload"] = part_json["payload"].decode(part_json["contentCharset"]) - print("PARSED PART:", json.to_string(part_json, default = str)) + # We decode various kinds of parts: + match part_json["contentTransferEncoding"]: + # This is just unencoded plaintext: + case None: pass + + # Typically see with attachments: + case "base64": + charset = part_json["contentCharset"] or "utf-8" + payload = part_json["payload"] + payload = base64.b64decode(payload) + part_json["payload"] = payload + + # Typically seen with HTML parts: + case "quoted-printable": + charset = part_json["contentCharset"] or "utf-8" + payload = part_json["payload"].encode(charset) + payload = quopri.decodestring(payload) + part_json["payload"] = payload.decode(charset) + + # print("PARSED PART:", json.to_string(part_json, default = str)) + + # Done here; return part_json @@ -138,35 +159,8 @@ def parse(raw_mail: str | bytes) -> Dict[str, Any]: # Iterate through each part of the mail for multipart mails: if parsed_mail.is_multipart(): for part in parsed_mail.walk(): - parse_part(part) - - # # Start by extracting basic details about the part: - # part_json = { - # "contentType": part.get("Content-Type"), - # "contentDisposition": part.get("Content-Disposition"), - # "contentTransferEncoding": part.get("Content-Transfer-Encoding"), - # "contentId": part.get("Content-ID"), - # # "part": part.get_payload(decode=True) - # } - # - # # Now check if this part is either plain or HTML text: - # content_type = part_json["contentType"].lower() - # if ( - # content_type.find("text/html") >= 0 or - # content_type.find("text/plain") >= 0 - # ): parts.append(parse_part(part)) - # - # # print("MULTIPART PART:", json.to_string(part_json, default = str)) - # # if ( - # # isinstance(part_json["part"], bytes) and - # # part_json[""].find() - # # ): - # # time.sleep(1.0) - # # try: part_json["part"] = part_json["part"].decode() - # # except: pass - # # print("MULTIPART PART:", json.to_string(part_json)) - # print("\n\n---\n\n") - # # if part_json[] + part_json = parse_part(part) + if part_json["contentDisposition"] in ["inline", "attachment"]: attachments.append() # When the mails are not multipart, just plaintext: else: print("PLAINTEXT PART:", parsed_mail.get_payload())