khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit '5a0667beaf2d7a57f2f407d03c8ed583140a7c36' as 'utils_v2'

Browse Source
This commit is contained in:
khushal
2024-12-17 14:19:24 +05:30
parent 2a4927da85 5a0667beaf
commit 52c0b0baaf
162 changed files with 135915 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils_v2/serialization/__init__.py
View File
utils_v2/serialization/json_serializer.py
+129
View File
@@ -0,0 +1,129 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Sunday 1st Sept. 2024.
OBJECTIVE:
To provide a way to convert any input data to serialized bytes, and back.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# To work with files:
from utils_v2.string import json
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class JSONSerializer:
def __init__(self):
"""
Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and
python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour.
"""
pass
@staticmethod
def serialize(data, encoding = "utf-8"):
"""
Serializes the data that is given to it.
The input has to be JSON-compatible.
:param data: The data to serialize.
:param encoding: The encoding to use.
:return: The bytes representing the data.
"""
# If the data is not already a JSON string, parse it. Then return it as bytes:
data = data if isinstance(data, str) else json.to_string(data, no_space = True)
return data.encode(encoding)
@staticmethod
def deserialize(data, encoding = "utf-8"):
"""
Deserializes the bytes that are given to it.
The input has to be JSON-compatible.
:param data: The bytes to deserialize.
:param encoding: The encoding to use.
:return: The data from the bytes that described it.
"""
data = data.decode(encoding)
return json.from_string(data)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/serialization/pickle_serializer.py
+122
View File
@@ -0,0 +1,122 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Saturday 26th Oct. 2024.
OBJECTIVE:
To provide a way to convert any input data to serialized bytes, and back.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# To work with pickling:
import pickle
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class PickleSerializer:
def __init__(self):
"""
Use this serializer when working with custom python objects. This is meant to be fully flexible, but efficiency
is not guaranteed.
"""
pass
@staticmethod
def serialize(data):
"""
Serializes the data that is given to it.
:param data: The data to serialize.
:return: The bytes representing the data.
"""
return pickle.dumps(data)
@staticmethod
def deserialize(data):
"""
Deserializes the bytes that are given to it.
:param data: The bytes to deserialize.
:return: The data from the bytes that described it.
"""
return pickle.loads(data)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/serialization/universal_serializer.py
+265
View File
@@ -0,0 +1,265 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Sunday 1st Sept. 2024.
OBJECTIVE:
To provide a way to convert any input data to serialized bytes, and back.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# To work with files:
from utils_v2.string import json
# To work with tabulated data:
import pandas as pd
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class UniversalSerializer:
def __init__(self):
"""
Use this when you are working with varied datatypes. You can add custom data-converters also using the
'add_converters' method. Otherwise, most default pythonic datatypes are supported out of the box. Note that this
is NOT recommended because of how large the serialized messages become. Try using 'JSONSerializer' when you know
you will be working specifically with JSOn-compatible inputs.
"""
# These are the converters to use when serializing data:
self.__forward_converters = {
"set": lambda x: list(x),
"tuple": lambda x: list(x),
"complex": lambda x: {"r": x.real, "i": x.imag},
"DataFrame": lambda x: x.to_dict()
}
# These are the converters to use when deserializing data:
self.__reverse_converters = {
"set": lambda x: set(x),
"tuple": lambda x: tuple(x),
"complex": lambda x: complex(x["r"], x["i"]),
"DataFrame": lambda x: pd.DataFrame.from_dict(x)
}
def add_converters(
self,
type_name,
forward_converter_func,
reverse_converter_func
):
"""
Add custom datatype converters.
RULES:
01. Each of the converter functions must take in exactly on argument and return one output of native python
type. This is very important.
02. Each forward and reverse converters must give symmetric results.
:param type_name: The name of the datatype. HINT: type(obj).__name__
:param forward_converter_func: The function to handle conversion to bytes. Use when serializing.
:param reverse_converter_func: The function to handle conversion from bytes. Used when deserializing.
:return: None.
"""
self.__forward_converters[type_name] = lambda x: forward_converter_func(x)
self.__reverse_converters[type_name] = lambda x: reverse_converter_func(x)
def __describe(self, data):
"""
Notes down the input datatypes of everything.
Does everything upto conversion to byes.
:param data: The data to process.
:return: The description of the datatypes and values of what was given.
"""
# Note down the type of data that was sent as the input:
data_type = type(data).__name__
# Handle iterables:
if isinstance(data, list): data = [self.__describe(item) for item in data]
elif isinstance(data, set): data = [self.__describe(item) for item in data]
elif isinstance(data, tuple): data = [self.__describe(item) for item in data]
elif isinstance(data, dict): data = [
{
"k": self.__describe(k),
"v": self.__describe(v)
} for k, v in data.items()
]
# Convert here, and return:
conv = self.__forward_converters.get(data_type)
if conv is not None: data = conv(data)
return {"d": data, "t": data_type}
def serialize(self, data, encoding = "utf-8"):
"""
Serializes the data that is given to it.
:param data: The data to serialize.
:param encoding: The encoding to use.
:return: The bytes representing the data.
"""
data = self.__describe(data)
data = json.to_string(data, no_space = True)
return data.encode(encoding)
def __interpret(self, data):
"""
Interprets the types of data that were serialized originally.
:param data: The data in the serialized form.
:return: Data where the appropriate datatypes have been applied.
"""
# Handle iterables:
if data["t"] == "list": data = [self.__interpret(item) for item in data["d"]]
elif data["t"] == "set": data = set([self.__interpret(item) for item in data["d"]])
elif data["t"] == "tuple": data = tuple([self.__interpret(item) for item in data["d"]])
elif data["t"] == "dict": data = {
self.__interpret(item["k"]): self.__interpret(item["v"])
for item in data["d"]
}
# Handle custom types:
else:
conv = self.__reverse_converters.get(data["t"])
data = data["d"]
if conv is not None: data = conv(data)
# Done here
return data
def deserialize(self, data, encoding = "utf-8"):
"""
Deserializes the bytes that are given to it.
:param data: The bytes to deserialize.
:param encoding: The encoding to use.
:return: The data from the bytes that described it.
"""
data = data.decode(encoding)
data = json.from_string(data)
return self.__interpret(data)
# ---------------------------------------------------------------------------------------------------------------------
class JSONSerializer:
def __init__(self):
"""
Use this serializer when dealing with JSON-compatible data like direct JSON-strings, python dicts, and
python-lists. Beware that non-compatible data will cause either direct exceptions or unexpected behaviour.
"""
pass
@staticmethod
def serialize(data, encoding = "utf-8"):
"""
Serializes the data that is given to it.
The input has to be JSON-compatible.
:param data: The data to serialize.
:param encoding: The encoding to use.
:return: The bytes representing the data.
"""
# If the data is not already a JSON string, parse it. Then return it as bytes:
data = data if isinstance(data, str) else json.to_string(data, no_space = True)
return data.encode(encoding)
@staticmethod
def deserialize(data, encoding = "utf-8"):
"""
Deserializes the bytes that are given to it.
The input has to be JSON-compatible.
:param data: The bytes to deserialize.
:param encoding: The encoding to use.
:return: The data from the bytes that described it.
"""
data = data.decode(encoding)
return json.from_string(data)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass