khushalps
8bf92c04ed
git-subtree-dir: utils_v2 git-subtree-split: 41797375144632a2cdda1e4c6c4a5769e4b80306
119 lines
4.8 KiB
Python
119 lines
4.8 KiB
Python
"""
|
|
|
|
AUTHOR:
|
|
|
|
Khushal P Soonderji
|
|
|
|
DATE:
|
|
|
|
Tuesday, 22nd Oct., 2024
|
|
|
|
OBJECTIVE:
|
|
|
|
To provide an easy way to split dataframes into chunks and divide the workload nto more manageable batches.
|
|
|
|
REFERENCES:
|
|
|
|
N/A
|
|
|
|
DOWNLOADS:
|
|
|
|
N/A
|
|
|
|
USAGE EXAMPLE:
|
|
|
|
for sub_df in DataFrameSplitter(df, chunk_size = 50):
|
|
|
|
# Do your task here:
|
|
pass
|
|
|
|
"""
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** IMPORT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MACROS / ONE-TIME INIT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** VARIABLES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** FUNCTIONS ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** CLASSES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
class DataFrameSplitter:
|
|
|
|
def __init__(self, df, chunk_size):
|
|
|
|
"""
|
|
Use this to process your dataframe in batches. Useful for when you need to send out alerts at intervals or need
|
|
to maintain checkpoints.
|
|
:param df: The dataframe to iterate over.
|
|
:param chunk_size: The max. no. of rows to pick at once.
|
|
"""
|
|
|
|
self.df = df
|
|
self.row_count = df.shape[0]
|
|
self.chunk_size = chunk_size
|
|
self.offset = 0
|
|
|
|
def __iter__(self):
|
|
self.offset = 0
|
|
return self
|
|
|
|
def __next__(self):
|
|
if self.offset >= self.row_count:
|
|
raise StopIteration
|
|
end = self.offset + self.chunk_size
|
|
chunk = self.df.iloc[self.offset:end]
|
|
self.offset = end
|
|
return chunk
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MAIN PROGRAM ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
pass
|