""" AUTHOR: Khushal P Soonderji DATE: Tuesday, 22nd Oct., 2024 OBJECTIVE: To provide an easy way to split dataframes into chunks and divide the workload nto more manageable batches. REFERENCES: N/A DOWNLOADS: N/A USAGE EXAMPLE: for sub_df in DataFrameSplitter(df, chunk_size = 50): # Do your task here: pass """ # ***************************************************************************************************************** # ***** **** # *** IMPORT *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** MACROS / ONE-TIME INIT *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** VARIABLES *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** FUNCTIONS *** # ***** **** # ***************************************************************************************************************** # --- Nothing Yet # ***************************************************************************************************************** # ***** **** # *** CLASSES *** # ***** **** # ***************************************************************************************************************** class DataFrameSplitter: def __init__(self, df, chunk_size): """ Use this to process your dataframe in batches. Useful for when you need to send out alerts at intervals or need to maintain checkpoints. :param df: The dataframe to iterate over. :param chunk_size: The max. no. of rows to pick at once. """ self.df = df self.row_count = df.shape[0] self.chunk_size = chunk_size self.offset = 0 def __iter__(self): self.offset = 0 return self def __next__(self): if self.offset >= self.row_count: raise StopIteration end = self.offset + self.chunk_size chunk = self.df.iloc[self.offset:end] self.offset = end return chunk # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** # ***** **** # ***************************************************************************************************************** if __name__ == "__main__": pass