""" AUTHOR: Khushal P Soonderji DATE: Monday, 13th Jan., 2025 OBJECTIVE: To provide an easy way to work with '.json' data and files. REFERENCES: N/A DOWNLOADS: N/A """ # ***************************************************************************************************************** # ***** **** # *** IMPORT *** # ***** **** # ***************************************************************************************************************** # To make sibling directories accessible for imports: import sys sys.path.append(".") sys.path.append("..") # System-level activities: import io # My utils: from utils_v2.string import json from utils_v2.string import regex from utils_v2.system import files from utils_v2.date_time import date_time from utils_v2.database.async_mysql_v2 import AsyncMySQL # To make HTTP calls: import httpx import socket # To work with date and time: import datetime import time # To work with datatypes: from typing import List # For handling NaN values: import pandas as pd # For debugging: from icecream import IceCreamDebugger # For asynchronous operations: import asyncio # ***************************************************************************************************************** # ***** **** # *** MACROS / ONE-TIME INIT *** # ***** **** # ***************************************************************************************************************** # To make API calls: http_client = httpx.AsyncClient( limits = httpx.Limits( max_connections = 100, # ............ Maximum number of connections allowed in the pool. max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive. ), timeout = httpx.Timeout( pool = 120.0, # .... Time to wait for a free connection from the pool. connect = 2.5, # ... Time to wait for establishing a connection to the server. write = 10.0, # .... Time to wait for sending data. read = 120.0 # ..... Time to wait for receiving data. ) ) # For debugging: printer = IceCreamDebugger(prefix = "FireF. | ", includeContext = True) no_context_printer = IceCreamDebugger(prefix = "FireF. | ", includeContext = False) # ***************************************************************************************************************** # ***** **** # *** VARIABLES *** # ***** **** # ***************************************************************************************************************** # This script's config.: SERVER_HOSTNAME = str(socket.gethostname()) SCRIPT_DATA = {} # For the database: sql_writer: AsyncMySQL | None = None sql_reader: AsyncMySQL | None = None # ***************************************************************************************************************** # ***** **** # *** FUNCTIONS *** # ***** **** # ***************************************************************************************************************** async def init( script_id: str, debug: bool ) -> bool: """ To initialize all credentials, instances, and connectivity for this whole script. :param script_id: The id to use to load cred and data from the internal service. :param debug: Whether, or not, you would like to print the debug messages. :return: True if initialized successfully, else False. """ # Declare the required global variables: global SCRIPT_DATA global sql_writer global sql_reader # Basic stuff: if not debug: printer.disable() # ┏┓ ┓ ┓ ┳┓ # ┃ ┏┓┏┓┏┫ ┏┓┏┓┏┫ ┃┃┏┓╋┏┓ # ┗┛┛ ┗ ┗┻ ┗┻┛┗┗┻ ┻┛┗┻┗┗┻ # Get the script credentials: response = await http_client.get( url = r"https://nexcom.ditscentre.in/internal/cred/get", headers = {"X-Script-Id": script_id} ) if response.status_code not in [200]: print("FATAL: SCRIPT CREDENTIALS LOADING FAILED!") return False script_cred = response.json().get("data") # Get the script data: response = await http_client.get( url = r"https://nexcom.ditscentre.in/internal/data/get", headers = {"X-Script-Id": script_id} ) if response.status_code not in [200]: print("FATAL: SCRIPT DATA LOADING FAILED!") return False SCRIPT_DATA = response.json().get("data") # Done with this step: printer("Cred and Data loaded.") # ┳┳┓ • ┳┓┳┓ # ┃┃┃┏┓┏┓┓┏┓┃┃┣┫ # ┛ ┗┗┻┛ ┗┗┻┻┛┻┛ sql_writer = AsyncMySQL( pool_size = script_cred["mariaDb"]["write"]["poolSize"], host = script_cred["mariaDb"]["write"]["host"], user = script_cred["mariaDb"]["write"]["user"], password = script_cred["mariaDb"]["write"]["password"], database = script_cred["mariaDb"]["write"]["database"] ) if not await sql_writer.connect(): print("FATAL: MARIA-DB WRITER CONNECTION FAILED!") return False sql_reader = AsyncMySQL( pool_size = script_cred["mariaDb"]["read"]["poolSize"], host = script_cred["mariaDb"]["read"]["host"], user = script_cred["mariaDb"]["read"]["user"], password = script_cred["mariaDb"]["read"]["password"], database = script_cred["mariaDb"]["read"]["database"] ) if not await sql_reader.connect(): print("FATAL: MARIA-DB READER CONNECTION FAILED!") return False printer("MariaDB connected.") # ┳┓ # ┃┃┏┓┏┓┏┓ # ┻┛┗┛┛┗┗ # If everything went well, we return with success: printer("Initialization done.") return True # --------------------------------------------------------------------------------------------------------------------- async def get_all_users() -> dict | None: """ Simply gets all the details of all the users. :return: The dictionary of all the users if successful, else None. """ # Get a list of all the users: rows_affected, result, exception = await sql_reader.execute_one( query = r"SELECT * FROM aaa.users;", commit = False, return_exception = True ) # If something goes wrong: if exception: printer(exception) # Done here: return None if exception else result # --------------------------------------------------------------------------------------------------------------------- async def get_all_clients() -> dict | None: """ Simply gets all the details of all the clients. :return: The dictionary of all the users if successful, else None. """ # Get a list of all the users: rows_affected, result, exception = await sql_reader.execute_one( query = r"SELECT * FROM aaa.accounts_master WHERE account_group_id = 974;", commit = False, return_exception = True ) # If something goes wrong: if exception: printer(exception) # Done here: return None if exception else result # --------------------------------------------------------------------------------------------------------------------- async def get_all_departments() -> dict | None: """ Simply gets all the details of all the departments. :return: The dictionary of all the users if successful, else None. """ # Get a list of all the users: rows_affected, result, exception = await sql_reader.execute_one( query = "SELECT * FROM aaa.departments WHERE entity_id = 119;", commit = False, return_exception = True ) # If something goes wrong: if exception: printer(exception) # Done here: return None if exception else result # --------------------------------------------------------------------------------------------------------------------- async def get_all_processes() -> dict | None: """ Simply gets all the details of all the processes. :return: The dictionary of all the users if successful, else None. """ # Get a list of all the users: rows_affected, result, exception = await sql_reader.execute_one( query = ( r"SELECT * from aaa.processes WHERE department_id IN " "(SELECT department_id FROM aaa.departments WHERE entity_id = 119);" ), commit = False, return_exception = True ) # If something goes wrong: if exception: printer(exception) # Done here: return None if exception else result # --------------------------------------------------------------------------------------------------------------------- async def main(debug: bool = False): # Load the CSV that contains all the activity records: no_context_printer("Reading Excel.") activity_df = pd.read_excel(r"/home/developer/Downloads/Telegram Desktop/TaskManager - Edited.xlsx", sheet_name = "Activity") printer(len(activity_df)) # Get a list of all the users: no_context_printer("Listing all users.") users = await get_all_users() users_by_pseudonym = {u["pseudonym"]: u for u in users[-1]} # Get a list of all clients: no_context_printer("Listing all clients.") clients = await get_all_clients() clients_by_name = {u["account_name"]: u for u in clients[-1]} # Get a list of all departments: no_context_printer("Listing all departments.") departments = await get_all_departments() departments_by_name = {u["department_name"]: u for u in departments[-1]} # Get a list of all processes: no_context_printer("Listing all processes.") processes = await get_all_processes() processes_by_name = {u["process_name"]: u for u in processes[-1]} # Add the columns to the dataframe: no_context_printer("Adding user id to Excel.") activity_df["user_id"] = activity_df["Username"].apply(lambda x: users_by_pseudonym.get(x, {}).get("user_id", None)) no_context_printer("Adding department id to Excel.") activity_df["department_id"] = activity_df["Department"].apply(lambda x: departments_by_name.get(x, {}).get("department_id", None)) no_context_printer("Adding process id to Excel.") activity_df["process_id"] = activity_df["Task"].apply(lambda x: processes_by_name.get(x, {}).get("process_id", None)) no_context_printer("Adding client id to Excel.") activity_df["client_id"] = activity_df["Client Name"].apply(lambda x: clients_by_name.get(x, {}).get("account_id", None)) # Save the file: print(activity_df[:10].to_string()) print("...") print(activity_df[-10:].to_string()) print(len(activity_df)) activity_df.to_csv(r"/home/developer/Downloads/TaskManager - Data Matched.csv", index = False) # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** # ***** **** # ***************************************************************************************************************** if __name__ == "__main__": # To get args. from the terminal: import argparse # Get the config. from the command-line: parser = argparse.ArgumentParser() parser.add_argument( "-s", "--script-id", dest = "script_id", type = str, help = "The id of this script (will affect the loaded config)." ) parser.add_argument( "-d", "--debug", dest = "debug", action = "store_true", help = "Whether, or not, you want to see debugging messages in the terminal.", default = False ) args = parser.parse_args() async def runner(): # Initialize and run the main code: if await init( script_id = args.script_id, debug = args.debug ): await main() # Disconnect from the database: if sql_writer: disconnected = await sql_writer.disconnect() if sql_reader: disconnected = await sql_reader.disconnect() asyncio.run(runner())