Resetting utils subtree.
This commit is contained in:
@@ -0,0 +1,392 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Monday, 13th Jan., 2025
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To provide an easy way to work with '.json' data and files.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.string import regex
|
||||
from utils_v2.system import files
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
|
||||
# To make HTTP calls:
|
||||
import httpx
|
||||
import socket
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
import time
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import List
|
||||
|
||||
# For handling NaN values:
|
||||
import pandas as pd
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
# For asynchronous operations:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make API calls:
|
||||
http_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 120.0 # ..... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# For debugging:
|
||||
printer = IceCreamDebugger(prefix = "FireF. | ", includeContext = True)
|
||||
no_context_printer = IceCreamDebugger(prefix = "FireF. | ", includeContext = False)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# This script's config.:
|
||||
SERVER_HOSTNAME = str(socket.gethostname())
|
||||
SCRIPT_DATA = {}
|
||||
|
||||
# For the database:
|
||||
sql_writer: AsyncMySQL | None = None
|
||||
sql_reader: AsyncMySQL | None = None
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
async def init(
|
||||
script_id: str,
|
||||
debug: bool
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
To initialize all credentials, instances, and connectivity for this whole script.
|
||||
:param script_id: The id to use to load cred and data from the internal service.
|
||||
:param debug: Whether, or not, you would like to print the debug messages.
|
||||
:return: True if initialized successfully, else False.
|
||||
"""
|
||||
|
||||
# Declare the required global variables:
|
||||
global SCRIPT_DATA
|
||||
global sql_writer
|
||||
global sql_reader
|
||||
|
||||
# Basic stuff:
|
||||
if not debug: printer.disable()
|
||||
|
||||
# ┏┓ ┓ ┓ ┳┓
|
||||
# ┃ ┏┓┏┓┏┫ ┏┓┏┓┏┫ ┃┃┏┓╋┏┓
|
||||
# ┗┛┛ ┗ ┗┻ ┗┻┛┗┗┻ ┻┛┗┻┗┗┻
|
||||
|
||||
# Get the script credentials:
|
||||
response = await http_client.get(
|
||||
url = r"https://nexcom.ditscentre.in/internal/cred/get",
|
||||
headers = {"X-Script-Id": script_id}
|
||||
)
|
||||
if response.status_code not in [200]:
|
||||
print("FATAL: SCRIPT CREDENTIALS LOADING FAILED!")
|
||||
return False
|
||||
script_cred = response.json().get("data")
|
||||
|
||||
# Get the script data:
|
||||
response = await http_client.get(
|
||||
url = r"https://nexcom.ditscentre.in/internal/data/get",
|
||||
headers = {"X-Script-Id": script_id}
|
||||
)
|
||||
if response.status_code not in [200]:
|
||||
print("FATAL: SCRIPT DATA LOADING FAILED!")
|
||||
return False
|
||||
SCRIPT_DATA = response.json().get("data")
|
||||
|
||||
# Done with this step:
|
||||
printer("Cred and Data loaded.")
|
||||
|
||||
# ┳┳┓ • ┳┓┳┓
|
||||
# ┃┃┃┏┓┏┓┓┏┓┃┃┣┫
|
||||
# ┛ ┗┗┻┛ ┗┗┻┻┛┻┛
|
||||
|
||||
sql_writer = AsyncMySQL(
|
||||
pool_size = script_cred["mariaDb"]["write"]["poolSize"],
|
||||
host = script_cred["mariaDb"]["write"]["host"],
|
||||
user = script_cred["mariaDb"]["write"]["user"],
|
||||
password = script_cred["mariaDb"]["write"]["password"],
|
||||
database = script_cred["mariaDb"]["write"]["database"]
|
||||
)
|
||||
if not await sql_writer.connect():
|
||||
print("FATAL: MARIA-DB WRITER CONNECTION FAILED!")
|
||||
return False
|
||||
|
||||
sql_reader = AsyncMySQL(
|
||||
pool_size = script_cred["mariaDb"]["read"]["poolSize"],
|
||||
host = script_cred["mariaDb"]["read"]["host"],
|
||||
user = script_cred["mariaDb"]["read"]["user"],
|
||||
password = script_cred["mariaDb"]["read"]["password"],
|
||||
database = script_cred["mariaDb"]["read"]["database"]
|
||||
)
|
||||
if not await sql_reader.connect():
|
||||
print("FATAL: MARIA-DB READER CONNECTION FAILED!")
|
||||
return False
|
||||
|
||||
printer("MariaDB connected.")
|
||||
|
||||
# ┳┓
|
||||
# ┃┃┏┓┏┓┏┓
|
||||
# ┻┛┗┛┛┗┗
|
||||
|
||||
# If everything went well, we return with success:
|
||||
printer("Initialization done.")
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def get_all_users() -> dict | None:
|
||||
|
||||
"""
|
||||
Simply gets all the details of all the users.
|
||||
:return: The dictionary of all the users if successful, else None.
|
||||
"""
|
||||
|
||||
# Get a list of all the users:
|
||||
rows_affected, result, exception = await sql_reader.execute_one(
|
||||
query = r"SELECT * FROM aaa.users;",
|
||||
commit = False,
|
||||
return_exception = True
|
||||
)
|
||||
|
||||
# If something goes wrong:
|
||||
if exception: printer(exception)
|
||||
|
||||
# Done here:
|
||||
return None if exception else result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def get_all_clients() -> dict | None:
|
||||
|
||||
"""
|
||||
Simply gets all the details of all the clients.
|
||||
:return: The dictionary of all the users if successful, else None.
|
||||
"""
|
||||
|
||||
# Get a list of all the users:
|
||||
rows_affected, result, exception = await sql_reader.execute_one(
|
||||
query = r"SELECT * FROM aaa.accounts_master WHERE account_group_id = 974;",
|
||||
commit = False,
|
||||
return_exception = True
|
||||
)
|
||||
|
||||
# If something goes wrong:
|
||||
if exception: printer(exception)
|
||||
|
||||
# Done here:
|
||||
return None if exception else result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def get_all_departments() -> dict | None:
|
||||
|
||||
"""
|
||||
Simply gets all the details of all the departments.
|
||||
:return: The dictionary of all the users if successful, else None.
|
||||
"""
|
||||
|
||||
# Get a list of all the users:
|
||||
rows_affected, result, exception = await sql_reader.execute_one(
|
||||
query = "SELECT * FROM aaa.departments WHERE entity_id = 119;",
|
||||
commit = False,
|
||||
return_exception = True
|
||||
)
|
||||
|
||||
# If something goes wrong:
|
||||
if exception: printer(exception)
|
||||
|
||||
# Done here:
|
||||
return None if exception else result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def get_all_processes() -> dict | None:
|
||||
|
||||
"""
|
||||
Simply gets all the details of all the processes.
|
||||
:return: The dictionary of all the users if successful, else None.
|
||||
"""
|
||||
|
||||
# Get a list of all the users:
|
||||
rows_affected, result, exception = await sql_reader.execute_one(
|
||||
query = (
|
||||
r"SELECT * from aaa.processes WHERE department_id IN "
|
||||
"(SELECT department_id FROM aaa.departments WHERE entity_id = 119);"
|
||||
),
|
||||
commit = False,
|
||||
return_exception = True
|
||||
)
|
||||
|
||||
# If something goes wrong:
|
||||
if exception: printer(exception)
|
||||
|
||||
# Done here:
|
||||
return None if exception else result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def main(debug: bool = False):
|
||||
|
||||
# Load the CSV that contains all the activity records:
|
||||
no_context_printer("Reading Excel.")
|
||||
activity_df = pd.read_excel(r"/home/developer/Downloads/Telegram Desktop/TaskManager - Edited.xlsx", sheet_name = "Activity")
|
||||
printer(len(activity_df))
|
||||
|
||||
# Get a list of all the users:
|
||||
no_context_printer("Listing all users.")
|
||||
users = await get_all_users()
|
||||
users_by_pseudonym = {u["pseudonym"]: u for u in users[-1]}
|
||||
|
||||
# Get a list of all clients:
|
||||
no_context_printer("Listing all clients.")
|
||||
clients = await get_all_clients()
|
||||
clients_by_name = {u["account_name"]: u for u in clients[-1]}
|
||||
|
||||
# Get a list of all departments:
|
||||
no_context_printer("Listing all departments.")
|
||||
departments = await get_all_departments()
|
||||
departments_by_name = {u["department_name"]: u for u in departments[-1]}
|
||||
|
||||
# Get a list of all processes:
|
||||
no_context_printer("Listing all processes.")
|
||||
processes = await get_all_processes()
|
||||
processes_by_name = {u["process_name"]: u for u in processes[-1]}
|
||||
|
||||
# Add the columns to the dataframe:
|
||||
no_context_printer("Adding user id to Excel.")
|
||||
activity_df["user_id"] = activity_df["Username"].apply(lambda x: users_by_pseudonym.get(x, {}).get("user_id", None))
|
||||
no_context_printer("Adding department id to Excel.")
|
||||
activity_df["department_id"] = activity_df["Department"].apply(lambda x: departments_by_name.get(x, {}).get("department_id", None))
|
||||
no_context_printer("Adding process id to Excel.")
|
||||
activity_df["process_id"] = activity_df["Task"].apply(lambda x: processes_by_name.get(x, {}).get("process_id", None))
|
||||
no_context_printer("Adding client id to Excel.")
|
||||
activity_df["client_id"] = activity_df["Client Name"].apply(lambda x: clients_by_name.get(x, {}).get("account_id", None))
|
||||
|
||||
# Save the file:
|
||||
print(activity_df[:10].to_string())
|
||||
print("...")
|
||||
print(activity_df[-10:].to_string())
|
||||
print(len(activity_df))
|
||||
activity_df.to_csv(r"/home/developer/Downloads/TaskManager - Data Matched.csv", index = False)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# To get args. from the terminal:
|
||||
import argparse
|
||||
|
||||
# Get the config. from the command-line:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-s", "--script-id",
|
||||
dest = "script_id",
|
||||
type = str,
|
||||
help = "The id of this script (will affect the loaded config)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d", "--debug",
|
||||
dest = "debug",
|
||||
action = "store_true",
|
||||
help = "Whether, or not, you want to see debugging messages in the terminal.",
|
||||
default = False
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
async def runner():
|
||||
|
||||
# Initialize and run the main code:
|
||||
if await init(
|
||||
script_id = args.script_id,
|
||||
debug = args.debug
|
||||
): await main()
|
||||
|
||||
# Disconnect from the database:
|
||||
if sql_writer: disconnected = await sql_writer.disconnect()
|
||||
if sql_reader: disconnected = await sql_reader.disconnect()
|
||||
|
||||
|
||||
asyncio.run(runner())
|
||||
Reference in New Issue
Block a user