khushalps/cosec_automation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20260217) Ready to test on prodcuction now

Browse Source
This commit is contained in:
Khushal P Soonderji
2026-02-17 14:53:09 +05:30
parent df92fb08e9
commit 64a6eac20f
5 changed files with 102 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helpers/async_tcaoff.py
+15 -4
View File
@@ -459,6 +459,21 @@ class AsyncTheCAOffice:
:return: True if successful, else False. :return: True if successful, else False.
""" """
# print("TEAM ADD:", json.to_string({
# "branchId": branch_id,
# "idDepartment": dept_id,
# "reportingTo": reporting_to,
# "name": team_name,
# "email": email,
# "phoneNo": phone_no,
# "role": role,
# "username": username,
# "password": password,
# "hierarchy": 1,
# "applicantNotes": applicant_notes
# }))
# return False
# # Ensure that JSON notes are converted to string: # # Ensure that JSON notes are converted to string:
# if isinstance(applicant_notes, (list, dict)): # if isinstance(applicant_notes, (list, dict)):
# applicant_notes = json.to_string(applicant_notes, no_space = True) # applicant_notes = json.to_string(applicant_notes, no_space = True)
@@ -529,8 +544,6 @@ class AsyncTheCAOffice:
:return: True if the attendance was marked, else False. :return: True if the attendance was marked, else False.
""" """
start_time = time.time()
# Prepare the payload: # Prepare the payload:
json_payload = { json_payload = {
"date": ( "date": (
@@ -569,7 +582,6 @@ class AsyncTheCAOffice:
"Attendance marked.", "Attendance marked.",
user_id, user_id,
) )
print("ONE TIME TAKEN:", time.time() - start_time)
return True return True
# If the call failed: # If the call failed:
@@ -581,7 +593,6 @@ class AsyncTheCAOffice:
user_id, user_id,
response_json response_json
) )
print("ONE TIME TAKEN:", time.time() - start_time)
return False return False
local/cache/in_out_summary_cache.json
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
local/cache/prev_day_in_out_summary_cache.json
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
scripts/common.py
+46 -32
View File
@@ -267,10 +267,12 @@ async def sync_branches_to_tcaoff(
tcaoff_branches = await tcaoff_client.branch_list() tcaoff_branches = await tcaoff_client.branch_list()
tcaoff_branches = [_["branch_name"].lower() for _ in tcaoff_branches] tcaoff_branches = [_["branch_name"].lower() for _ in tcaoff_branches]
tcaoff_branches = list(set(tcaoff_branches)) tcaoff_branches = list(set(tcaoff_branches))
print("TCAOFF BRANCHES: ", tcaoff_branches)
# Get only the branch names from Cosec: # Get only the branch names from Cosec:
cosec_branches = [tcaoff_client.remove_special_chars(_["Branch Name"]) for _ in cosec_muster_roll["report"]] cosec_branches = [tcaoff_client.remove_special_chars(_["Branch Name"]) for _ in cosec_muster_roll["report"]]
cosec_branches = list(set(cosec_branches)) cosec_branches = list(set(cosec_branches))
print("COSEC BRANCHES: ", cosec_branches)
# Loop through the data from Cosec and add missing branches to TCAOFF: # Loop through the data from Cosec and add missing branches to TCAOFF:
for cosec_branch in cosec_branches: for cosec_branch in cosec_branches:
@@ -338,61 +340,75 @@ async def sync_teams_to_tcaoff(
# Get the list of existing team members from TCAOFF: # Get the list of existing team members from TCAOFF:
# NOTE: `pseudonym` is the unique username of the user. # NOTE: `pseudonym` is the unique username of the user.
tcaoff_teams = await tcaoff_client.team_list() tcaoff_teams = await tcaoff_client.team_list()
synced_team_ids = []
for t in tcaoff_teams: # # OLD WAY - Based on human-readable user names:
app_notes = json.from_string(t["json_notes"]).get("applicantNotes", {}) # # Identify which users have not already been sync'd:
if isinstance(app_notes, str): app_notes = json.from_string(app_notes) # synced_team_ids = []
if app_notes is None: # for t in tcaoff_teams:
app_notes = {} # app_notes = json.from_string(t["json_notes"]).get("applicantNotes", {})
cosec_notes = app_notes.get("cosec", {}) # if isinstance(app_notes, str): app_notes = json.from_string(app_notes)
print("COSEC NOTES:", cosec_notes) # if app_notes is None:
if cosec_notes: # app_notes = {}
user_id = cosec_notes.get("User ID") # cosec_notes = app_notes.get("cosec", {})
if not user_id: user_id = cosec_notes["UserID"] # # print("COSEC NOTES:", cosec_notes)
synced_team_ids.append(user_id) # if cosec_notes:
# user_id = cosec_notes.get("User ID")
# if not user_id: user_id = cosec_notes["UserID"]
# synced_team_ids.append(user_id)
# tcaoff_teams = [_["pseudonym"] for _ in tcaoff_teams]
# tcaoff_teams = list(set(tcaoff_teams))
# NEW WAY - Based on unique Employee Ids:
# Identify which users have not already been sync'd:
synced_team_ids = [_["pseudonym"] for _ in tcaoff_teams]
tcaoff_teams = [_["pseudonym"] for _ in tcaoff_teams] tcaoff_teams = [_["pseudonym"] for _ in tcaoff_teams]
tcaoff_teams = list(set(tcaoff_teams)) tcaoff_teams = list(set(tcaoff_teams))
# Loop through the data from Cosec and add missing team-members to TCAOFF: # Loop through the data from Cosec and add missing team-members to TCAOFF:
for cosec_team in cosec_muster_roll["report"]: for cosec_team in cosec_muster_roll["report"]:
if cosec_team["User ID"] not in synced_team_ids:
# Extract Cosec Details:
cosec_user_id = cosec_team["User ID"].strip()
cosec_user_name = cosec_team["User Name"].strip()
cosec_branch_name = tcaoff_client.remove_special_chars(cosec_team["Branch Name"]).lower()
cosec_dept_name = tcaoff_client.remove_special_chars(cosec_team["Department Name"]).lower()
cosec_grade_name = (cosec_team.get("Grade Name") or "Unknown").strip()
if cosec_user_id not in synced_team_ids:
# Count the user: # Count the user:
response["total"] += 1 response["total"] += 1
# Remove special chars:
branch_name = tcaoff_client.remove_special_chars(cosec_team["Branch Name"]).lower()
dept_name = tcaoff_client.remove_special_chars(cosec_team["Branch Name"]).lower()
# Check the branch id and department id: # Check the branch id and department id:
branch_id = tcaoff_branches_lookup.get(branch_name) branch_id = tcaoff_branches_lookup.get(cosec_branch_name)
dept_id = tcaoff_depts_lookup.get(dept_name) dept_id = tcaoff_depts_lookup.get(cosec_dept_name)
if branch_id is None: if branch_id is None:
err_printer("TEAM SYNC ERR", "Branch Not Found in TCAOFF", branch_name) err_printer("TEAM SYNC ERR", "Branch Not Found in TCAOFF", cosec_branch_name)
response["fail"] += 1 response["fail"] += 1
continue continue
if dept_id is None: if dept_id is None:
err_printer("TEAM SYNC ERR", "Dept. Not Found in TCAOFF", dept_name) err_printer("TEAM SYNC ERR", "Dept. Not Found in TCAOFF", cosec_dept_name)
response["fail"] += 1 response["fail"] += 1
continue continue
# Add the team: # Add the team:
tcaoff_username = regex.replace( tcaoff_username = regex.replace(
text = cosec_team["User Name"], text = cosec_user_name,
pattern = r"[^\w\d\._]", pattern = r"[^\w\d\._]",
substitute_text = "" substitute_text = ""
).strip().lower() ).strip().lower()
tcaoff_email_id = tcaoff_username + "@velankanigroup.com"
# print("Need to Add:", json.to_string(team_json)) # print("Need to Add:", json.to_string(team_json))
success = await tcaoff_client.team_add( success = await tcaoff_client.team_add(
branch_id = branch_id, branch_id = branch_id,
dept_id = dept_id, dept_id = dept_id,
reporting_to = None, reporting_to = None,
team_name = cosec_team["User Name"].strip(), team_name = cosec_user_name,
email = tcaoff_username + "@velankanigroup.com", email = tcaoff_email_id,
phone_no = "9876543210", phone_no = "9876543210",
role = (cosec_team.get("Grade Name") or "Unknown").strip(), role = cosec_grade_name,
username = tcaoff_username, username = cosec_user_id,
password = "Vispl@123", password = "Vispl@123",
applicant_notes = {"cosec": cosec_team} applicant_notes = {"cosec": cosec_team}
) )
@@ -402,7 +418,7 @@ async def sync_teams_to_tcaoff(
response["fail"] += 1 response["fail"] += 1
# Done here: # Done here:
printer("TCAOFF-Cosec Depts. Sync.:", json.to_string(response)) printer("TCAOFF-Cosec Team Sync.:", json.to_string(response))
return response return response
@@ -668,6 +684,7 @@ async def sync_attendance_to_tcaoff(
if not cosec_notes: continue if not cosec_notes: continue
# print("Cosec Notes:", cosec_notes) # print("Cosec Notes:", cosec_notes)
cosec_id_to_tcaoff_team[cosec_notes.get("User ID") or cosec_notes.get("UserID")] = t cosec_id_to_tcaoff_team[cosec_notes.get("User ID") or cosec_notes.get("UserID")] = t
cosec_id_to_tcaoff_team = {k:v for k, v in cosec_id_to_tcaoff_team.items() if k == v["pseudonym"]}
# Create the tasks to fire: # Create the tasks to fire:
tasks = [] tasks = []
@@ -679,7 +696,7 @@ async def sync_attendance_to_tcaoff(
# Match it to the TCAOFF team-member: # Match it to the TCAOFF team-member:
tcaoff_team = cosec_id_to_tcaoff_team.get(cosec_user_id) tcaoff_team = cosec_id_to_tcaoff_team.get(cosec_user_id)
if tcaoff_team is None: if tcaoff_team is None:
printer(f"TCAOFF SYNC ERR: Cosesc User Id '{cosec_user_id}' not found in TCAOFF") printer(f"TCAOFF SYNC ERR: Cosesc User Id not found in TCAOFF", cosec_user_id)
continue continue
# Create the task for this attendance: # Create the task for this attendance:
@@ -704,8 +721,7 @@ async def sync_attendance_to_tcaoff(
yield lst[i:i + size] yield lst[i:i + size]
count = 0 count = 0
for chunk in chunks(tasks[:30]): for chunk in chunks(tasks[:], size = 100):
start_time = time.time()
count += 1 count += 1
printer("Task Chunk:", count) printer("Task Chunk:", count)
_res = await asyncio.gather(*chunk) _res = await asyncio.gather(*chunk)
@@ -714,8 +730,6 @@ async def sync_attendance_to_tcaoff(
results["success"] += success results["success"] += success
results["failure"] += failure results["failure"] += failure
results["total"] += len(_res) results["total"] += len(_res)
print("BATCH TIME TAKEN:", time.time() - start_time)
print("\n\n---\n\n")
# Done here: # Done here:
printer(results) printer(results)
scripts/cron.py
+39 -39
View File
@@ -137,7 +137,7 @@ async def yesterday_cron(
# for the driver's resources to get freed: # for the driver's resources to get freed:
success = common.get_in_out_summary( success = common.get_in_out_summary(
cosec_creds = cosec_creds, cosec_creds = cosec_creds,
from_dt = date_time.get_current_ist_date_time() - datetime.timedelta(days = 2), from_dt = date_time.get_current_ist_date_time() - datetime.timedelta(days = 3),
to_dt = date_time.get_current_ist_date_time(), to_dt = date_time.get_current_ist_date_time(),
cache_file = common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE, cache_file = common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE,
test_mode = test_mode test_mode = test_mode
@@ -196,37 +196,37 @@ async def today_cron(
# Try the part that needs COSEC: # Try the part that needs COSEC:
try: try:
# # MUSTER-ROLL: # MUSTER-ROLL:
#
# # Get the Muster Roll and then wait # Get the Muster Roll and then wait
# # for the driver's resources to get freed: # for the driver's resources to get freed:
# success = common.get_muster_roll( success = common.get_muster_roll(
# cosec_creds = cosec_creds, cosec_creds = cosec_creds,
# on_date = date_time.get_current_ist_date_time().replace( on_date = date_time.get_current_ist_date_time().replace(
# hour = 0, hour = 0,
# minute = 0, minute = 0,
# second = 0 second = 0
# ), ),
# cache_file = common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE, cache_file = common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE,
# test_mode = test_mode test_mode = test_mode
# ) )
#
# # Sync data between Cosec and TCAOFF: # Sync data between Cosec and TCAOFF:
# if success: if success:
# print("MUSTER ROLL: Sync'ing with TCAOFF") print("MUSTER ROLL: Sync'ing with TCAOFF")
# cosec_muster_roll = json.from_file(common.MUSTER_ROLL_CACHE_FILE) cosec_muster_roll = json.from_file(common.MUSTER_ROLL_CACHE_FILE)
# await common.sync_branches_to_tcaoff( await common.sync_branches_to_tcaoff(
# tcaoff_client = tcaoff_client, tcaoff_client = tcaoff_client,
# cosec_muster_roll = cosec_muster_roll, cosec_muster_roll = cosec_muster_roll,
# ) )
# await common.sync_departments_to_tcaoff( await common.sync_departments_to_tcaoff(
# tcaoff_client = tcaoff_client, tcaoff_client = tcaoff_client,
# cosec_muster_roll = cosec_muster_roll, cosec_muster_roll = cosec_muster_roll,
# ) )
# await common.sync_teams_to_tcaoff( await common.sync_teams_to_tcaoff(
# tcaoff_client = tcaoff_client, tcaoff_client = tcaoff_client,
# cosec_muster_roll = cosec_muster_roll, cosec_muster_roll = cosec_muster_roll,
# ) )
# IN-OUT SUMMARY: # IN-OUT SUMMARY:
@@ -281,11 +281,11 @@ async def set_scheduler(
# If in test mode: # If in test mode:
if test_mode: if test_mode:
printer("Starting Test") printer("Starting Test")
# await yesterday_cron( await yesterday_cron(
# cosec_creds = copy.deepcopy(cosec_creds), cosec_creds = copy.deepcopy(cosec_creds),
# tcaoff_client = tcaoff_client, tcaoff_client = tcaoff_client,
# # test_mode = test_mode # test_mode = test_mode
# ) )
await today_cron( await today_cron(
cosec_creds = copy.deepcopy(cosec_creds), cosec_creds = copy.deepcopy(cosec_creds),
tcaoff_client = tcaoff_client, tcaoff_client = tcaoff_client,
@@ -355,12 +355,12 @@ if __name__ == "__main__":
tcaoff_client = AsyncTheCAOffice( tcaoff_client = AsyncTheCAOffice(
username = tcaoff_creds["creds"]["username"], username = tcaoff_creds["creds"]["username"],
password = tcaoff_creds["creds"]["password"], password = tcaoff_creds["creds"]["password"],
debug_only_errors = True debug_only_errors = False
) )
# Schedule the activities: # Schedule the activities:
asyncio.run(set_scheduler( asyncio.run(set_scheduler(
cosec_creds = cosec_creds, cosec_creds = cosec_creds,
tcaoff_client = tcaoff_client, tcaoff_client = tcaoff_client,
test_mode = True test_mode = False
)) ))