khushalps/cosec_automation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20260210) Made a faster way to loop through the attendance records in one go to compute the work done.

Browse Source
This commit is contained in:
Khushal P Soonderji
2026-02-10 13:34:04 +05:30
parent 2484c86d95
commit 3b9621f80e
3 changed files with 222 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
local/cache/prev_day_in_out_summary_cache.json
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
scripts/common.py
+219 -119
View File
@@ -60,7 +60,7 @@ from utils_v2.string import regex
from utils_v2.date_time import date_time from utils_v2.date_time import date_time
# To work with datatypes: # To work with datatypes:
from typing import List, Dict, Any from typing import List, Dict, Any, Union
from collections import defaultdict from collections import defaultdict
# To run a cron-like scheduler: # To run a cron-like scheduler:
@@ -253,137 +253,237 @@ def get_in_out_summary(
# --------------------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------
def compute_work_done(
in_out_df: pd.DataFrame
) -> List[Dict[str, Union[str, int, float, None]]]:
# Create the structure that will be given as the output:
flattened_work_reports = []
# Create an internal dict with the structure:
# work_reports["user"]["date"] = {...}
work_reports = defaultdict(lambda: defaultdict(dict))
# Loop through the full DataFrame once,
# and figure out the first in and last out times:
for index, row in in_out_df.iterrows():
# Extract user and date:
user_id = row["User ID"]
punch_dt = date_time.parse_date_time(row["Punch Time"], timezone = date_time.TIMEZONE_IST)
punch_date = punch_dt.strftime("%Y-%m-%d")
punch_ts = punch_dt.timestamp()
# Handle the first in time:
if row["I/O Type"] == "In":
if work_reports[user_id][punch_date].get("first_in") is None:
work_reports[user_id][punch_date]["first_in"] = punch_ts
# Handle the last out time:
if row["I/O Type"] == "Out":
if work_reports[user_id][punch_date].get("first_in") is not None:
work_reports[user_id][punch_date]["last_out"] = punch_ts
# Now use the first_in and last out times of each record to figure out the amount of work done:
for user_id, user_reports in work_reports.items():
for punch_date, punch_info in user_reports.items():
# Some defaults:
min_work_seconds = 10.0 * 60.0 * 60.0
work_seconds = 0.0
work_ot_seconds = 0.0
work_status = "A"
# Extract, clean and compute punch timing:
first_in = punch_info.get("first_in")
last_out = punch_info.get("last_out")
# When the user has a valid in-time, but no known out time,
# we assume that he worked a full day:
if first_in is not None and not last_out:
last_out = first_in + min_work_seconds
work_seconds = last_out - first_in
work_ot_seconds = 0.0
# When the user has neither an in-time, nor an out-time,
# we assume that he was absent the whole day:
elif not first_in and not last_out:
work_seconds = 0.0
work_ot_seconds = 0.0
# When we have both - an in-time and an out-time - we compute the work done.
# Any work over 10 hours will be counted as overtime:
elif first_in and last_out:
work_seconds = last_out - first_in
work_ot_seconds = max(0.0, work_seconds - min_work_seconds)
# Finally, compute the work status.
# 'A' --> Absent
# 'H' --> Half Day
# 'P' --> Present (Full Day)
# 'OT' -> Overtime
work_hours = work_seconds / (60 * 60)
if work_hours > 10.0: work_status = "OT"
elif 7.5 < work_hours <= 10.0: work_status = "P"
elif 2.5 < work_hours <= 5.0: work_status = "H"
else: work_status = "A"
# Save the data:
flattened_work_reports.append({
"user_id": user_id,
"work_date": punch_date,
"first_in": first_in,
"last_out": last_out,
"work_seconds": work_seconds,
"work_ot_seconds": work_ot_seconds,
"work_hours": work_hours,
"work_status": work_status,
})
# Done here:
return flattened_work_reports
# ---------------------------------------------------------------------------------------------------------------------
async def sync_attendance_to_tcaoff( async def sync_attendance_to_tcaoff(
tcaoff_client: AsyncTheCAOffice, tcaoff_client: AsyncTheCAOffice,
cosec_in_out_summary: dict, cosec_in_out_summary: dict,
verbose: bool = False verbose: bool = False
) -> Dict[str, int]: ) -> Dict[str, int]:
# Get the list of existing team members from TCAOFF: # # Get the list of existing team members from TCAOFF:
# NOTE: `pseudonym` is the unique username of the user. # # NOTE: `pseudonym` is the unique username of the user.
tcaoff_teams = await tcaoff_client.team_list() # tcaoff_teams = await tcaoff_client.team_list()
# print("TCAOFF TEAMS:", json.to_string(tcaoff_teams)) # # print("TCAOFF TEAMS:", json.to_string(tcaoff_teams))
#
# Get a mapping from Cosec id to TCAOFF record: # # Get a mapping from Cosec id to TCAOFF record:
cosec_id_to_tcaoff_team = {} # cosec_id_to_tcaoff_team = {}
for t in tcaoff_teams: # for t in tcaoff_teams:
app_notes = json.from_string(t["json_notes"]).get("applicantNotes", {}) # app_notes = json.from_string(t["json_notes"]).get("applicantNotes", {})
if isinstance(app_notes, str): app_notes = json.from_string(app_notes) # if isinstance(app_notes, str): app_notes = json.from_string(app_notes)
if not app_notes: continue # if not app_notes: continue
cosec_notes = app_notes.get("cosec", {}) # cosec_notes = app_notes.get("cosec", {})
if not cosec_notes: continue # if not cosec_notes: continue
print("Cosec Notes:", cosec_notes) # print("Cosec Notes:", cosec_notes)
cosec_id_to_tcaoff_team[cosec_notes.get("User ID") or cosec_notes.get("UserID")] = t # cosec_id_to_tcaoff_team[cosec_notes.get("User ID") or cosec_notes.get("UserID")] = t
# print("COSEC to TCAOFF TEAMS:", json.to_string(cosec_id_to_tcaoff_team)) # # print("COSEC to TCAOFF TEAMS:", json.to_string(cosec_id_to_tcaoff_team))
# Convert the data to a DataFrame: # Convert the data to a DataFrame:
in_out_df = pd.DataFrame(cosec_in_out_summary["report"]) in_out_df = pd.DataFrame(cosec_in_out_summary["report"])
# in_out_df["Punch Time"] = pd.to_datetime(in_out_df["Punch Time"], unit = "s", utc = True).dt.tz_convert("Asia/Kolkata")
print(in_out_df) print(in_out_df)
in_out_df.info() in_out_df.info()
# Convert the dt column to actual dt objects and apply the timezone on them, # Compute the work done:
# then enlist the unique dates: work_reports = compute_work_done(in_out_df)
in_out_df["Punch Time"] = pd.to_datetime(in_out_df["Punch Time"], unit = "s", utc = True).dt.tz_convert("Asia/Kolkata") print("WORK REPORTS:", work_reports)
unique_dates = sorted(in_out_df['Punch Time'].dt.date.unique()) print("WORK REPORTS:", json.to_string(work_reports))
print(f"UNIQUE DATES ({len(unique_dates)}):", unique_dates)
# Get the unique user ids: # # Convert the dt column to actual dt objects and apply the timezone on them,
unique_cosec_user_ids = in_out_df["User ID"].unique().tolist() # # then enlist the unique dates:
print(f"UNIQUE USER IDS ({len(unique_cosec_user_ids)}):", unique_cosec_user_ids) # in_out_df["Punch Time"] = pd.to_datetime(in_out_df["Punch Time"], unit = "s", utc = True).dt.tz_convert("Asia/Kolkata")
# unique_dates = sorted(in_out_df['Punch Time'].dt.date.unique())
# We will create all async. tasks for firing attendance marking: # print(f"UNIQUE DATES ({len(unique_dates)}):", unique_dates)
tasks = [] #
# # Get the unique user ids:
# For every user: # unique_cosec_user_ids = in_out_df["User ID"].unique().tolist()
for user_count, cosec_user_id in enumerate(unique_cosec_user_ids): # print(f"UNIQUE USER IDS ({len(unique_cosec_user_ids)}):", unique_cosec_user_ids)
#
# Debugging: # # We will create all async. tasks for firing attendance marking:
now_time = date_time.get_current_date_time(as_string = True) # tasks = []
printer("Cosec User:", cosec_user_id, user_count, now_time) #
# # For every user:
# Find the equivalent TCAOFF team member record: # for user_count, cosec_user_id in enumerate(unique_cosec_user_ids):
tcaoff_team = cosec_id_to_tcaoff_team.get(cosec_user_id) #
if tcaoff_team is None: # # Debugging:
print(f"TCAOFF SYNC ERR: Cosesc User Id '{cosec_user_id}' not found in TCAOFF") # now_time = date_time.get_current_date_time(as_string = True)
continue # printer("Cosec User:", cosec_user_id, user_count, now_time)
#
# For every date: # # Find the equivalent TCAOFF team member record:
for work_dt in unique_dates: # tcaoff_team = cosec_id_to_tcaoff_team.get(cosec_user_id)
# if tcaoff_team is None:
# Fetch only the successful events: # print(f"TCAOFF SYNC ERR: Cosesc User Id '{cosec_user_id}' not found in TCAOFF")
user_allowed_events = in_out_df[ # continue
(in_out_df["User ID"] == cosec_user_id) & #
(in_out_df["Event Status"] == "Allowed") & # # For every date:
(in_out_df["Punch Time"].dt.date == work_dt) # for work_dt in unique_dates:
] #
# # Fetch only the successful events:
# Debugging: # user_allowed_events = in_out_df[
weekday = work_dt.weekday() + 1 # ... 1 = Monday, 7 = Sunday # (in_out_df["User ID"] == cosec_user_id) &
events = len(user_allowed_events) # (in_out_df["Event Status"] == "Allowed") &
if verbose: printer(cosec_user_id, work_dt, weekday, events) # (in_out_df["Punch Time"].dt.date == work_dt)
# ]
# Check if the summary is empty: #
if user_allowed_events.empty: # # Debugging:
if verbose: printer("Nothing to sync.", cosec_user_id, work_dt, weekday, events) # weekday = work_dt.weekday() + 1 # ... 1 = Monday, 7 = Sunday
continue # events = len(user_allowed_events)
# if verbose: printer(cosec_user_id, work_dt, weekday, events)
# We note down the first "In" time of the user #
# and the last "Out" time of the user: # # Check if the summary is empty:
first_in = None # if user_allowed_events.empty:
last_out = None # if verbose: printer("Nothing to sync.", cosec_user_id, work_dt, weekday, events)
for idx, row in user_allowed_events.iterrows(): # continue
# print(f"{row['I/O Type']: >4} at row: {idx: <5} | ts: {row['Punch Time']: <15} | dt: {date_time.parse_date_time(row['Punch Time'])}") #
if row["I/O Type"] == "In" and first_in is None: first_in = row["Punch Time"].timestamp() # # We note down the first "In" time of the user
if row["I/O Type"] == "Out" and first_in is not None: last_out = row["Punch Time"].timestamp() # # and the last "Out" time of the user:
# first_in = None
# Figure out the worked time: # last_out = None
if first_in is None and last_out is None: # for idx, row in user_allowed_events.iterrows():
time_worked = { # # print(f"{row['I/O Type']: >4} at row: {idx: <5} | ts: {row['Punch Time']: <15} | dt: {date_time.parse_date_time(row['Punch Time'])}")
"work_seconds": 0.0, # if row["I/O Type"] == "In" and first_in is None: first_in = row["Punch Time"].timestamp()
"work_date": work_dt # if row["I/O Type"] == "Out" and first_in is not None: last_out = row["Punch Time"].timestamp()
} #
elif first_in is None or last_out is None: # # Figure out the worked time:
time_worked = { # if first_in is None and last_out is None:
"work_seconds": 60 * 60 * 10.0, # ... 10 hours represented in seconds. # time_worked = {
"work_date": work_dt # "work_seconds": 0.0,
} # "work_date": work_dt
else: # }
time_worked = { # elif first_in is None or last_out is None:
"work_seconds": last_out - first_in, # time_worked = {
"work_date": work_dt # "work_seconds": 60 * 60 * 10.0, # ... 10 hours represented in seconds.
} # "work_date": work_dt
# }
# Figure out the number of hours worked: # else:
hours_worked = time_worked["work_seconds"] / (60.0 * 60.0) # time_worked = {
if hours_worked > 10.0: status = "OT" # "work_seconds": last_out - first_in,
elif 7.5 < hours_worked <= 10.0: status = "P" # "work_date": work_dt
elif 2.5 < hours_worked <= 5.0: status = "H" # }
else: status = "A" #
# # Figure out the number of hours worked:
# Create this TCAOFF task: # hours_worked = time_worked["work_seconds"] / (60.0 * 60.0)
task = tcaoff_client.attendance_mark( # if hours_worked > 10.0: status = "OT"
user_id = tcaoff_team["user_id"], # elif 7.5 < hours_worked <= 10.0: status = "P"
status = status, # elif 2.5 < hours_worked <= 5.0: status = "H"
over_time = max(0.0, hours_worked - 10.0), # else: status = "A"
attendance_date = time_worked["work_date"].strftime("%Y-%m-%d"), #
json_notes = { # # Create this TCAOFF task:
"totHours": hours_worked, # task = tcaoff_client.attendance_mark(
"firstIn": first_in, # user_id = tcaoff_team["user_id"],
"lastOut": last_out, # status = status,
} # over_time = max(0.0, hours_worked - 10.0),
) # attendance_date = time_worked["work_date"].strftime("%Y-%m-%d"),
tasks.append(task) # json_notes = {
# "totHours": hours_worked,
# break # "firstIn": first_in,
# "lastOut": last_out,
print("TASK COUNT:", len(tasks)) # }
# )
# Now we fire all the tasks: # tasks.append(task)
now_time = date_time.get_current_date_time(as_string = True) #
printer("Marking Attendance", len(tasks), now_time) # # break
results = await asyncio.gather(*tasks) #
now_time = date_time.get_current_date_time(as_string = True) # print("TASK COUNT:", len(tasks))
printer("Marked Attendance", len(tasks), now_time, results) #
# # Now we fire all the tasks:
# now_time = date_time.get_current_date_time(as_string = True)
# printer("Marking Attendance", len(tasks), now_time)
# results = await asyncio.gather(*tasks)
# now_time = date_time.get_current_date_time(as_string = True)
# printer("Marked Attendance", len(tasks), now_time, results)
# ***************************************************************************************************************** # *****************************************************************************************************************
scripts/cron.py
+2 -1
View File
@@ -154,7 +154,7 @@ async def yesterday_cron(
# If something goes wrong in the COSEC step: # If something goes wrong in the COSEC step:
except Exception as exception: except Exception as exception:
err_printer(exception) err_printer(exception)
raise if test_mode: raise
# Log out of TCAOFF: # Log out of TCAOFF:
success = await tcaoff_client.logout() success = await tcaoff_client.logout()
@@ -164,6 +164,7 @@ async def yesterday_cron(
except Exception as exception: except Exception as exception:
err_printer(exception) err_printer(exception)
await tcaoff_client.logout() await tcaoff_client.logout()
if test_mode: raise
# --------------------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------------------