khushalps/cosec_automation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20260317) Tried headless and parallel mode.

Browse Source
This commit is contained in:
Khushal P Soonderji
2026-03-17 18:09:04 +05:30
parent e25daf5fd5
commit 3a68412b1b
12 changed files with 15781 additions and 14463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cosec_web/cosec_web.py
+32 -7
View File
@@ -105,6 +105,9 @@ class CosecWeb:
driver_dir: str,
user_data_dir: str,
downloads_dir: str,
window_width: int = 1920,
window_height: int = 1080,
headless: bool = True,
debug: bool = True,
debug_prefix: str = "Cosec-Web | "
):
@@ -117,6 +120,9 @@ class CosecWeb:
:param driver_dir: The directory where the driver is placed.
:param user_data_dir: The directory where you want the web-driver to store user data.
:param downloads_dir: The directory where you want to hold the downloaded files.
:param window_width: The width of the browser window.
:param window_height: The height of the browser window.
:param headless: Whether, or not, to use Chrome browser in headless (No UI) mode.
:param debug: Whether, or not, to show debug messages.
:param debug_prefix: The prefix to show before the debug messages.
"""
@@ -135,6 +141,9 @@ class CosecWeb:
self.driver_dir = driver_dir
self.user_data_dir = user_data_dir
self.downloads_dir = downloads_dir
self.headless = headless
self.window_width = window_width
self.window_height = window_height
# Figure out which driver to use:
self.driver_path = None
@@ -148,7 +157,7 @@ class CosecWeb:
# If we don't have a matching driver:
if self.driver_path is None: raise NotImplementedError("OS and/or CPU Architecture Not Supported.")
# Else, we initialize the driver:
# Else, we start constructing the preferred configuration::
self._nc_printer("Using driver:", self.driver_path)
self._service = Service(self.driver_path)
self._options = webdriver.ChromeOptions()
@@ -158,26 +167,42 @@ class CosecWeb:
self._options.add_experimental_option(
"prefs",
{
"downloads.default_directory": downloads_dir, # ... The custom downloads path.
"downloads.prompt_for_download": False, # ......... Do not show save dialog.
"downloads.directory_upgrade": True, # ............ If path doesn't exist, try to create it.
"safebrowsing.enabled": True, # .................. Allow safe downloads.
"downloads.default_directory": downloads_dir, # ....... The custom 'downloads' path.
"downloads.prompt_for_download": False, # ............. Do not show save dialog.
"downloads.directory_upgrade": True, # ................ If path doesn't exist, try to create it.
"safebrowsing.enabled": True, # ....................... Allow safe downloads.
"safebrowsing.disable_download_protection": True, # ... Don't show "keep" button before downloading.
"profile.default_content_settings.popups": 0,
"profile.default_content_setting_values.automatic_downloads": 1,
"profile.content_settings.exceptions.automatic_downloads.*.setting": 1,
}
)
# For headless mode:
if headless:
self._printer("HEADLESS MODE!")
self._options.add_argument("--headless=new") # .................................. Modern headless mode.
self._options.add_argument("--disable-gpu") # ................................... Optional (mostly for Windows).
self._options.add_argument(f"--window-size={window_width},{window_height}") # ... The internally simulated screen size.
# Initialize the driver:
self._driver = webdriver.Chrome(service = self._service, options = self._options)
self._driver.set_page_load_timeout(300)
self._driver.execute_cdp_cmd("Page.setDownloadBehavior", {
"behavior": "allow",
"downloadPath": downloads_dir
})
self._printer("Driver ready.")
# For later use, we will need variables to hold window information:
self._original_window = None
self._current_window = None
def __del__(self):
self._printer("Deinitializing!")
try: self.quit()
except Exception as exception: self._printer(exception)
# ┳┓ ┓ •
# ┃┃┏┓┣┓┓┏┏┓┏┓┓┏┓┏┓
# ┻┛┗ ┗┛┗┻┗┫┗┫┗┛┗┗┫
@@ -748,8 +773,8 @@ class CosecWeb:
self._driver.switch_to.window(self._current_window)
self._printer("Context switched.", self._current_window)
# Make the window full-screen for standardized behaviour:
self._driver.maximize_window()
# Set the size of the window so that you are certain that all the elements will be visible:
self._driver.set_window_size(self.window_width, self.window_height)
# The username text field:
element = wait.until(EC.presence_of_element_located((By.ID, "loginid")))
cron/__init__.py
View File
cron/attendance.py
-220
View File
@@ -1,220 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
CREATED: Wed, 26th Nov, 2025
UPDATED: Wed, 26th Nov, 2025
OBJECTIVE:
To achieve so-and-so-objective...
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For system-level activities:
import os
import copy
# To work with date and time:
import time
import datetime
# To work with tabulate data:
import pandas as pd
# Cosec-related:
from cosec_web.cosec_web import CosecWeb
# TCAOFF-related:
from tcaoff import tcaoff
# My utils:
from utils_v2.system import files
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with datatypes:
from typing import List, Dict, Any
from collections import defaultdict
# To run a cron-like scheduler:
from apscheduler.schedulers.background import BackgroundScheduler
# Needed functions:
from cron.reports import *
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def manual_attendance(
cosec_creds: dict,
tcaoff_creds: dict,
target_date: date_time.datetime
) -> None:
"""
To mark yesterday's attendance.
:param cosec_creds: The credentials to use to log into Matrix COSEC.
:param tcaoff_creds: The credentials to use to log into TCAOFF.
:param target_date: The date to sync attendance.
:return: None.
"""
print("ATTENDANCE SYNC")
# Try the whole process once:
try:
# Log in to TCAOFF:
tcaoff.login(tcaoff_creds)
# IN-OUT SUMMARY REPORT:
try:
# Get the previous day's In/Out Summary and then wait
# for the driver's resources to get freed:
success = get_in_out_summary(
cosec_creds = cosec_creds,
target_date = target_date,
cache_file = prev_day_in_out_summary_cache_file
)
# Sync data between Cosec and TCAOFF:
if success:
print("IN/OUT SUMMARY: Sync'ing with TCAOFF")
sync_attendance_to_tcaoff(
tcaoff_creds = tcaoff_creds,
cosec_in_out_summary = json.from_file(prev_day_in_out_summary_cache_file),
)
# If something goes wrong:
except Exception as e:
print("IN-OUT SUMMARY FETCH FAILED!")
raise
# Log out from TCAOFF:
tcaoff.logout(tcaoff_creds)
# If TCAOFF's login or logout fails:
except Exception as e:
print("REPORT CRON FAILED LOOP!")
print("EXCEPTION:", e)
tcaoff.logout(tcaoff_creds)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description = "Manual attendance sync. script.")
parser.add_argument(
"--from-date",
help = "The date from which you want to sync attendance.",
default = date_time.get_current_ist_date_time()
)
parser.add_argument(
"--to-date",
help = "The date till when you want to sync attendance.",
default = date_time.get_current_ist_date_time()
)
args = parser.parse_args()
args.from_date = date_time.parse_date_time(args.from_date, timezone = date_time.TIMEZONE_IST)
args.to_date = date_time.parse_date_time(args.to_date, timezone = date_time.TIMEZONE_IST)
if args.from_date > args.to_date: args.from_date, args.to_date = args.to_date, args.from_date
print("From Date:", args.from_date)
print(" To Date:", args.to_date)
dates_list = [args.from_date]
while args.from_date < args.to_date:
args.from_date = args.from_date + datetime.timedelta(days = 1)
dates_list.append(args.from_date)
print(f"DATES ({len(dates_list)}):", dates_list)
# Load the credentials:
cosec_creds = json.from_file(cosec_creds_file)
tcaoff_creds = json.from_file(tcaoff_creds_file)
# Run the attendance for each specified date:
for target_date in dates_list:
print("TARGET DATE:", target_date)
manual_attendance(
cosec_creds = cosec_creds,
tcaoff_creds = tcaoff_creds,
target_date = date_time.parse_date_time(
target_date,
date_formats = [
"%Y%m%d",
"%Y-%m-%d"
],
) + date_time.timedelta(
hours = 23,
minutes = 59
)
)
cron/reports.py
-926
View File
@@ -1,926 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
CREATED: Wed, 26th Nov, 2025
UPDATED: Wed, 26th Nov, 2025
OBJECTIVE:
To achieve so-and-so-objective...
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For system-level activities:
import os
import copy
# To work with date and time:
import time
import datetime
# To work with tabulate data:
import pandas as pd
# Cosec-related:
from cosec_web.cosec_web import CosecWeb
# TCAOFF-related:
from tcaoff import tcaoff
# My utils:
from utils_v2.system import files
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with datatypes:
from typing import List, Dict, Any
from collections import defaultdict
# To run a cron-like scheduler:
from apscheduler.schedulers.background import BackgroundScheduler
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# File paths:
file_dir = files.get_file_directory(include_filename = False)
proj_dir = files.get_parent_directory(file_dir, depth = 1)
cosec_creds_file = os.path.join(proj_dir, "creds", "cosec.json")
tcaoff_creds_file = os.path.join(proj_dir, "creds", "tcaoff.json")
muster_roll_cache_file = os.path.join(proj_dir, "local", "cache", "muster_roll_cache.json")
in_out_summary_cache_file = os.path.join(proj_dir, "local", "cache", "in_out_summary_cache.json")
prev_day_in_out_summary_cache_file = os.path.join(proj_dir, "local", "cache", "prev_day_in_out_summary_cache.json")
chrome_driver_dir = os.path.join(proj_dir, r"drivers","chrome")
user_data_dir = os.path.join(proj_dir, r"browser", "user_data")
downloads_dir = os.path.join(proj_dir, r"downloads")
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def sync_branches_to_tcaoff(
tcaoff_creds: dict,
cosec_muster_roll: dict,
) -> Dict[str, int]:
# Start with a basic response structure:
response = defaultdict(int)
# Get the list of existing branches from TCAOFF:
tcaoff_branches = tcaoff.branch_list(tcaoff_creds)
tcaoff_branches = [_["branch_name"].lower() for _ in tcaoff_branches]
tcaoff_branches = list(set(tcaoff_branches))
# Get only the branch names from Cosec:
cosec_branches = [tcaoff.remove_special_chars(_["Branch Name"]) for _ in cosec_muster_roll["report"]]
cosec_branches = list(set(cosec_branches))
# Loop through the data from Cosec and add missing branches to TCAOFF:
for cosec_branch in cosec_branches:
if cosec_branch.lower() not in tcaoff_branches:
success = tcaoff.branch_add(
tcaoff_creds,
branch_name = cosec_branch
)
response["total"] += 1
if success: response["success"] += 1
else: response["fail"] += 1
# Done here:
print("TCAOFF-Cosec Branches Sync.:", json.to_string(response))
return response
# ---------------------------------------------------------------------------------------------------------------------
def sync_departments_to_tcaoff(
tcaoff_creds: dict,
cosec_muster_roll: dict,
) -> Dict[str, int]:
# Start with a basic response structure:
response = defaultdict(int)
# Get the list of existing departments from TCAOFF:
tcaoff_depts = tcaoff.department_list(tcaoff_creds)
tcaoff_depts = [_["department_name"].lower() for _ in tcaoff_depts]
tcaoff_depts = list(set(tcaoff_depts))
# print("TCAOFF DEPTS:", tcaoff_depts)
# Get only the department names from Cosec:
cosec_depts = [tcaoff.remove_special_chars(_["Department Name"]) for _ in cosec_muster_roll["report"]]
cosec_depts = list(set(cosec_depts))
# print("COSEC DEPTS:", cosec_depts)
# Loop through the data from Cosec and add missing departments to TCAOFF:
for cosec_dept in cosec_depts:
if cosec_dept.lower() not in tcaoff_depts:
success = tcaoff.department_add(
tcaoff_creds,
department_name = cosec_dept
)
response["total"] += 1
if success: response["success"] += 1
else: response["fail"] += 1
# Done here:
print("TCAOFF-Cosec Depts. Sync.:", json.to_string(response))
return response
# ---------------------------------------------------------------------------------------------------------------------
def sync_teams_to_tcaoff(
tcaoff_creds: dict,
cosec_muster_roll: dict,
) -> Dict[str, int]:
# Start with a basic response structure:
response = defaultdict(int)
# Map out branch ids:
tcaoff_branches = tcaoff.branch_list(tcaoff_creds)
tcaoff_branches_lookup = {d["branch_name"].lower(): d["branch_id"] for d in tcaoff_branches}
# Map out dept. ids:
tcaoff_depts = tcaoff.department_list(tcaoff_creds)
tcaoff_depts_lookup = {d["department_name"].lower():d["department_id"] for d in tcaoff_depts}
# Get the list of existing team members from TCAOFF:
# NOTE: `pseudonym` is the unique username of the user.
tcaoff_teams = tcaoff.team_list(tcaoff_creds)
synced_team_ids = []
for t in tcaoff_teams:
app_notes = json.from_string(t["json_notes"]).get("applicantNotes", {})
if isinstance(app_notes, str): app_notes = json.from_string(app_notes)
if app_notes is None:
# print("NULL APP NOTES??:", t["json_notes"])
app_notes = {}
cosec_notes = app_notes.get("cosec", {})
# print("COSEC NOTES:", cosec_notes)
if cosec_notes: synced_team_ids.append(cosec_notes["User ID"])
tcaoff_teams = [_["pseudonym"] for _ in tcaoff_teams]
tcaoff_teams = list(set(tcaoff_teams))
print("Team Ids:", synced_team_ids)
# # Get only the department names from Cosec:
# cosec_depts = [remove_special_chars(_["Department Name"]) for _ in cosec_muster_roll["report"]]
# cosec_depts = list(set(cosec_depts))
# # print("COSEC DEPTS:", cosec_depts)
# Loop through the data from Cosec and add missing departments to TCAOFF:
for cosec_team in cosec_muster_roll["report"]:
if cosec_team["User ID"] not in synced_team_ids:
# Count the user:
response["total"] += 1
# Check the branch id and department id:
branch_id = tcaoff_branches_lookup.get(tcaoff.remove_special_chars(cosec_team["Branch Name"]).lower())
dept_id = tcaoff_depts_lookup.get(tcaoff.remove_special_chars(cosec_team["Department Name"]).lower())
if branch_id is None:
print(f"TEAM SYNC ERR: Branch '{tcaoff.remove_special_chars(cosec_team['Branch Name'])}' not found in TCAOFF")
response["fail"] += 1
continue
if dept_id is None:
print(f"TEAM SYNC ERR: Dept. '{tcaoff.remove_special_chars(cosec_team['Department Name'])}' not found in TCAOFF")
response["fail"] += 1
continue
# Add the team:
tcaoff_username = regex.replace(
text = cosec_team["User Name"],
pattern = r"[^\w\d\._]",
substitute_text = ""
).strip().lower()
# team_json = {
# "branchId": branch_id,
# "idDepartment": dept_id,
# "reportingTo": None,
# "name": cosec_team['User Name'].strip(),
# "email": tcaoff_username + "@velankanigroup.com",
# "phoneNo": "9876543210",
# "role": cosec_team['Grade Name'].strip(),
# "username": tcaoff_username,
# "password": "Vispl@123",
# "hierarchy": 1,
# "application_notes": {"cosec": cosec_team}
# }
# print("Need to Add:", json.to_string(team_json))
success = tcaoff.team_add(
tcaoff_creds,
branch_id = branch_id,
dept_id = dept_id,
reporting_to = None,
team_name = cosec_team["User Name"].strip(),
email = tcaoff_username + "@velankanigroup.com",
phone_no = "9876543210",
role = (cosec_team.get("Grade Name") or "Unknown").strip(),
username = tcaoff_username,
password = "Vispl@123",
applicant_notes = {"cosec": cosec_team}
)
# response["total"] += 1
if success: response["success"] += 1
else:
print("COULDN'T ADD:", json.to_string(cosec_team))
response["fail"] += 1
# if response["total"] >= 100: break
# Done here:
print("TCAOFF-Cosec Depts. Sync.:", json.to_string(response))
return response
# ---------------------------------------------------------------------------------------------------------------------
def sync_attendance_to_tcaoff(
tcaoff_creds: dict,
cosec_in_out_summary: dict,
) -> Dict[str, int]:
# Get the list of existing team members from TCAOFF:
# NOTE: `pseudonym` is the unique username of the user.
tcaoff_teams = tcaoff.team_list(tcaoff_creds)
print("TCAOFF TEAMS:", json.to_string(tcaoff_teams))
# Get a mapping from Cosec id to TCAOFF record:
cosec_id_to_tcaoff_team = {}
for t in tcaoff_teams:
app_notes = json.from_string(t["json_notes"]).get("applicantNotes", {})
if isinstance(app_notes, str): app_notes = json.from_string(app_notes)
if not app_notes: continue
cosec_notes = app_notes.get("cosec", {})
if not cosec_notes: continue
# print("COSEC NOTES:", cosec_notes)
print("Cosec Notes:", cosec_notes)
# cosec_id_to_tcaoff_team[cosec_notes["User ID"]] = t
cosec_id_to_tcaoff_team[cosec_notes.get("User ID") or cosec_notes.get("UserID")] = t
print("COSEC to TCAOFF TEAMS:", json.to_string(cosec_id_to_tcaoff_team))
# print("Hi, attendance!")
# print(json.to_string(cosec_in_out_summary["report"][0]))
# Convert the data to a DataFrame:
in_out_df = pd.DataFrame(cosec_in_out_summary["report"])
# print(in_out_df[:35].to_string())
# Get the unique user ids:
user_ids = in_out_df["User ID"].unique().tolist()
print(f"USER IDS ({len(user_ids)}):", user_ids)
# Start with a blank user to time mapping:
user_id_to_time = {}
status_count = defaultdict(int)
# For each unique user id,
# Check the no of times when he successfully triggered the attendance-punching device:
for user_id in user_ids:
# Fetch only the successful events:
user_allowed_events = in_out_df[
(in_out_df["User ID"] == user_id) &
(in_out_df["Event Status"] == "Allowed")
]
# Now get the last date (since records can be spread across days),
# and keep the records of only the last day:
last_dt = user_allowed_events["Punch Time"].iloc[-1]
last_dt = date_time.parse_date_time(last_dt)
last_dt = date_time.datetime(
year = last_dt.year,
month = last_dt.month,
day = last_dt.day,
hour = 0,
minute = 0,
second = 0
).timestamp()
print("LAST DATE-TIME:", last_dt)
user_allowed_events = user_allowed_events[user_allowed_events["Punch Time"] >= last_dt]
print("\n\n---\n\n")
print("USER ID:", user_id)
print("\n")
print("Allowed Events:")
print("--------------")
print(user_allowed_events.to_string())
print("\n")
print("Interpretation:")
print("--------------")
# We note down the first "In" time of the user
# and the last "Out" time of the user:
first_in = None
last_out = None
for idx, row in user_allowed_events.iterrows():
print(f"{row['I/O Type']: >4} at row: {idx: <5} | ts: {row['Punch Time']: <15} | dt: {date_time.parse_date_time(row['Punch Time'])}")
if row["I/O Type"] == "In" and first_in is None: first_in = row["Punch Time"]
if row["I/O Type"] == "Out" and first_in is not None: last_out = row["Punch Time"]
print("\n")
print("Timestamps:")
print("----------")
print("FIRST-IN:", date_time.parse_date_time(first_in))
print("LAST-OUT:", date_time.parse_date_time(last_out))
# Figure out the worked time:
if first_in is None and last_out is None:
user_id_to_time[user_id] = {
"work_seconds": 0.0,
"work_date": last_dt
}
elif first_in is None or last_out is None:
user_id_to_time[user_id] = {
"work_seconds": 60 * 60 * 10.0, # ... 10 hours represented in seconds.
"work_date": last_dt
}
else:
user_id_to_time[user_id] = {
"work_seconds": last_out - first_in,
"work_date": last_dt
}
# Add fixed known details:
user_id_to_time[user_id]["first_in"] = first_in
user_id_to_time[user_id]["last_out"] = last_out
print("\n")
print("Result:")
print("------")
print("Total Seconds :", user_id_to_time[user_id]["work_seconds"])
print("Total Hours :", user_id_to_time[user_id]["work_seconds"] / (60 * 60))
# break
# # Wait till you get an "IN" event, ten wait till you get an "OUT" event.
# # Keep totalling between them. If you end with an "IN" event, consider the
# # current time as the "OUT" event:
# in_at = None
# for idx, row in user_allowed_events.iterrows():
# if row["I/O Type"] == "In" and in_at is None:
# print(f" In at row: {idx: <5} | ts: {row['Punch Time']: <15} | dt: {date_time.parse_date_time(row['Punch Time'])}")
# in_at = row["Punch Time"]
# if row["I/O Type"] == "Out" and in_at is not None:
# print(f"Out at row: {idx: <5} | ts: {row['Punch Time']: <15} | dt: {date_time.parse_date_time(row['Punch Time'])}")
# user_id_to_time[user_id] += row["Punch Time"] - in_at
# in_at = None
# # if in_at is not None:
# # user_id_to_time[user_id] += date_time.get_current_date_time().timestamp() - in_at
# print("\n")
# print("Result:")
# print("------")
# print("Total Seconds :", user_id_to_time[user_id])
# print("Total Hours :", user_id_to_time[user_id] / (60*60))
# # break
# # Show the results:
# user_id_to_hours = {k:v/(60*60) for k, v in user_id_to_time.items()}
# print("\n\n---\n\n")
# print("WORK-HOURS DONE:", json.to_string(user_id_to_hours))
# Start with a basic response structure:
response = defaultdict(int)
# At this point, you have the amount of work done be each Cosec team member.
# Now we loop through the results and sync them on TCAOFF:
for user_id, work in user_id_to_time.items():
response["total"] += 1
# Find the equivalent TCAOFF team member record:
tcaoff_team = cosec_id_to_tcaoff_team.get(user_id)
if tcaoff_team is None:
print(f"TCAOFF SYNC ERR: Cosesc User Id '{user_id}' not found in TCAOFF")
response["fail"] += 1
continue
# Update the attendance on TCAOFF:
hours_worked = work["work_seconds"] / (60.0 * 60.0)
if hours_worked > 10.0: status = "OT"
elif 7.5 < hours_worked <= 10.0: status = "P"
elif 2.5 < hours_worked <= 5.0: status = "H"
else: status = "A"
status_count[status] += 1
success = tcaoff.attendance_mark(
tcaoff_creds,
user_id = tcaoff_team["user_id"],
status = status,
over_time = max(0.0, hours_worked - 10.0),
attendance_date = work["work_date"],
json_notes = {
"totHours": hours_worked,
"firstIn": work["first_in"],
"lastOut": work["last_out"],
}
)
if success: response["success"] += 1
else: response["fail"] += 1
# Show the stats:
print("WORKING HOUR STATUS STATS:", json.to_string(status_count))
# Done here:
print("TCAOFF-Cosec Attendance Sync.:", json.to_string(response))
return response
# ---------------------------------------------------------------------------------------------------------------------
def kill_chrome() -> None:
# First kill the previous processes,
# then wait if old processes were killed:
kill_count = CosecWeb.kill_chrome_processes()
if kill_count > 0: time.sleep(2.5)
# ---------------------------------------------------------------------------------------------------------------------
def get_muster_roll(
cosec_creds: dict,
cache_file: str = muster_roll_cache_file
) -> bool:
"""
Get the latest Muster-Roll from Matrix Cosec. It saves the data into a local cache file.
:param cosec_creds: The credentials (and config) to operate Cosec Matrix.
:param cache_file: The cache file to store the results in.
:return: True if the automated fetch was successful, else False.
"""
# Start by assuming failure:
success = False
# Force close other running Chrome processes:
kill_chrome()
# Create an instance of the automation object:
cosec = CosecWeb(
cosec_url = cosec_creds["creds"]["url"],
username = cosec_creds["creds"]["username"],
password = cosec_creds["creds"]["password"],
driver_dir = chrome_driver_dir,
user_data_dir = user_data_dir,
downloads_dir = downloads_dir,
)
# Perform the login:
cosec.login(initial_sleep = 2.5)
# Get the in/out report:
report_path = cosec.get_muster_roll(
initial_sleep = 1.0,
on_date = date_time.get_current_utc_date_time(),
group_ids = cosec_creds["musterRollConfig"]["groupIds"],
download_timeout = 60.0
)
# Log out to end the cycle:
cosec.logout()
# Close the browser window:
cosec.quit()
# report_path = r"D:\kps\PycharmProjects\cosec\cosec_web\sample_files\muster_roll.xls"
# Now process the report,
# and save it to the JSON file:
if report_path is not None:
# Read the data:
report_data = CosecWeb.read_muster_roll_xls(report_path)
# Do the remaining cleanup and formatting:
report_data = report_data[[
"User ID", "User Name", "Category Name",
"Grade Name", "Branch Name", "Department Name",
"Direct Reporting", "Level-1"
]]
report_data = report_data.where(report_data.notna(), None)
report_data = report_data.to_dict(orient = "records")
# Loop through the report for inferred data.
# Add the ids of the "Direct Reporting" and "Level-1" values:
for person in report_data:
person["Direct Reporting ID"] = None
person["Level-1 ID"] = None
for check_against in report_data:
if person["Direct Reporting"] == check_against["User Name"]:
person["Direct Reporting ID"] = check_against["User ID"]
if person["Level-1"] == check_against["User Name"]:
person["Level-1 ID"] = check_against["User ID"]
# Sort by hierarchy (BFS):
pass
print(f"MUSTER ROLL ({len(report_data)}):", json.to_string(report_data))
# Save the data to a JSON file:
json.to_file(
file = cache_file,
python_data = {
"ts": date_time.get_current_utc_date_time(as_string = False).timestamp(),
"report": report_data
},
no_space = True
)
# Note down success:
success = True
# Done here:
return success
# ---------------------------------------------------------------------------------------------------------------------
def get_in_out_summary(
cosec_creds: dict,
target_date: date_time.datetime = None,
cache_file: str = in_out_summary_cache_file,
) -> bool:
"""
Get the latest In-Out-Summary from Matrix Cosec. It saves the data into a local cache file.
:param cosec_creds: The credentials (and config) to operate Cosec Matrix.
:param target_date: The date to check for in Out-Of-Out-Summary.
:param cache_file: The cache file to use to store the results.
:return: True if the automated fetch was successful, else False.
"""
# Start by assuming failure:
success = False
# Force close other running Chrome processes:
kill_chrome()
# Create an instance of the automation object:
cosec = CosecWeb(
cosec_url = cosec_creds["creds"]["url"],
username = cosec_creds["creds"]["username"],
password = cosec_creds["creds"]["password"],
driver_dir = chrome_driver_dir,
user_data_dir = user_data_dir,
downloads_dir = downloads_dir,
)
# Perform the login:
cosec.login(initial_sleep = 2.5)
# Get the in/out report:
if target_date is None: target_date = date_time.get_current_ist_date_time()
from_dt = target_date - datetime.timedelta(
days = cosec_creds["generalConfig"]["timedelta"]["days"],
hours = cosec_creds["generalConfig"]["timedelta"]["hours"],
minutes = cosec_creds["generalConfig"]["timedelta"]["minutes"],
seconds = cosec_creds["generalConfig"]["timedelta"]["seconds"],
)
to_dt = target_date
report_path = cosec.get_in_out_summary(
initial_sleep = 1.0,
from_date = from_dt,
to_date = to_dt,
group_ids = cosec_creds["inOutConfig"]["groupIds"],
download_timeout = 60.0,
timezone = cosec_creds["generalConfig"]["timezone"],
)
# Log out to end the cycle:
cosec.logout()
# Close the browser window:
cosec.quit()
# report_path = r"D:\kps\PycharmProjects\cosec\cosec_web\sample_files\in_out_summary.xls"
# Now process the report,
# and save it to the JSON file:
if report_path is not None:
# Read the data:
report_data = CosecWeb.read_in_out_summary_xls(report_path)
# Assume that the punch time in the data is IST data,
# then normalize it to UTC:
def parse_dt(x):
if pd.isnull(x): return None
else: return date_time.to_timezone(
datetime_object = date_time.as_if_timezone(
datetime_object = date_time.parse_date_time(x),
timezone = cosec_creds["generalConfig"]["timezone"]
),
timezone = date_time.TIMEZONE_UTC
).timestamp()
report_data["Punch Time"] = report_data["Punch Time"].apply(lambda x: parse_dt(x))
# Do the remaining cleanup and formatting:
report_data = report_data.where(report_data.notna(), None)
report_data = report_data.to_dict(orient = "records")
report_data = {
"ts": date_time.get_current_utc_date_time(as_string = False).timestamp(),
"report": report_data
}
# Save the data to a JSON file:
json.to_file(
file = cache_file,
python_data = report_data,
no_space = True
)
# Note down success:
success = True
# Done here:
return success
# ---------------------------------------------------------------------------------------------------------------------
def today_cron(
cosec_creds: dict,
tcaoff_creds: dict,
) -> None:
"""
To get today's stats about the team.
:param cosec_creds: The credentials to use to log into Matrix COSEC.
:param tcaoff_creds: The credentials to use to log into TCAOFF.
:return: None.
"""
print("TODAY CRON")
# Try the whole process once:
try:
# Log in to TCAOFF:
tcaoff.login(tcaoff_creds)
# MUSTER-ROLL REPORT:
try:
# Get the Muster Roll and then wait
# for the driver's resources to get freed:
success = get_muster_roll(
cosec_creds = cosec_creds,
cache_file = muster_roll_cache_file
)
# Sync data between Cosec and TCAOFF:
if success:
print("MUSTER ROLL: Sync'ing with TCAOFF")
cosec_muster_roll = json.from_file(muster_roll_cache_file)
sync_branches_to_tcaoff(
tcaoff_creds = tcaoff_creds,
cosec_muster_roll = cosec_muster_roll,
)
sync_departments_to_tcaoff(
tcaoff_creds = tcaoff_creds,
cosec_muster_roll = cosec_muster_roll,
)
sync_teams_to_tcaoff(
tcaoff_creds = tcaoff_creds,
cosec_muster_roll = cosec_muster_roll,
)
# If something goes wrong:
except Exception as e:
print("MUSTER ROLL FETCH FAILED!")
print("EXCEPTION:", e)
# IN-OUT REPORT:
try:
# Get the previous day's In/Out Summary and then wait
# for the driver's resources to get freed:
success = get_in_out_summary(
cosec_creds = cosec_creds,
target_date = date_time.get_current_ist_date_time(),
cache_file = in_out_summary_cache_file
)
# Sync data between Cosec and TCAOFF:
if success:
print("IN/OUT SUMMARY: Sync'ing with TCAOFF")
sync_attendance_to_tcaoff(
tcaoff_creds = tcaoff_creds,
cosec_in_out_summary = json.from_file(prev_day_in_out_summary_cache_file),
)
# If something goes wrong:
except Exception as e:
print("IN-OUT SUMMARY FETCH FAILED!")
print("EXCEPTION:", e)
raise
# Log out from TCAOFF:
tcaoff.logout(tcaoff_creds)
# If TCAOFF's login or logout fails:
except Exception as e:
print("REPORT CRON FAILED LOOP!")
print("EXCEPTION:", e)
tcaoff.logout(tcaoff_creds)
raise
# ---------------------------------------------------------------------------------------------------------------------
def yesterday_cron(
cosec_creds: dict,
tcaoff_creds: dict,
) -> None:
"""
To mark yesterday's attendance.
:param cosec_creds: The credentials to use to log into Matrix COSEC.
:param tcaoff_creds: The credentials to use to log into TCAOFF.
:return: None.
"""
print("YESTERDAY CRON")
# Try the whole process once:
try:
# Log in to TCAOFF:
tcaoff.login(tcaoff_creds)
# IN-OUT SUMMARY REPORT:
try:
# Get the previous day's In/Out Summary and then wait
# for the driver's resources to get freed:
success = get_in_out_summary(
cosec_creds = cosec_creds,
target_date = date_time.get_current_ist_date_time() - datetime.timedelta(days = 1),
cache_file = prev_day_in_out_summary_cache_file
)
# Sync data between Cosec and TCAOFF:
if success:
print("IN/OUT SUMMARY: Sync'ing with TCAOFF")
sync_attendance_to_tcaoff(
tcaoff_creds = tcaoff_creds,
cosec_in_out_summary = json.from_file(prev_day_in_out_summary_cache_file),
)
# If something goes wrong:
except Exception as e:
print("IN-OUT SUMMARY FETCH FAILED!")
print("EXCEPTION:", e)
raise
# Log out from TCAOFF:
tcaoff.logout(tcaoff_creds)
# If TCAOFF's login or logout fails:
except Exception as e:
print("REPORT CRON FAILED LOOP!")
print("EXCEPTION:", e)
tcaoff.logout(tcaoff_creds)
# ---------------------------------------------------------------------------------------------------------------------
def set_scheduler(
cosec_creds: dict,
tcaoff_creds: dict,
) -> None:
"""
Set up the scheduler (cron) that will time the activities. The background task happens at fixed times, the
foreground task happens in a loop.
:param cosec_creds: The credentials to use to log into Matrix COSEC.
:param tcaoff_creds: The credentials to use to log into TCAOFF.
:return: None.
"""
# Create the scheduler:
scheduler = BackgroundScheduler()
# Populate the tasks in the scheduler:
for h, m in zip(
[8, 23],
[30, 30]
):
scheduler.add_job(
yesterday_cron,
"cron",
hour = h,
minute = m,
args = [
copy.deepcopy(cosec_creds),
copy.deepcopy(tcaoff_creds)
]
)
# Populate the tasks in the scheduler:
for h in range(0, 23):
for m in [0]:
scheduler.add_job(
today_cron,
"cron",
hour = h,
minute = m,
args = [
copy.deepcopy(cosec_creds),
copy.deepcopy(tcaoff_creds)
]
)
# Run the scheduler:
scheduler.start()
while True: time.sleep(10.0)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
# Explicitly mention the expected file paths for other devs to maintain:
print("PROJ. DIR. :", proj_dir)
print("CREDS FILE :", cosec_creds_file)
print("M-ROLL CACHE:", muster_roll_cache_file)
print("IN-OUT CACHE:", in_out_summary_cache_file)
# Read required credentials:
cosec_creds = json.from_file(cosec_creds_file)
tcaoff_creds = json.from_file(tcaoff_creds_file)
# Schedule the activities:
set_scheduler(
cosec_creds = cosec_creds,
tcaoff_creds = tcaoff_creds,
)
drivers/chrome/linux64/THIRD_PARTY_NOTICES.chromedriver
+14550 -13188
View File
File diff suppressed because it is too large Load Diff
drivers/chrome/linux64/chromedriver
BIN
View File
Binary file not shown.
drivers/chrome/win64/THIRD_PARTY_NOTICES.chromedriver
+1050 -59
View File
File diff suppressed because it is too large Load Diff
drivers/chrome/win64/chromedriver.exe
BIN
View File
Binary file not shown.
scripts/common.py
+77 -26
View File
@@ -22,8 +22,7 @@
N/A
"""
import pathlib
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
@@ -60,12 +59,9 @@ from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with datatypes:
from typing import List, Dict, Any, Union
from typing import List, Dict, Any, Union, Callable
from collections import defaultdict
# To run a cron-like scheduler:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
# For async activities:
import asyncio
@@ -85,17 +81,18 @@ FILE_DIR = files.get_file_directory(include_filename = False)
PROJ_DIR = files.get_parent_directory(FILE_DIR, depth = 1)
CACHE_DIR = os.path.join(PROJ_DIR, "local", "cache")
CREDS_DIR = os.path.join(PROJ_DIR, "creds")
# ---
COSEC_CREDS_FILE = os.path.join(CREDS_DIR, "cosec.json")
TCAOFF_CREDS_FILE = os.path.join(CREDS_DIR, "tcaoff.json")
MUSTER_ROLL_CACHE_FILE = os.path.join(CACHE_DIR, "muster_roll_cache.json")
IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "in_out_summary_cache.json")
PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "prev_day_in_out_summary_cache.json")
MANUAL_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "manual_in_out_summary_cache.json")
WORK_REPORTS_CACHE_FILE = os.path.join(CACHE_DIR, "work_reports_cache.json")
# # ---
# COSEC_CREDS_FILE = os.path.join(CREDS_DIR, "cosec.json")
# TCAOFF_CREDS_FILE = os.path.join(CREDS_DIR, "tcaoff.json")
# MUSTER_ROLL_CACHE_FILE = os.path.join(CACHE_DIR, "muster_roll_cache.json")
# IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "in_out_summary_cache.json")
# PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "prev_day_in_out_summary_cache.json")
# MANUAL_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "manual_in_out_summary_cache.json")
# WORK_REPORTS_CACHE_FILE = os.path.join(CACHE_DIR, "work_reports_cache.json")
# # ---
CHROME_DRIVER_DIR = os.path.join(PROJ_DIR, "drivers", "chrome")
USER_DATA_DIR = os.path.join(PROJ_DIR, "browser", "user_data")
DOWNLOADS_DIR = os.path.join(PROJ_DIR, "downloads")
# USER_DATA_DIR = os.path.join(PROJ_DIR, "browser", "user_data", os.environ.get("ORG", "default"))
# DOWNLOADS_DIR = os.path.join(PROJ_DIR, "downloads", os.environ.get("ORG", "default"))
# ---
TEST_MODE_MUSTER_ROLL_FILE_PATH = os.path.join(PROJ_DIR, "cosec_web", "sample_files", "muster_roll.xls")
TEST_MODE_IN_OUT_SUMMARY_FILE_PATH = os.path.join(PROJ_DIR, "cosec_web", "sample_files", "in_out_summary.xls")
@@ -137,6 +134,41 @@ err_printer = IceCreamDebugger(prefix = "[ERR] Common | ", includeContext = True
# *****************************************************************************************************************
def get_muster_roll_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "muster_roll_cache.json")
def get_in_out_summary_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "in_out_summary_cache.json")
def get_prev_day_in_out_summary_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "prev_day_in_out_summary_cache.json")
def get_manual_in_out_summary_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "manual_in_out_summary_cache.json")
def get_work_reports_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "work_reports_cache.json")
# ---
def get_browser_user_data_directory(): return os.path.join(PROJ_DIR, "browser", "user_data", os.environ.get("ORG", "default"))
def get_browser_downloads_directory(): return os.path.join(PROJ_DIR, "downloads", os.environ.get("ORG", "default"))
# ---------------------------------------------------------------------------------------------------------------------
def init_paths():
"""
We will be segregating the data of the various organizations by their sub-dirs. This quick function ensures that
those segregated paths exist. Call it at the start of your script when you have set the organization name to the
temporary environment variable.
"""
for path in [
os.path.join(CACHE_DIR, os.environ.get("ORG", "default")),
os.path.join(PROJ_DIR, "browser", "user_data", os.environ.get("ORG", "default")),
os.path.join(PROJ_DIR, "downloads", os.environ.get("ORG", "default"))
]:
if not os.path.exists(path):
printer("Making", path)
files.make_directory(path)
# ---------------------------------------------------------------------------------------------------------------------
def kill_chrome() -> None:
"""
@@ -148,6 +180,7 @@ def kill_chrome() -> None:
# then wait if old processes were killed:
kill_count = CosecWeb.kill_chrome_processes()
if kill_count > 0: time.sleep(2.5)
printer(kill_count)
# ---------------------------------------------------------------------------------------------------------------------
@@ -156,7 +189,7 @@ def kill_chrome() -> None:
def get_muster_roll(
cosec_creds: dict,
on_date: datetime.datetime,
cache_file: str = MUSTER_ROLL_CACHE_FILE,
cache_file: str | Callable | None = None,
test_mode: bool = False
) -> bool:
@@ -169,6 +202,12 @@ def get_muster_roll(
:return: True if the automated fetch was successful, else False.
"""
# Figure out path(s):
if not isinstance(cache_file, str):
if isinstance(cache_file, Callable): cache_file = cache_file()
else: cache_file = get_muster_roll_cache_file_path()
printer(cache_file)
# Start by assuming failure:
success = False
report_path = None
@@ -189,8 +228,9 @@ def get_muster_roll(
username = cosec_creds["creds"]["username"],
password = cosec_creds["creds"]["password"],
driver_dir = CHROME_DRIVER_DIR,
user_data_dir = USER_DATA_DIR,
downloads_dir = DOWNLOADS_DIR,
user_data_dir = get_browser_user_data_directory(),
downloads_dir = get_browser_downloads_directory(),
headless = False
)
# Perform the login:
@@ -314,7 +354,7 @@ async def sync_departments_to_tcaoff(
# Loop through the data from Cosec and add missing departments to TCAOFF:
for cosec_dept in cosec_depts:
if cosec_dept.lower() not in tcaoff_depts:
success = tcaoff_client.department_add(department_name = cosec_dept)
success = await tcaoff_client.department_add(department_name = cosec_dept)
response["total"] += 1
if success: response["success"] += 1
else: response["fail"] += 1
@@ -434,7 +474,7 @@ def get_in_out_summary(
cosec_creds: dict,
from_dt: date_time.datetime = None,
to_dt: date_time.datetime = None,
cache_file: str = IN_OUT_SUMMARY_CACHE_FILE,
cache_file: str | Callable | None = None,
test_mode: bool = False
) -> bool:
@@ -448,6 +488,12 @@ def get_in_out_summary(
:return: True if the automated fetch was successful, else False.
"""
# Figure out path(s):
if not isinstance(cache_file, str):
if isinstance(cache_file, Callable): cache_file = cache_file()
else: cache_file = get_in_out_summary_cache_file_path()
printer(cache_file)
# Start by assuming failure:
success = False
report_path = None
@@ -468,8 +514,9 @@ def get_in_out_summary(
username = cosec_creds["creds"]["username"],
password = cosec_creds["creds"]["password"],
driver_dir = CHROME_DRIVER_DIR,
user_data_dir = USER_DATA_DIR,
downloads_dir = DOWNLOADS_DIR,
user_data_dir = get_browser_user_data_directory(),
downloads_dir = get_browser_downloads_directory(),
headless = False
)
# Perform the login:
@@ -522,6 +569,7 @@ def get_in_out_summary(
}
# Save the data to a JSON file:
printer("Saving", cache_file)
json.to_file(
file = cache_file,
python_data = report_data,
@@ -709,6 +757,9 @@ async def sync_attendance_to_tcaoff(
:return: The dict that gives you the count of the successful and failed attendance marking API calls.
"""
# Figure out path(s):
work_reports_cache_file = get_work_reports_cache_file_path()
# Results:
results = defaultdict(int)
@@ -753,8 +804,8 @@ async def sync_attendance_to_tcaoff(
# Now we save the work reports for next time,
# and then we check for the ones that have changed:
work_reports = []
if os.path.exists(WORK_REPORTS_CACHE_FILE):
cached_work_reports = json.from_file(WORK_REPORTS_CACHE_FILE)
if os.path.exists(work_reports_cache_file):
cached_work_reports = json.from_file(work_reports_cache_file)
hashed_new_work_reports = {
wr["user_id"] + "." + wr["work_date"] : wr
for wr in new_work_reports
@@ -767,7 +818,7 @@ async def sync_attendance_to_tcaoff(
if v == hashed_old_work_reports.get(k, {}): continue
work_reports.append(v)
else: work_reports = new_work_reports
json.to_file(WORK_REPORTS_CACHE_FILE, new_work_reports)
json.to_file(work_reports_cache_file, new_work_reports)
# Create the tasks to fire:
tasks = []
scripts/cron.py
+40 -22
View File
@@ -112,7 +112,7 @@ err_printer = IceCreamDebugger(prefix = "[ERR] Cron | ", includeContext = True)
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# *****************************************************************************************************************-
async def yesterday_cron(
@@ -139,7 +139,7 @@ async def yesterday_cron(
cosec_creds = cosec_creds,
from_dt = date_time.get_current_ist_date_time() - datetime.timedelta(days = 3),
to_dt = date_time.get_current_ist_date_time(),
cache_file = common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE,
cache_file = common.get_prev_day_in_out_summary_cache_file_path,
test_mode = test_mode
)
@@ -148,7 +148,7 @@ async def yesterday_cron(
printer("IN/OUT SUMMARY: Sync'ing with TCAOFF")
await common.sync_attendance_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_in_out_summary = json.from_file(common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE),
cosec_in_out_summary = json.from_file(common.get_prev_day_in_out_summary_cache_file_path()),
chunk_size = 10
)
@@ -198,7 +198,7 @@ async def today_cron(
try:
# MUSTER-ROLL:
# -----------
# Get the Muster Roll and then wait
# for the driver's resources to get freed:
success = common.get_muster_roll(
@@ -208,14 +208,14 @@ async def today_cron(
minute = 0,
second = 0
),
cache_file = common.MUSTER_ROLL_CACHE_FILE,
cache_file = common.get_muster_roll_cache_file_path,
test_mode = test_mode
)
# Sync data between Cosec and TCAOFF:
if success:
print("MUSTER ROLL: Sync'ing with TCAOFF")
cosec_muster_roll = json.from_file(common.MUSTER_ROLL_CACHE_FILE)
cosec_muster_roll = json.from_file(common.get_muster_roll_cache_file_path())
await common.sync_branches_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_muster_roll = cosec_muster_roll,
@@ -230,14 +230,14 @@ async def today_cron(
)
# IN-OUT SUMMARY:
# --------------
# Get the previous day's In/Out Summary and then wait
# for the driver's resources to get freed:
success = common.get_in_out_summary(
cosec_creds = cosec_creds,
from_dt = date_time.get_current_ist_date_time().replace(hour = 0, minute = 0, second = 0, microsecond = 0),
to_dt = date_time.get_current_ist_date_time(),
cache_file = common.IN_OUT_SUMMARY_CACHE_FILE,
cache_file = common.get_in_out_summary_cache_file_path,
test_mode = test_mode
)
@@ -246,7 +246,7 @@ async def today_cron(
printer("IN/OUT SUMMARY: Sync'ing with TCAOFF")
await common.sync_attendance_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_in_out_summary = json.from_file(common.IN_OUT_SUMMARY_CACHE_FILE),
cosec_in_out_summary = json.from_file(common.get_in_out_summary_cache_file_path()),
chunk_size = 10
)
@@ -354,18 +354,20 @@ if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument(
"--test", "--test-mode",
help = "Picks the last cache file instead of fetching records from COSEC.",
action = "store_true",
default = False,
)
ap.add_argument(
"--immediate",
help = "To do one run immediately first, then set the scheduler.",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--verbose",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--cosec-creds",
@@ -377,16 +379,30 @@ if __name__ == "__main__":
help = "The credentials JSON from which you would like to connect to TheCAOffice.",
default = "tcaoff.json"
)
ap.add_argument(
"--org", "--organization",
help = "When handling multiple organizations, this word will be used to keep their files separate.",
default = "default"
)
args = ap.parse_args()
# Hold temporary environment variables:
os.environ["ORG"] = args.org
# Read required credentials:
cosec_creds_path = os.path.join(common.CREDS_DIR, args.cosec_creds)
tcaoff_creds_path = os.path.join(common.CREDS_DIR, args.tcaoff_creds)
# ---
cosec_creds = json.from_file(cosec_creds_path)
tcaoff_creds = json.from_file(tcaoff_creds_path)
# Explicitly mention the expected file paths for other devs to maintain:
print("PROJ. DIR. :", common.PROJ_DIR)
print("COSEC CREDS :", common.COSEC_CREDS_FILE)
print("TCAOFF CREDS:", common.TCAOFF_CREDS_FILE)
print("COSEC CREDS :", cosec_creds_path)
print("TCAOFF CREDS:", tcaoff_creds_path)
# Read required credentials:
cosec_creds = json.from_file(os.path.join(common.CREDS_DIR, args.cosec_creds))
tcaoff_creds = json.from_file(os.path.join(common.CREDS_DIR, args.tcaoff_creds))
# Ensure that certain required paths exist:
common.init_paths()
# Create the clients:
tcaoff_client = AsyncTheCAOffice(
@@ -397,9 +413,11 @@ if __name__ == "__main__":
)
# Schedule the activities:
asyncio.run(set_scheduler(
cosec_creds = cosec_creds,
tcaoff_client = tcaoff_client,
immediate = args.immediate,
test_mode = args.test
))
asyncio.run(
set_scheduler(
cosec_creds = cosec_creds,
tcaoff_client = tcaoff_client,
immediate = args.immediate,
test_mode = args.test
)
)
scripts/manual_attendance.py
+24 -14
View File
@@ -149,16 +149,17 @@ async def manual_attendance(
cosec_creds = cosec_creds,
from_dt = from_date,
to_dt = to_date,
cache_file = common.MANUAL_SUMMARY_CACHE_FILE,
cache_file = common.get_manual_in_out_summary_cache_file_path,
test_mode = test_mode
)
# Sync data between Cosec and TCAOFF:
if success:
printer("IN/OUT SUMMARY: Sync'ing with TCAOFF")
print("SYNC FROM:", common.get_manual_in_out_summary_cache_file_path())
await common.sync_attendance_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_in_out_summary = json.from_file(common.MANUAL_SUMMARY_CACHE_FILE),
cosec_in_out_summary = json.from_file(common.get_manual_in_out_summary_cache_file_path()),
chunk_size = 10
)
@@ -193,12 +194,12 @@ if __name__ == "__main__":
ap.add_argument(
"--test", "--test-mode",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--verbose",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--from-date",
@@ -220,16 +221,30 @@ if __name__ == "__main__":
help = "The credentials JSON from which you would like to connect to TheCAOffice.",
default = "tcaoff.json"
)
ap.add_argument(
"--org", "--organization",
help = "When handling multiple organizations, this word will be used to keep their files separate.",
default = "default"
)
args = ap.parse_args()
# Hold temporary environment variables:
os.environ["ORG"] = args.org
# Read required credentials:
cosec_creds_path = os.path.join(common.CREDS_DIR, args.cosec_creds)
tcaoff_creds_path = os.path.join(common.CREDS_DIR, args.tcaoff_creds)
# ---
cosec_creds = json.from_file(cosec_creds_path)
tcaoff_creds = json.from_file(tcaoff_creds_path)
# Explicitly mention the expected file paths for other devs to maintain:
print("PROJ. DIR. :", common.PROJ_DIR)
print("COSEC CREDS :", common.COSEC_CREDS_FILE)
print("TCAOFF CREDS:", common.TCAOFF_CREDS_FILE)
print("COSEC CREDS :", cosec_creds_path)
print("TCAOFF CREDS:", tcaoff_creds_path)
# Read required credentials:
cosec_creds = json.from_file(os.path.join(common.CREDS_DIR, args.cosec_creds))
tcaoff_creds = json.from_file(os.path.join(common.CREDS_DIR, args.tcaoff_creds))
# Ensure that certain required paths exist:
common.init_paths()
# Date-handling:
args.from_date = date_time.parse_date_time(args.from_date, timezone = date_time.TIMEZONE_IST)
@@ -239,11 +254,6 @@ if __name__ == "__main__":
args.to_date = args.to_date.replace(hour = 23, minute = 59, second = 59, microsecond = 999999)
print("From Date:", args.from_date)
print(" To Date:", args.to_date)
# dates_list = [args.from_date]
# while args.from_date < args.to_date:
# args.from_date = args.from_date + datetime.timedelta(days = 1)
# dates_list.append(args.from_date)
# print(f"DATES ({len(dates_list)}):", dates_list)
# Create the clients:
tcaoff_client = AsyncTheCAOffice(
tcaoff/async_tcaoff.py
+8 -1
View File
@@ -186,8 +186,15 @@ class AsyncTheCAOffice:
# ┛
@staticmethod
def remove_special_chars(s: str) -> str:
def remove_special_chars(s: str | Any) -> str:
# If the input is not a string,
# we convert that to a string:
if not isinstance(s, str):
s = str(s)
# If the input is a string,
# we ensure we remove unsupported chars:
return regex.replace(
text = s,
pattern = r"[^\w\d\- _]",