khushalps/cosec_automation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20260317) Tried headless and parallel mode.

Browse Source
This commit is contained in:
Khushal P Soonderji
2026-03-17 18:09:04 +05:30
parent e25daf5fd5
commit 3a68412b1b
12 changed files with 15781 additions and 14463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/common.py
+77 -26
View File
@@ -22,8 +22,7 @@
N/A
"""
import pathlib
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
@@ -60,12 +59,9 @@ from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with datatypes:
from typing import List, Dict, Any, Union
from typing import List, Dict, Any, Union, Callable
from collections import defaultdict
# To run a cron-like scheduler:
from apscheduler.schedulers.asyncio import AsyncIOScheduler
# For async activities:
import asyncio
@@ -85,17 +81,18 @@ FILE_DIR = files.get_file_directory(include_filename = False)
PROJ_DIR = files.get_parent_directory(FILE_DIR, depth = 1)
CACHE_DIR = os.path.join(PROJ_DIR, "local", "cache")
CREDS_DIR = os.path.join(PROJ_DIR, "creds")
# ---
COSEC_CREDS_FILE = os.path.join(CREDS_DIR, "cosec.json")
TCAOFF_CREDS_FILE = os.path.join(CREDS_DIR, "tcaoff.json")
MUSTER_ROLL_CACHE_FILE = os.path.join(CACHE_DIR, "muster_roll_cache.json")
IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "in_out_summary_cache.json")
PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "prev_day_in_out_summary_cache.json")
MANUAL_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "manual_in_out_summary_cache.json")
WORK_REPORTS_CACHE_FILE = os.path.join(CACHE_DIR, "work_reports_cache.json")
# # ---
# COSEC_CREDS_FILE = os.path.join(CREDS_DIR, "cosec.json")
# TCAOFF_CREDS_FILE = os.path.join(CREDS_DIR, "tcaoff.json")
# MUSTER_ROLL_CACHE_FILE = os.path.join(CACHE_DIR, "muster_roll_cache.json")
# IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "in_out_summary_cache.json")
# PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "prev_day_in_out_summary_cache.json")
# MANUAL_SUMMARY_CACHE_FILE = os.path.join(CACHE_DIR, "manual_in_out_summary_cache.json")
# WORK_REPORTS_CACHE_FILE = os.path.join(CACHE_DIR, "work_reports_cache.json")
# # ---
CHROME_DRIVER_DIR = os.path.join(PROJ_DIR, "drivers", "chrome")
USER_DATA_DIR = os.path.join(PROJ_DIR, "browser", "user_data")
DOWNLOADS_DIR = os.path.join(PROJ_DIR, "downloads")
# USER_DATA_DIR = os.path.join(PROJ_DIR, "browser", "user_data", os.environ.get("ORG", "default"))
# DOWNLOADS_DIR = os.path.join(PROJ_DIR, "downloads", os.environ.get("ORG", "default"))
# ---
TEST_MODE_MUSTER_ROLL_FILE_PATH = os.path.join(PROJ_DIR, "cosec_web", "sample_files", "muster_roll.xls")
TEST_MODE_IN_OUT_SUMMARY_FILE_PATH = os.path.join(PROJ_DIR, "cosec_web", "sample_files", "in_out_summary.xls")
@@ -137,6 +134,41 @@ err_printer = IceCreamDebugger(prefix = "[ERR] Common | ", includeContext = True
# *****************************************************************************************************************
def get_muster_roll_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "muster_roll_cache.json")
def get_in_out_summary_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "in_out_summary_cache.json")
def get_prev_day_in_out_summary_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "prev_day_in_out_summary_cache.json")
def get_manual_in_out_summary_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "manual_in_out_summary_cache.json")
def get_work_reports_cache_file_path(): return os.path.join(CACHE_DIR, os.environ.get("ORG", "default"), "work_reports_cache.json")
# ---
def get_browser_user_data_directory(): return os.path.join(PROJ_DIR, "browser", "user_data", os.environ.get("ORG", "default"))
def get_browser_downloads_directory(): return os.path.join(PROJ_DIR, "downloads", os.environ.get("ORG", "default"))
# ---------------------------------------------------------------------------------------------------------------------
def init_paths():
"""
We will be segregating the data of the various organizations by their sub-dirs. This quick function ensures that
those segregated paths exist. Call it at the start of your script when you have set the organization name to the
temporary environment variable.
"""
for path in [
os.path.join(CACHE_DIR, os.environ.get("ORG", "default")),
os.path.join(PROJ_DIR, "browser", "user_data", os.environ.get("ORG", "default")),
os.path.join(PROJ_DIR, "downloads", os.environ.get("ORG", "default"))
]:
if not os.path.exists(path):
printer("Making", path)
files.make_directory(path)
# ---------------------------------------------------------------------------------------------------------------------
def kill_chrome() -> None:
"""
@@ -148,6 +180,7 @@ def kill_chrome() -> None:
# then wait if old processes were killed:
kill_count = CosecWeb.kill_chrome_processes()
if kill_count > 0: time.sleep(2.5)
printer(kill_count)
# ---------------------------------------------------------------------------------------------------------------------
@@ -156,7 +189,7 @@ def kill_chrome() -> None:
def get_muster_roll(
cosec_creds: dict,
on_date: datetime.datetime,
cache_file: str = MUSTER_ROLL_CACHE_FILE,
cache_file: str | Callable | None = None,
test_mode: bool = False
) -> bool:
@@ -169,6 +202,12 @@ def get_muster_roll(
:return: True if the automated fetch was successful, else False.
"""
# Figure out path(s):
if not isinstance(cache_file, str):
if isinstance(cache_file, Callable): cache_file = cache_file()
else: cache_file = get_muster_roll_cache_file_path()
printer(cache_file)
# Start by assuming failure:
success = False
report_path = None
@@ -189,8 +228,9 @@ def get_muster_roll(
username = cosec_creds["creds"]["username"],
password = cosec_creds["creds"]["password"],
driver_dir = CHROME_DRIVER_DIR,
user_data_dir = USER_DATA_DIR,
downloads_dir = DOWNLOADS_DIR,
user_data_dir = get_browser_user_data_directory(),
downloads_dir = get_browser_downloads_directory(),
headless = False
)
# Perform the login:
@@ -314,7 +354,7 @@ async def sync_departments_to_tcaoff(
# Loop through the data from Cosec and add missing departments to TCAOFF:
for cosec_dept in cosec_depts:
if cosec_dept.lower() not in tcaoff_depts:
success = tcaoff_client.department_add(department_name = cosec_dept)
success = await tcaoff_client.department_add(department_name = cosec_dept)
response["total"] += 1
if success: response["success"] += 1
else: response["fail"] += 1
@@ -434,7 +474,7 @@ def get_in_out_summary(
cosec_creds: dict,
from_dt: date_time.datetime = None,
to_dt: date_time.datetime = None,
cache_file: str = IN_OUT_SUMMARY_CACHE_FILE,
cache_file: str | Callable | None = None,
test_mode: bool = False
) -> bool:
@@ -448,6 +488,12 @@ def get_in_out_summary(
:return: True if the automated fetch was successful, else False.
"""
# Figure out path(s):
if not isinstance(cache_file, str):
if isinstance(cache_file, Callable): cache_file = cache_file()
else: cache_file = get_in_out_summary_cache_file_path()
printer(cache_file)
# Start by assuming failure:
success = False
report_path = None
@@ -468,8 +514,9 @@ def get_in_out_summary(
username = cosec_creds["creds"]["username"],
password = cosec_creds["creds"]["password"],
driver_dir = CHROME_DRIVER_DIR,
user_data_dir = USER_DATA_DIR,
downloads_dir = DOWNLOADS_DIR,
user_data_dir = get_browser_user_data_directory(),
downloads_dir = get_browser_downloads_directory(),
headless = False
)
# Perform the login:
@@ -522,6 +569,7 @@ def get_in_out_summary(
}
# Save the data to a JSON file:
printer("Saving", cache_file)
json.to_file(
file = cache_file,
python_data = report_data,
@@ -709,6 +757,9 @@ async def sync_attendance_to_tcaoff(
:return: The dict that gives you the count of the successful and failed attendance marking API calls.
"""
# Figure out path(s):
work_reports_cache_file = get_work_reports_cache_file_path()
# Results:
results = defaultdict(int)
@@ -753,8 +804,8 @@ async def sync_attendance_to_tcaoff(
# Now we save the work reports for next time,
# and then we check for the ones that have changed:
work_reports = []
if os.path.exists(WORK_REPORTS_CACHE_FILE):
cached_work_reports = json.from_file(WORK_REPORTS_CACHE_FILE)
if os.path.exists(work_reports_cache_file):
cached_work_reports = json.from_file(work_reports_cache_file)
hashed_new_work_reports = {
wr["user_id"] + "." + wr["work_date"] : wr
for wr in new_work_reports
@@ -767,7 +818,7 @@ async def sync_attendance_to_tcaoff(
if v == hashed_old_work_reports.get(k, {}): continue
work_reports.append(v)
else: work_reports = new_work_reports
json.to_file(WORK_REPORTS_CACHE_FILE, new_work_reports)
json.to_file(work_reports_cache_file, new_work_reports)
# Create the tasks to fire:
tasks = []
scripts/cron.py
+40 -22
View File
@@ -112,7 +112,7 @@ err_printer = IceCreamDebugger(prefix = "[ERR] Cron | ", includeContext = True)
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# *****************************************************************************************************************-
async def yesterday_cron(
@@ -139,7 +139,7 @@ async def yesterday_cron(
cosec_creds = cosec_creds,
from_dt = date_time.get_current_ist_date_time() - datetime.timedelta(days = 3),
to_dt = date_time.get_current_ist_date_time(),
cache_file = common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE,
cache_file = common.get_prev_day_in_out_summary_cache_file_path,
test_mode = test_mode
)
@@ -148,7 +148,7 @@ async def yesterday_cron(
printer("IN/OUT SUMMARY: Sync'ing with TCAOFF")
await common.sync_attendance_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_in_out_summary = json.from_file(common.PREV_DAY_IN_OUT_SUMMARY_CACHE_FILE),
cosec_in_out_summary = json.from_file(common.get_prev_day_in_out_summary_cache_file_path()),
chunk_size = 10
)
@@ -198,7 +198,7 @@ async def today_cron(
try:
# MUSTER-ROLL:
# -----------
# Get the Muster Roll and then wait
# for the driver's resources to get freed:
success = common.get_muster_roll(
@@ -208,14 +208,14 @@ async def today_cron(
minute = 0,
second = 0
),
cache_file = common.MUSTER_ROLL_CACHE_FILE,
cache_file = common.get_muster_roll_cache_file_path,
test_mode = test_mode
)
# Sync data between Cosec and TCAOFF:
if success:
print("MUSTER ROLL: Sync'ing with TCAOFF")
cosec_muster_roll = json.from_file(common.MUSTER_ROLL_CACHE_FILE)
cosec_muster_roll = json.from_file(common.get_muster_roll_cache_file_path())
await common.sync_branches_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_muster_roll = cosec_muster_roll,
@@ -230,14 +230,14 @@ async def today_cron(
)
# IN-OUT SUMMARY:
# --------------
# Get the previous day's In/Out Summary and then wait
# for the driver's resources to get freed:
success = common.get_in_out_summary(
cosec_creds = cosec_creds,
from_dt = date_time.get_current_ist_date_time().replace(hour = 0, minute = 0, second = 0, microsecond = 0),
to_dt = date_time.get_current_ist_date_time(),
cache_file = common.IN_OUT_SUMMARY_CACHE_FILE,
cache_file = common.get_in_out_summary_cache_file_path,
test_mode = test_mode
)
@@ -246,7 +246,7 @@ async def today_cron(
printer("IN/OUT SUMMARY: Sync'ing with TCAOFF")
await common.sync_attendance_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_in_out_summary = json.from_file(common.IN_OUT_SUMMARY_CACHE_FILE),
cosec_in_out_summary = json.from_file(common.get_in_out_summary_cache_file_path()),
chunk_size = 10
)
@@ -354,18 +354,20 @@ if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument(
"--test", "--test-mode",
help = "Picks the last cache file instead of fetching records from COSEC.",
action = "store_true",
default = False,
)
ap.add_argument(
"--immediate",
help = "To do one run immediately first, then set the scheduler.",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--verbose",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--cosec-creds",
@@ -377,16 +379,30 @@ if __name__ == "__main__":
help = "The credentials JSON from which you would like to connect to TheCAOffice.",
default = "tcaoff.json"
)
ap.add_argument(
"--org", "--organization",
help = "When handling multiple organizations, this word will be used to keep their files separate.",
default = "default"
)
args = ap.parse_args()
# Hold temporary environment variables:
os.environ["ORG"] = args.org
# Read required credentials:
cosec_creds_path = os.path.join(common.CREDS_DIR, args.cosec_creds)
tcaoff_creds_path = os.path.join(common.CREDS_DIR, args.tcaoff_creds)
# ---
cosec_creds = json.from_file(cosec_creds_path)
tcaoff_creds = json.from_file(tcaoff_creds_path)
# Explicitly mention the expected file paths for other devs to maintain:
print("PROJ. DIR. :", common.PROJ_DIR)
print("COSEC CREDS :", common.COSEC_CREDS_FILE)
print("TCAOFF CREDS:", common.TCAOFF_CREDS_FILE)
print("COSEC CREDS :", cosec_creds_path)
print("TCAOFF CREDS:", tcaoff_creds_path)
# Read required credentials:
cosec_creds = json.from_file(os.path.join(common.CREDS_DIR, args.cosec_creds))
tcaoff_creds = json.from_file(os.path.join(common.CREDS_DIR, args.tcaoff_creds))
# Ensure that certain required paths exist:
common.init_paths()
# Create the clients:
tcaoff_client = AsyncTheCAOffice(
@@ -397,9 +413,11 @@ if __name__ == "__main__":
)
# Schedule the activities:
asyncio.run(set_scheduler(
cosec_creds = cosec_creds,
tcaoff_client = tcaoff_client,
immediate = args.immediate,
test_mode = args.test
))
asyncio.run(
set_scheduler(
cosec_creds = cosec_creds,
tcaoff_client = tcaoff_client,
immediate = args.immediate,
test_mode = args.test
)
)
scripts/manual_attendance.py
+24 -14
View File
@@ -149,16 +149,17 @@ async def manual_attendance(
cosec_creds = cosec_creds,
from_dt = from_date,
to_dt = to_date,
cache_file = common.MANUAL_SUMMARY_CACHE_FILE,
cache_file = common.get_manual_in_out_summary_cache_file_path,
test_mode = test_mode
)
# Sync data between Cosec and TCAOFF:
if success:
printer("IN/OUT SUMMARY: Sync'ing with TCAOFF")
print("SYNC FROM:", common.get_manual_in_out_summary_cache_file_path())
await common.sync_attendance_to_tcaoff(
tcaoff_client = tcaoff_client,
cosec_in_out_summary = json.from_file(common.MANUAL_SUMMARY_CACHE_FILE),
cosec_in_out_summary = json.from_file(common.get_manual_in_out_summary_cache_file_path()),
chunk_size = 10
)
@@ -193,12 +194,12 @@ if __name__ == "__main__":
ap.add_argument(
"--test", "--test-mode",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--verbose",
action = "store_true",
default = False,
default = False
)
ap.add_argument(
"--from-date",
@@ -220,16 +221,30 @@ if __name__ == "__main__":
help = "The credentials JSON from which you would like to connect to TheCAOffice.",
default = "tcaoff.json"
)
ap.add_argument(
"--org", "--organization",
help = "When handling multiple organizations, this word will be used to keep their files separate.",
default = "default"
)
args = ap.parse_args()
# Hold temporary environment variables:
os.environ["ORG"] = args.org
# Read required credentials:
cosec_creds_path = os.path.join(common.CREDS_DIR, args.cosec_creds)
tcaoff_creds_path = os.path.join(common.CREDS_DIR, args.tcaoff_creds)
# ---
cosec_creds = json.from_file(cosec_creds_path)
tcaoff_creds = json.from_file(tcaoff_creds_path)
# Explicitly mention the expected file paths for other devs to maintain:
print("PROJ. DIR. :", common.PROJ_DIR)
print("COSEC CREDS :", common.COSEC_CREDS_FILE)
print("TCAOFF CREDS:", common.TCAOFF_CREDS_FILE)
print("COSEC CREDS :", cosec_creds_path)
print("TCAOFF CREDS:", tcaoff_creds_path)
# Read required credentials:
cosec_creds = json.from_file(os.path.join(common.CREDS_DIR, args.cosec_creds))
tcaoff_creds = json.from_file(os.path.join(common.CREDS_DIR, args.tcaoff_creds))
# Ensure that certain required paths exist:
common.init_paths()
# Date-handling:
args.from_date = date_time.parse_date_time(args.from_date, timezone = date_time.TIMEZONE_IST)
@@ -239,11 +254,6 @@ if __name__ == "__main__":
args.to_date = args.to_date.replace(hour = 23, minute = 59, second = 59, microsecond = 999999)
print("From Date:", args.from_date)
print(" To Date:", args.to_date)
# dates_list = [args.from_date]
# while args.from_date < args.to_date:
# args.from_date = args.from_date + datetime.timedelta(days = 1)
# dates_list.append(args.from_date)
# print(f"DATES ({len(dates_list)}):", dates_list)
# Create the clients:
tcaoff_client = AsyncTheCAOffice(