(20260109) Almost done with Cosec-TCAOFF sync.
This commit is contained in:
@@ -0,0 +1,642 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
CREATED: Thu, 27th Nov, 2025
|
||||
UPDATED: Thu, 27th Nov, 2025
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To define a caching class that uses the persistent disk to hold cached data.
|
||||
This is a good way to cache data across multiple processes on the same machine (as long as they have access to
|
||||
the directory where the cached data is being stored).
|
||||
|
||||
WARNING:
|
||||
--------
|
||||
THIS IS A MODERATELY STATEFUL WAY OF IMPLEMENTING CACHING. THE USER'S REQUEST NEEDS TO HIT THE EXACT SAME
|
||||
MACHINE AGAIN FOR THE CACHING TO BE MEANINGFUL. IT CAN HIT ANY PROCESS ON THE SAME MACHINE, BUT THE PHYSICAL
|
||||
MACHINE WILL HAVE TO BE THE SAME.
|
||||
|
||||
The idea is simple:
|
||||
-------------------
|
||||
Everything that needs to be cached should have a key (identifier) and a value (the actual data). The key becomes
|
||||
the name of the file that holds the data on disk. The data itself is pickled using Python's pickle library. This
|
||||
way it retains its native datatypes.
|
||||
|
||||
TROUBLESHOOTING:
|
||||
----------------
|
||||
There's always a possibility that another user or process may accidentally mess with the caching directory and
|
||||
throw the whole system off. The simplest workaround is to delete the caching directory and rerun the program.
|
||||
On a fresh restart, the program will re-create the directory and things start from zero.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For system-level activities:
|
||||
import os
|
||||
|
||||
# For data-processing:
|
||||
import pickle
|
||||
|
||||
# For working with tabulated data:
|
||||
import pandas as pd
|
||||
|
||||
# Other utils:
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.string import regex
|
||||
from utils_v2.cache_v2.async_base import AsyncCachingBase
|
||||
from utils_v2.system import files
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import List, Any
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class AsyncDiskCache(AsyncCachingBase):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
caching_dir: str,
|
||||
lock_on_read: bool = False,
|
||||
lock_wait_timeout: int | float = 5.0,
|
||||
expiry_check_interval: int | float = 60.0,
|
||||
debug = True,
|
||||
debug_prefix = "ADiskCache | "
|
||||
):
|
||||
|
||||
"""
|
||||
Implements a simple cache in disk that holds and returns all native datatypes. All timestamps are normalized to
|
||||
UTC timezone.
|
||||
NOTE: It is NOT async, actually. It has been built on top of an async class so it has been declared as if it is.
|
||||
:param caching_dir: Directory where the cached data is stored.
|
||||
:param lock_on_read: Whether, or not, read operations lock the cached data. All modification operations (write,
|
||||
delete, update) will trigger a lock. Reading can be done without such a constraint.
|
||||
:param lock_wait_timeout: When cached data is being accessed, it may be locked. This is the default timeout
|
||||
for the file to get unlocked.
|
||||
:param expiry_check_interval: How frequently to check for expiry of all the cached data. When you request very
|
||||
specific data, its expiry will be checked before serving; but this interval defines a general cleanup to
|
||||
release memory.
|
||||
:param debug: Whether, or not, you want to show debugging messages from the start.
|
||||
:param debug_prefix: The prefix text to show with the debugging messages.
|
||||
"""
|
||||
|
||||
# Invoke the parent class's constructor:
|
||||
super().__init__(
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix
|
||||
)
|
||||
|
||||
# Note down the variables:
|
||||
self.__caching_dir = caching_dir
|
||||
self.__lock_on_read = lock_on_read
|
||||
self.__lock_wait_timeout = lock_wait_timeout
|
||||
self.__expiry_check_interval = expiry_check_interval
|
||||
self.__last_expiry_check_ts = date_time.get_current_utc_date_time(as_string = False).timestamp()
|
||||
|
||||
# Create a local dir that will hold the cached data:
|
||||
if not os.path.exists(self.__caching_dir):
|
||||
files.make_directory(self.__caching_dir)
|
||||
|
||||
# Create a lookup table and store that as a separate file:
|
||||
if not os.path.exists(os.path.join(self.__caching_dir, "lookup.pkl")):
|
||||
lookup_df = pd.DataFrame(columns = ["key", "exp"])
|
||||
self.to_disk(key = "lookup", value = lookup_df)
|
||||
|
||||
# If we yet don't have a lookup file:
|
||||
if not os.path.exists(os.path.join(self.__caching_dir, "lookup.pkl")):
|
||||
exception = RuntimeError("CRITICAL: Lookup file not found!")
|
||||
self._printer("LOOKUP FAIL!", exception)
|
||||
|
||||
# ┓┏ ┓
|
||||
# ┣┫┏┓┃┏┓┏┓┏┓┏
|
||||
# ┛┗┗ ┗┣┛┗ ┛ ┛
|
||||
# ┛
|
||||
|
||||
@property
|
||||
def now_utc(self) -> int | float:
|
||||
|
||||
"""
|
||||
Returns the current UTC time as a timestamp.
|
||||
:return: The current UTC time as a timestamp.
|
||||
"""
|
||||
|
||||
# Return the current time in UTC as a timestamp:
|
||||
return date_time.get_current_utc_date_time(as_string = False).timestamp()
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
Checks if a particular key exists in the cached data.
|
||||
:param key:
|
||||
:return:
|
||||
"""
|
||||
|
||||
async def __lock(
|
||||
self,
|
||||
key: str
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
To mark a key as locked so that no other process tries to access it at the same time. Read access may yet be
|
||||
granted depending on the value of 'lock_on_read'.
|
||||
:param key: The identifier of the cached data.
|
||||
:return: True if successful, False otherwise.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
# Try to write a lock file with the
|
||||
# UTC timestamp in it for reference of when it was created:
|
||||
files.write_file(
|
||||
file_path = os.path.join(self.__caching_dir, f"{key}.lock"),
|
||||
file_data = str(self.now_utc),
|
||||
mode = "w",
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# No exception means success:
|
||||
return True
|
||||
|
||||
except Exception as exception:
|
||||
self._printer("LOCK FAIL!", key, exception)
|
||||
self._printer(exception)
|
||||
return False
|
||||
|
||||
async def __unlock(
|
||||
self,
|
||||
key: str
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
To mark a key as unlocked so that other processes may start accessing it. Reading access may always be unlocked
|
||||
depending on the value of 'lock_on_read'.
|
||||
:param key: The identifier of the cached data.
|
||||
:return: True if successful, False otherwise.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
# Delete the file that indicates that a key is locked:
|
||||
files.delete_file(
|
||||
file_path = os.path.join(self.__caching_dir, f"{key}.lock"),
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# No exception means success:
|
||||
return True
|
||||
|
||||
except Exception as exception:
|
||||
self._printer("UNLOCK FAIL!", key, exception)
|
||||
return False
|
||||
|
||||
async def __wait_for_unlock(
|
||||
self,
|
||||
key: str,
|
||||
lock_wait_timeout: int | float = None
|
||||
) -> None:
|
||||
|
||||
"""
|
||||
THis waits for a particular key to be unlocked.
|
||||
:param key: The identifier of the cached data.
|
||||
:param lock_wait_timeout: The max amount to wait for a particular key to be unlocked. If null, the default value
|
||||
set in 'lock_wait_timeout' (from the constructor) will be used. You may override that value by passing a
|
||||
custom value here.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# Figure out when the time will run out:
|
||||
timeout = lock_wait_timeout or self.__lock_wait_timeout
|
||||
exp_utc = self.now_utc + timeout
|
||||
|
||||
# Wait for either the key to get unlocked,
|
||||
# or the time to run out:
|
||||
while os.path.exists(os.path.join(self.__caching_dir, f"{key}.lock")):
|
||||
if self.now_utc >= exp_utc:
|
||||
exception = TimeoutError(f"Key '{key}' was locked for more than {timeout:.2} seconds.")
|
||||
self._printer("LOCK WAIT TIMED OUT!", key, timeout, exception)
|
||||
raise exception
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
async def to_disk(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
lock_wait_timeout: int | float = None,
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
Stores the 'value' to the disk and keeps the 'key' as the filename.
|
||||
:param key: The identifier of the cached data. Becomes the name of the file when stored on disk.
|
||||
:param value: The actual cached data.
|
||||
:param lock_wait_timeout: How long (in seconds) to wait for the key to get unlocked.
|
||||
:return: True if cached, else False.
|
||||
"""
|
||||
|
||||
# We first wait for the key to get unlocked:
|
||||
await self.__wait_for_unlock(
|
||||
key = key,
|
||||
lock_wait_timeout = lock_wait_timeout
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
# First lock the key so that no other process can modify it:
|
||||
if not await self.__lock(key = key):
|
||||
raise RuntimeError("Failed to lock key '{key}'.")
|
||||
|
||||
# Try to store the data:
|
||||
files.write_file(
|
||||
file_path = os.path.join(self.__caching_dir, f"{key}.pkl"),
|
||||
file_data = pickle.dumps(value),
|
||||
mode = "wb",
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Unlock the key:
|
||||
if not await self.__unlock(key = key):
|
||||
raise RuntimeError("Failed to unlock key '{key}'.")
|
||||
|
||||
# Done here:
|
||||
return True
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
self._printer("CACHE SAVING FAIL!", exception)
|
||||
await self.__unlock(key = key)
|
||||
return False
|
||||
|
||||
async def from_disk(
|
||||
self,
|
||||
key: str
|
||||
) -> Any:
|
||||
|
||||
"""
|
||||
Reads cached data from the disk.
|
||||
:param key: The identifier of the cached data. It is the name of the file when stored on disk.
|
||||
:return: The read data.
|
||||
"""
|
||||
|
||||
# If configured that way,
|
||||
# we must lock the key before reading:
|
||||
if self.__lock_on_read:
|
||||
if not await self.__lock(key = key):
|
||||
raise RuntimeError("Failed to lock key '{key}'.")
|
||||
|
||||
try:
|
||||
|
||||
# Try to read the data:
|
||||
data = files.read_file(
|
||||
file_path = os.path.join(self.__caching_dir, f"{key}.pkl"),
|
||||
mode = "rb",
|
||||
raise_exception = True
|
||||
)
|
||||
data = pickle.loads(data)
|
||||
|
||||
# Unlock the key:
|
||||
if self.__lock_on_read:
|
||||
if not await self.__unlock(key = key):
|
||||
raise RuntimeError("Failed to unlock key '{key}'.")
|
||||
|
||||
# Done here:
|
||||
return data
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
self._printer("CACHE MISS!", exception)
|
||||
await self.__unlock(key = key)
|
||||
return False
|
||||
|
||||
async def clear_expired_keys(self) -> None:
|
||||
|
||||
"""
|
||||
Clears all the expired cached data.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# If the expiry check interval has not been crossed,
|
||||
# we need not go through the process:
|
||||
ref_utc = self.now_utc
|
||||
if ref_utc - self.__last_expiry_check_ts < self.__expiry_check_interval: return
|
||||
|
||||
# Otherwise, we note down the current timestamp and proceed:
|
||||
self.__last_expiry_check_ts = ref_utc
|
||||
|
||||
# Load the lookup and check what all needs to be deleted:
|
||||
ref_utc = self.now_utc
|
||||
lookup_df = await self.from_disk(key = "lookup")
|
||||
to_del = lookup_df[lookup_df["exp"] <= ref_utc]["key"].to_list()
|
||||
|
||||
|
||||
|
||||
# # Enlist all the keys that need to be deleted:
|
||||
# to_del = []
|
||||
# for k, v in self.__cached_data.items():
|
||||
# if ref_utc >= v["exp"]: to_del.append(k)
|
||||
#
|
||||
# # Delete the expired keys:
|
||||
# for k in to_del:
|
||||
# self.__cached_data.pop(k)
|
||||
|
||||
# ┏┓
|
||||
# ┃ ┏┓┏┓┏┓
|
||||
# ┗┛┗┛┛ ┗
|
||||
|
||||
async def list_keys(
|
||||
self,
|
||||
match: str | None = None,
|
||||
raise_exception: bool = False
|
||||
) -> List[str] | None:
|
||||
|
||||
"""
|
||||
To get a list of all the keys that match the pattern.
|
||||
:param match: To match a glob-style pattern. THIS IS NOT FULL-FLEDGED REGEX.
|
||||
:param raise_exception: Whether to raise or suppress exceptions.
|
||||
:return: The list of keys if successful, else None.
|
||||
"""
|
||||
|
||||
# Clear the expired keys:
|
||||
await self.clear_expired_keys()
|
||||
|
||||
# Start with an empty list:
|
||||
keys_list = []
|
||||
|
||||
try:
|
||||
|
||||
# Enlist all the keys,
|
||||
# test the pattern against all the keys and keep only those that match:
|
||||
ref_utc = self.now_utc
|
||||
for k, v in self.__cached_data.items():
|
||||
if v["exp"] > ref_utc:
|
||||
if match is None: keys_list.append(k)
|
||||
elif regex.match(text = str(k), pattern = match):
|
||||
keys_list.append(k)
|
||||
|
||||
# Done here:
|
||||
return list(set(keys_list)) if keys_list else None
|
||||
|
||||
# If an exception occurs in the process:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
if raise_exception: raise
|
||||
else: return None
|
||||
|
||||
async def ttl(
|
||||
self,
|
||||
key: str | bytes,
|
||||
raise_exception: bool = False
|
||||
) -> int | float | None:
|
||||
|
||||
"""
|
||||
To get the no. of seconds till the expiry of some key.
|
||||
:param key: The key to check the expiry of.
|
||||
:param raise_exception: Whether to raise or suppress exceptions.
|
||||
:return: -1 if the key is persistent (i.e., no expiry time set), -2 if the key does not exist, or the time left
|
||||
in seconds if the key exists and is not persistent. If something goes wrong, you will get a null value.
|
||||
"""
|
||||
|
||||
# Clear the expired keys:
|
||||
await self.clear_expired_keys()
|
||||
|
||||
# Start by assuming failure:
|
||||
ttl = None
|
||||
|
||||
try:
|
||||
|
||||
# Get the expiry:
|
||||
exp_utc = self.__cached_data.get(key, {}).get("exp", None)
|
||||
if exp_utc is None: ttl = -2
|
||||
else: ttl = exp_utc - self.now_utc()
|
||||
return ttl
|
||||
|
||||
# If an exception occurs in the process:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
if raise_exception: raise
|
||||
else: return None
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str | bytes,
|
||||
value: Any,
|
||||
expiry: float = None,
|
||||
raise_exception: bool = False
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
Saves some value to the cache. If an expiry is specified, the data will be deleted after that many seconds.
|
||||
:param key: The key with which the data will be stored and retrieved.
|
||||
:param value: The value to store.
|
||||
:param expiry: The time in seconds after which the data will expire. Must be a positive number.
|
||||
:param raise_exception: Whether to raise or suppress exceptions.
|
||||
:return: True if cached, else False.
|
||||
"""
|
||||
|
||||
# Clear the expired keys:
|
||||
await self.clear_expired_keys()
|
||||
|
||||
try:
|
||||
|
||||
# If the expiry is not specified,
|
||||
# make it an unreasonably far off future date:
|
||||
if expiry is None: expiry = self.now_utc + 3_15_36_000
|
||||
|
||||
# Here we actually try to set the data:
|
||||
self.__cached_data[key] = {
|
||||
"val": value,
|
||||
"exp": self.now_utc + expiry
|
||||
}
|
||||
|
||||
# Done here:
|
||||
return True
|
||||
|
||||
# If an exception occurs in the process:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
if raise_exception: raise
|
||||
else: return False
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str | bytes,
|
||||
raise_exception: bool = False,
|
||||
on_fail: Any = None
|
||||
) -> Any:
|
||||
|
||||
"""
|
||||
Retrieve the cached value.
|
||||
:param key: The key with which the data was saved.
|
||||
:param raise_exception: Whether to raise or suppress exceptions.
|
||||
:param on_fail: What to return if the process fails due to an exception.
|
||||
:return: The retrieved data or null if not found.
|
||||
"""
|
||||
|
||||
# Clear the expired keys:
|
||||
await self.clear_expired_keys()
|
||||
|
||||
try:
|
||||
|
||||
# Here we try to fetch the data:
|
||||
data = None
|
||||
raw = self.__cached_data.get(key)
|
||||
|
||||
# Check for expiry if it was a cache hit:
|
||||
if raw: data = raw["val"] if raw["exp"] > self.now_utc else None
|
||||
|
||||
# Done here:
|
||||
return data
|
||||
|
||||
# If an exception occurs in the process:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
if raise_exception: raise
|
||||
else: return on_fail
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str | bytes,
|
||||
raise_exception: bool = False
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
Prematurely delete the value from the cache before it expires.
|
||||
:param key: The key with which the data was saved.
|
||||
:param raise_exception: Whether to raise or suppress exceptions.
|
||||
:return: True if deleted, else False.
|
||||
"""
|
||||
|
||||
# Clear the expired keys:
|
||||
await self.clear_expired_keys()
|
||||
|
||||
try:
|
||||
|
||||
# Try to manually delete the key before expiry:
|
||||
response = self.__cached_data.pop(key, None)
|
||||
return True if response else False
|
||||
|
||||
# If an exception occurs in the process:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
if raise_exception: raise
|
||||
else: return False
|
||||
|
||||
async def count(
|
||||
self,
|
||||
key: str | bytes,
|
||||
value: int = 1,
|
||||
expiry: float = None,
|
||||
raise_exception: bool = False
|
||||
) -> int | None:
|
||||
|
||||
"""
|
||||
To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the
|
||||
counter will be created and the value will be incremented from 0.
|
||||
:param key: The name of the counter.
|
||||
:param value: The amount to increment the value by. Send negative values to count backwards.
|
||||
:param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created.
|
||||
This value has to be an integer. If a float is passed, the value will be rounded off.
|
||||
:param raise_exception: Whether to raise or suppress exceptions.
|
||||
:return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed.
|
||||
"""
|
||||
|
||||
# Clear the expired keys:
|
||||
await self.clear_expired_keys()
|
||||
|
||||
try:
|
||||
|
||||
# Check if
|
||||
already_existed = False if self.__cached_data.get(key, None) is None else True
|
||||
|
||||
# If it already exists, we just increment the value;
|
||||
# else we create the value and increment the value starting from zero:
|
||||
if already_existed: self.__cached_data[key]["val"] = self.__cached_data[key]["val"] + value
|
||||
else: self.__cached_data[key] = {
|
||||
"val": value,
|
||||
"exp": self.now_utc + expiry
|
||||
}
|
||||
|
||||
# Return the new value:
|
||||
return self.__cached_data[key]["val"]
|
||||
|
||||
# If an exception occurs in the process:
|
||||
except Exception as exception:
|
||||
self._printer(exception)
|
||||
if raise_exception: raise
|
||||
else: return None
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
Reference in New Issue
Block a user