diff --git a/utils_v2/ai/image_classification/async_blur.py b/utils_v2/ai/image_classification/async_blur.py index c921a4e..f69af42 100644 --- a/utils_v2/ai/image_classification/async_blur.py +++ b/utils_v2/ai/image_classification/async_blur.py @@ -11,7 +11,7 @@ OBJECTIVE: To provide an easy way to assess images for blurriness. - Tried and implemented using HuggingFace models. + Tried and implemented using HuggingFace behaviour_models. REFERENCES: @@ -193,7 +193,7 @@ if __name__ == "__main__": async def main(): image = r"/home/developer/Downloads/low-res-check.png" - assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/image_classification/BlurOrBokeh") + assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh") usable = await assessor.is_ok(image) classes = await assessor.classify(image) print("IS OKAY:", usable) diff --git a/utils_v2/ai/image_classification/async_nsfw.py b/utils_v2/ai/image_classification/async_nsfw.py index c59189b..446c34a 100644 --- a/utils_v2/ai/image_classification/async_nsfw.py +++ b/utils_v2/ai/image_classification/async_nsfw.py @@ -11,7 +11,7 @@ OBJECTIVE: To provide an easy way to assess images for adult content. - Tried and implemented using HuggingFace models. + Tried and implemented using HuggingFace behaviour_models. REFERENCES: diff --git a/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py b/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py index 04ec460..fb6d6c7 100644 --- a/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py +++ b/utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py @@ -13,12 +13,12 @@ To provide an easy way to get masks from dichotomous image segmentation. This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait". - You may experiment with other models too, but make sure that model is made for "dichotomous" behaviour. This + You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This means that the model should have only two classes like "foreground", and "background". The specified model was trained for implementing portrait mode style blurring of backgrounds. The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not - support drop-in replacement for other models, please be aware about this. + support drop-in replacement for other behaviour_models, please be aware about this. REFERENCES: diff --git a/utils_v2/ai/object_detection/yolo.py b/utils_v2/ai/object_detection/yolo.py index 91747f2..a435ca6 100644 --- a/utils_v2/ai/object_detection/yolo.py +++ b/utils_v2/ai/object_detection/yolo.py @@ -10,7 +10,7 @@ OBJECTIVE: - To provide a class to detect objects in images using YOLO models. + To provide a class to detect objects in images using YOLO behaviour_models. REFERENCES: @@ -166,7 +166,7 @@ class YoloDetect: if __name__ == "__main__": - # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "models", "yolov8x.pt") + # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt") model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt" # sample_image_path = r"/home/developer/Downloads/2_cats.jpg" sample_image_path = r"/home/developer/Downloads/flattened_image.jpg" diff --git a/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc b/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc new file mode 100644 index 0000000..724908a Binary files /dev/null and b/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc differ diff --git a/utils_v2/api/async_quart.py b/utils_v2/api/async_quart.py index 4e29493..ffcd260 100644 --- a/utils_v2/api/async_quart.py +++ b/utils_v2/api/async_quart.py @@ -55,8 +55,12 @@ import datetime # System-level activities: import io +<<<<<<< HEAD +======= +import os +>>>>>>> 0450b2a445509ef536ccf8598ab31a0e4ba09f18 -# For Pydantic data-models: +# For Pydantic data-behaviour_models: import pydantic # For hashing and shortening the hash: @@ -700,7 +704,9 @@ def should_not_be_under_maintenance(attr_name): """ Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create - a boolean variable within the scope of the 'current_app' for this to work. + a boolean variable within the scope of the 'current_app' for this to work. An alternate to this is to set the value + in an environment variable named 'IS_UNDER_MAINTENANCE' to a string value of either 'True' or 'False' for + multi-worker deployments. :param attr_name: The name of the boolean variable that will hold the information about the app being under maintenance. If its value is True at the time of checking, the incoming request will be rejected. :return: The decorator factory. @@ -717,7 +723,9 @@ def should_not_be_under_maintenance(attr_name): # Get the attribute and check if it indicates that the app is under maintenance, # call the wrapped function if not under maintenance: app_attr = getattr(current_app, attr_name) - if app_attr: response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() + env_attr = True if os.environ.get("IS_UNDER_MAINTENANCE", "False").lower() == "true" else False + if app_attr or env_attr: + response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() else: response = await func(*args, **kwargs) kwargs["decorator_count"] -= 1 diff --git a/utils_v2/cache/__pycache__/__init__.cpython-310.pyc b/utils_v2/cache/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..06f9307 Binary files /dev/null and b/utils_v2/cache/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc b/utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc new file mode 100644 index 0000000..0a16447 Binary files /dev/null and b/utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc differ diff --git a/utils_v2/cache/async_redis_cache.py b/utils_v2/cache/async_redis_cache.py index 14df725..a0ab8c0 100644 --- a/utils_v2/cache/async_redis_cache.py +++ b/utils_v2/cache/async_redis_cache.py @@ -123,6 +123,52 @@ def cache_it(cache = None, expiry = 120): return decorator +# --------------------------------------------------------------------------------------------------------------------- + + +def cache_class_methods(attr_name, expiry = 120): + + """ + This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. + If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the + cache instead of going through the whole function again. + :param attr_name: The name of the variable that has an instance of "AsyncRedisCache". + :param expiry: The time in seconds after which the cached data must be cleared. + :return: The decorator that automatically caches your data. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(self, *args, **kwargs): + + # Get the cache object first: + cache_obj = getattr(self, attr_name) + + # We first use the name of the function and the inputs given to it to generate a key for Redis with the + # simple hashing and shortening by way of base64 strings: + inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) + sha256_hash = hashlib.sha256() + sha256_hash.update(inputs_given.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Now we check if we have the value in cache: + try: response = await cache_obj.get(base64_key, raise_exception = True) + + # If the key doesn't exist, we pass through the function and store the results. + except: + response = await func(self, *args, **kwargs) + await cache_obj.set(key = base64_key, value = response, expiry = expiry) + + # Return the response from the wrapped function. + return response + + return wrapper + + return decorator + + # ***************************************************************************************************************** # ***** **** # *** CLASSES *** diff --git a/utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc b/utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc new file mode 100644 index 0000000..b97ae9b Binary files /dev/null and b/utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc differ diff --git a/utils_v2/database/async_mysql_v2.py b/utils_v2/database/async_mysql_v2.py index 911bfaf..fc977c6 100644 --- a/utils_v2/database/async_mysql_v2.py +++ b/utils_v2/database/async_mysql_v2.py @@ -96,7 +96,8 @@ class AsyncMySQL: """ A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as - JSON-like structures (list or dict). + JSON-like structures (list or dict). The format for the results was very specific to our use case for serving + Bicree's requirement. This may not serve your requirement at all. :param pool_size: The number of connections to maintain n a pool. :param args: Any arguments to pass. Not used. :param kwargs: Pass the connection configuration from here. @@ -207,8 +208,6 @@ class AsyncMySQL: await asyncio.sleep(backoff_seconds) backoff_seconds = backoff_seconds * backoff_multiplier - print("LEN:", len(results)) - # If the results are blank: if len(results) == 0: return { "status": results[0][0]["status"], @@ -238,9 +237,6 @@ class AsyncMySQL: formatted_results["seconds"] = time.perf_counter() - start_ts # Done here: - print(f"{procedure_name}:") - print(json.to_string(formatted_results)) - print("\n") if return_exception: return formatted_results, exception else: return formatted_results @@ -274,12 +270,20 @@ if __name__ == "__main__": :return: None. """ + # cred_json = { + # "host": "del.ditscentre.in", + # "user": "bicree", + # "port": 3306, + # "password": "9c3b2808a4aa281129d399fe09e69b53", + # "database": "bicree" + # } + cred_json = { "host": "del.ditscentre.in", - "user": "bicree", + "user": "caOffice", "port": 3306, - "password": "9c3b2808a4aa281129d399fe09e69b53", - "database": "bicree" + "password": "jstArchon", + "database": "caOffice" } db_conn = AsyncMySQL( @@ -299,10 +303,16 @@ if __name__ == "__main__": # ) # ) + # result = await db_conn.call_procedure_and_get_json( + # procedure_name = "listSummary", + # procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",) + # ) + result = await db_conn.call_procedure_and_get_json( - procedure_name = "listSummary", - procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",) + procedure_name = "campaign_activity_report", + procedure_args = (10000000,) ) + print("RESULT:", json.to_string(result, default = str)) start_time = time.time() diff --git a/utils_v2/image/scanner/scanner.py b/utils_v2/image/scanner/scanner.py index 424ddd1..28fe69b 100644 --- a/utils_v2/image/scanner/scanner.py +++ b/utils_v2/image/scanner/scanner.py @@ -387,7 +387,7 @@ if __name__ == "__main__": import time my_scanner = DocumentScanner( - layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], ocr_languages = ["en"] diff --git a/utils_v2/image/scanner/scanner_v2.py b/utils_v2/image/scanner/scanner_v2.py index 2a5b9fc..fc63627 100644 --- a/utils_v2/image/scanner/scanner_v2.py +++ b/utils_v2/image/scanner/scanner_v2.py @@ -353,7 +353,7 @@ if __name__ == "__main__": import time my_scanner = DocumentScanner( - layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], ocr_languages = ["en"]