diff --git a/ai/image_classification/async_blur.py b/ai/image_classification/async_blur.py index c921a4e..f69af42 100644 --- a/ai/image_classification/async_blur.py +++ b/ai/image_classification/async_blur.py @@ -11,7 +11,7 @@ OBJECTIVE: To provide an easy way to assess images for blurriness. - Tried and implemented using HuggingFace models. + Tried and implemented using HuggingFace behaviour_models. REFERENCES: @@ -193,7 +193,7 @@ if __name__ == "__main__": async def main(): image = r"/home/developer/Downloads/low-res-check.png" - assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/image_classification/BlurOrBokeh") + assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh") usable = await assessor.is_ok(image) classes = await assessor.classify(image) print("IS OKAY:", usable) diff --git a/ai/image_classification/async_nsfw.py b/ai/image_classification/async_nsfw.py index c59189b..446c34a 100644 --- a/ai/image_classification/async_nsfw.py +++ b/ai/image_classification/async_nsfw.py @@ -11,7 +11,7 @@ OBJECTIVE: To provide an easy way to assess images for adult content. - Tried and implemented using HuggingFace models. + Tried and implemented using HuggingFace behaviour_models. REFERENCES: diff --git a/ai/image_segmentation/async_dichotomous_segmentation.py b/ai/image_segmentation/async_dichotomous_segmentation.py index 04ec460..fb6d6c7 100644 --- a/ai/image_segmentation/async_dichotomous_segmentation.py +++ b/ai/image_segmentation/async_dichotomous_segmentation.py @@ -13,12 +13,12 @@ To provide an easy way to get masks from dichotomous image segmentation. This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait". - You may experiment with other models too, but make sure that model is made for "dichotomous" behaviour. This + You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This means that the model should have only two classes like "foreground", and "background". The specified model was trained for implementing portrait mode style blurring of backgrounds. The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not - support drop-in replacement for other models, please be aware about this. + support drop-in replacement for other behaviour_models, please be aware about this. REFERENCES: diff --git a/ai/object_detection/yolo.py b/ai/object_detection/yolo.py index 91747f2..a435ca6 100644 --- a/ai/object_detection/yolo.py +++ b/ai/object_detection/yolo.py @@ -10,7 +10,7 @@ OBJECTIVE: - To provide a class to detect objects in images using YOLO models. + To provide a class to detect objects in images using YOLO behaviour_models. REFERENCES: @@ -166,7 +166,7 @@ class YoloDetect: if __name__ == "__main__": - # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "models", "yolov8x.pt") + # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt") model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt" # sample_image_path = r"/home/developer/Downloads/2_cats.jpg" sample_image_path = r"/home/developer/Downloads/flattened_image.jpg" diff --git a/api/__pycache__/async_quart.cpython-310.pyc b/api/__pycache__/async_quart.cpython-310.pyc index e948687..66e8ec4 100644 Binary files a/api/__pycache__/async_quart.cpython-310.pyc and b/api/__pycache__/async_quart.cpython-310.pyc differ diff --git a/api/__pycache__/metrics_prometheus.cpython-310.pyc b/api/__pycache__/metrics_prometheus.cpython-310.pyc index 2970917..724908a 100644 Binary files a/api/__pycache__/metrics_prometheus.cpython-310.pyc and b/api/__pycache__/metrics_prometheus.cpython-310.pyc differ diff --git a/api/async_quart.py b/api/async_quart.py index 0c29f69..8749a76 100644 --- a/api/async_quart.py +++ b/api/async_quart.py @@ -46,13 +46,8 @@ from utils_v2.string import json from utils_v2.date_time import date_time from utils_v2.security import sanitizers from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.log import APILogModel from utils_v2.api.response import ResponseModel -from utils_v2.api.metrics_prometheus import ( - TOTAL_REQUEST_COUNT, - LIVE_REQUEST_COUNT, - REQUEST_LATENCY, - MetricsAPI -) # To work with date and time: import time @@ -60,11 +55,9 @@ import datetime # System-level activities: import io -import distro -import socket -import platform +import os -# For Pydantic data-models: +# For Pydantic data-behaviour_models: import pydantic # For hashing and shortening the hash: @@ -90,11 +83,6 @@ import asyncio # ***************************************************************************************************************** -# Info for logging that will stay constant during runtime: -SERVER_HOSTNAME = socket.gethostname() -PLATFORM_INFO = platform.uname() -HOST_OS = distro.name(True) - # Chars to choose from for random strings: ALPHANUMERIC_CHARS = string.ascii_letters + string.digits @@ -359,7 +347,7 @@ def set_api_version(api_version): # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -417,7 +405,7 @@ def read_input( # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -556,7 +544,7 @@ def validate_input( # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -572,6 +560,8 @@ def validate_input( def log_request_to_mongo( attr_name, collection: str = "logs", + api_version: str = None, + project: str = None, log_type: str = None, operation: str = None, log_input: bool = True, @@ -586,6 +576,8 @@ def log_request_to_mongo( :param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the scope of 'current_app'. :param collection: The name of the collection to write the log into. + :param api_version: The version code of the API endpoint that is being logged. + :param project: The name of the project that the endpoint was built for. :param log_type: A hint to identify what the log was for. :param operation: A hint to identify what was action was being performed. :param log_input: Whether, or not, you would like to log the input that came in. @@ -602,6 +594,9 @@ def log_request_to_mongo( # Let the next in-line decorator know that it has been wrapped: kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + # Set the api version as needed: + kwargs["api_version"] = kwargs.get("api_version", api_version) + # Make variables and extract available info.: exception = None response = None @@ -635,7 +630,8 @@ def log_request_to_mongo( elif isinstance(response, tuple): response_to_log, http_code_to_log = response else: response_to_log, http_code_to_log = str(response), 200 - # Try to get the information about the request: + # Try to get the information about the request. + # There will be no data in any of these if the decorator was used to catch start-up and shut-down events. request_method = None request_url = None request_route = None @@ -653,46 +649,42 @@ def log_request_to_mongo( except: pass # Construct the log: - # for k in sensitive_keys: kwargs.get("inbound_headers", {}).pop(k, None) - # for k in sensitive_keys: kwargs.get("inbound_data", {}).pop(k, None) - log_json = { - "hostname": SERVER_HOSTNAME, - "os": f"{HOST_OS}", - "cpu": f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})", - "logId": kwargs.get("log_id"), - "logChain": kwargs.get("inbound_headers", {}).get("X-Log-Chain"), - "log": log_type, - "operation": operation, - "apiVer": kwargs.get("api_version"), - "method": request_method, - "url": request_url, - "route": request_route, - "ts": request_ts, - "tat": time.perf_counter() - start_ts, - "cpuTime": time.process_time() - cpu_start_ts, - "headers": kwargs.get("inbound_headers"), - "data": kwargs.get("inbound_data") if log_input else "not logged", - "files": { + api_log = APILogModel( + project = project, + log = log_type, + operation = operation, + apiVer = kwargs.get("api_version"), + logId = kwargs.get("log_id"), + logChain = kwargs.get("inbound_headers", {}).get("X-Log-Chain"), + method = request_method, + url = request_url, + route = request_route, + ts = request_ts, + tat = time.perf_counter() - start_ts, + cpuTime = time.process_time() - cpu_start_ts, + headers = kwargs.get("inbound_headers"), + data = kwargs.get("inbound_data") if log_input else "not logged", + files = { k: { "name": v["name"], "size": v["size"] } for k, v in kwargs.get("inbound_files", {}).items() }, - "exception": None if exception is None else describe_exception(exception), - "response": response_to_log, - "httpCode": http_code_to_log - } + exception = None if exception is None else describe_exception(exception), + response = response_to_log, + httpCode = http_code_to_log + ) # Write the log: app_attr = getattr(current_app, attr_name) inserted_id = await app_attr.insert_one( collection = collection, - document = log_json + document = api_log.model_dump() ) # Return the response from the wrapped function. if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -709,7 +701,9 @@ def should_not_be_under_maintenance(attr_name): """ Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create - a boolean variable within the scope of the 'current_app' for this to work. + a boolean variable within the scope of the 'current_app' for this to work. An alternate to this is to set the value + in an environment variable named 'IS_UNDER_MAINTENANCE' to a string value of either 'True' or 'False' for + multi-worker deployments. :param attr_name: The name of the boolean variable that will hold the information about the app being under maintenance. If its value is True at the time of checking, the incoming request will be rejected. :return: The decorator factory. @@ -726,14 +720,16 @@ def should_not_be_under_maintenance(attr_name): # Get the attribute and check if it indicates that the app is under maintenance, # call the wrapped function if not under maintenance: app_attr = getattr(current_app, attr_name) - if app_attr: response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() + env_attr = True if os.environ.get("IS_UNDER_MAINTENANCE", "False").lower() == "true" else False + if app_attr or env_attr: + response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() else: response = await func(*args, **kwargs) kwargs["decorator_count"] -= 1 # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -780,7 +776,7 @@ def only_whitelisted_ips(attr_name): # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -888,7 +884,7 @@ def limit_rate( # Return the response from the function call: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -927,7 +923,7 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -950,51 +946,6 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): return decorator -# --------------------------------------------------------------------------------------------------------------------- - - -def measure_metrics_for_prometheus(): - - """ - Use this decorator to automatically measure metrics for using in Prometheus. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # We use the context of the measurement class: - async with MetricsAPI( - method = f"{request.method}", - endpoint = str(request.url_rule.rule) - ) as metrics: - - # Invoke the wrapped function: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Interpret the HTTP code: - if isinstance(response, ResponseModel): _, metrics.http_code = response.for_quart() - elif isinstance(response, tuple): metrics.http_code = response[1] - else: metrics.http_code = 200 - - # Done here: - if ( - kwargs["decorator_count"] == 1 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** diff --git a/api/metrics_prometheus.py b/api/log.py similarity index 51% rename from api/metrics_prometheus.py rename to api/log.py index f7e9e3e..9f5acef 100644 --- a/api/metrics_prometheus.py +++ b/api/log.py @@ -3,15 +3,14 @@ AUTHOR: Khushal P Soonderji - Sharvil J Daiya DATE: - Saturday, 28th Sept., 2024 + Thursday, 12th Sept., 2024 OBJECTIVE: - To have one place from where several metrics are measured using easy to use context managers. + To have a structure to the response sent from the API calls. REFERENCES: @@ -31,16 +30,21 @@ # ***************************************************************************************************************** -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") +# System-level activities: +import distro +import socket +import platform -# To measure time: -import time +# For data-modelling: +from pydantic import BaseModel, Field +from typing import Any, Optional, List, Literal -# To capture metrics for Prometheus: -from prometheus_client import Counter, Summary, Gauge +# To work with date and time: +import datetime + +# My utils: +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.date_time import date_time # ***************************************************************************************************************** @@ -50,31 +54,11 @@ from prometheus_client import Counter, Summary, Gauge # ***************************************************************************************************************** -# Define the Prometheus metrics. -# FOR THE MICROSERVICE AS A WHOLE: -WORKER_COUNT = Counter( - name = "ms_workers_active_total", - documentation = "The number of threads for the microservice being monitored.", - labelnames = ["project_name", "service_name", "host_name"] -) - -# Define the Prometheus metrics. -# FOR INDIVIDUAL API ENDPOINTS: -REQUEST_LATENCY = Summary( - name = "http_request_latency_seconds", - documentation = "Latency of HTTP requests in seconds.", - labelnames = ["method", "endpoint", "http_status"] -) -TOTAL_REQUEST_COUNT = Counter( - name = "http_requests_total", - documentation = "Total HTTP requests.", - labelnames = ["method", "endpoint", "http_status"] -) -LIVE_REQUEST_COUNT = Gauge( - name = "http_requests_live_total", - documentation = "To check if an API endpoint is being served right now.", - labelnames = ["endpoint"] -) +# Info for logging that will stay constant during runtime: +SERVER_HOSTNAME = str(socket.gethostname()) +PLATFORM_INFO = platform.uname() +HOST_OS = str(distro.name(True)) +HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" # ***************************************************************************************************************** @@ -94,59 +78,39 @@ LIVE_REQUEST_COUNT = Gauge( # ***************************************************************************************************************** -class MetricsAPI: +class APILogModel(BaseModel): - """ - Use this class through its context manager to automatically measure all the metrics in one place. - This was originally created to measure the performance of API endpoints made in Quart, but it should work with - other frameworks as well. - """ + # To identify the machine the code is running on. + # DO NOT MODIFY THESE: + hostname: str = SERVER_HOSTNAME + os: str = HOST_OS + cpu: str = HOST_CPU - def __init__( - self, - method = None, - endpoint = None, - raise_exception = False - ): + # To identify the project and actions: + project: Optional[str] = None + log: str + operation: str + apiVer: Optional[str] = None + logId: Optional[str] = None + logChain: Optional[str] = None - # Make provisions for things to note. - # NOTE: THESE MUST BE SET FROM OUTSIDE: - self.method = method - self.endpoint = endpoint - self.http_code = None - self.__raise_exception = raise_exception + # Timing metrics: + ts: datetime.datetime + tat: float + cpuTime: float - async def __aenter__(self): + # To understand the request that came in: + method: Optional[str] = None + url: Optional[str] = None + route: Optional[str] = None + headers: Optional[Any] = None + data: Optional[Any] = None + files: Optional[Any] = None - # Note down the start time immediately: - self.__start_ts = time.perf_counter() - self.__cpu_start_ts = time.process_time() - - # Note down the metrics: - LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1) - - # Setup done: - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - - # Note down the metrics: - LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1) - - TOTAL_REQUEST_COUNT.labels( - self.method, - self.endpoint, - self.http_code - ).inc() - - REQUEST_LATENCY.labels( - self.method, - self.endpoint, - self.http_code - ).observe(time.perf_counter() - self.__start_ts) - - # Handle the exception as per the user's preference: - return False if self.__raise_exception else True + # To understand the output that went out: + exception: Optional[Any] = None + response: Optional[Any] = None + httpCode: Optional[int] = None # ***************************************************************************************************************** @@ -168,31 +132,8 @@ class MetricsAPI: if __name__ == "__main__": - import asyncio - import random - from prometheus_client import generate_latest + my_log = APILogModel( + log = "internal" + ) - async def simulate_endpoint(): - - async with MetricsAPI( - method = random.choice(["GET", "POST"]), - endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}" - ) as metrics: - - # Simulate some action on some endpoint: - await asyncio.sleep(1.0) - - # Note down the values: - metrics.http_code = 200 - - async def main(): - - print("Simulating endpoints...") - tasks = [simulate_endpoint() for _ in range(250)] - await asyncio.gather(*tasks) - print("Done!") - - print("METRICS:") - print(generate_latest().decode()) - - asyncio.run(main()) + print(my_log) diff --git a/api/response.py b/api/response.py index 3c51696..02e3d72 100644 --- a/api/response.py +++ b/api/response.py @@ -30,9 +30,11 @@ # ***************************************************************************************************************** +# For data-modelling: from pydantic import BaseModel from typing import Any, Optional, List +# My utils: from utils_v2.api.codes import StatusCodes, HttpCodes @@ -64,6 +66,11 @@ from utils_v2.api.codes import StatusCodes, HttpCodes class ResponseModel(BaseModel): + """ + A model for how the response should be when developing API endpoints. + """ + + # The fields that you want in your response: status_code: StatusCodes message: Optional[str | List] = None data: Optional[Any] = None @@ -74,6 +81,11 @@ class ResponseModel(BaseModel): def for_quart(self): + """ + Call this when you are using either Flask or Quart as your framework. + :return: The output as expected by Flask and Quart. + """ + # Construct the basic structure: response_dict = { "status": 1 if self.status_code.value[0] else 0, diff --git a/cache/__pycache__/__init__.cpython-310.pyc b/cache/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..06f9307 Binary files /dev/null and b/cache/__pycache__/__init__.cpython-310.pyc differ diff --git a/cache/__pycache__/async_redis_cache.cpython-310.pyc b/cache/__pycache__/async_redis_cache.cpython-310.pyc new file mode 100644 index 0000000..0a16447 Binary files /dev/null and b/cache/__pycache__/async_redis_cache.cpython-310.pyc differ diff --git a/cache/async_redis_cache.py b/cache/async_redis_cache.py index 14df725..a0ab8c0 100644 --- a/cache/async_redis_cache.py +++ b/cache/async_redis_cache.py @@ -123,6 +123,52 @@ def cache_it(cache = None, expiry = 120): return decorator +# --------------------------------------------------------------------------------------------------------------------- + + +def cache_class_methods(attr_name, expiry = 120): + + """ + This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. + If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the + cache instead of going through the whole function again. + :param attr_name: The name of the variable that has an instance of "AsyncRedisCache". + :param expiry: The time in seconds after which the cached data must be cleared. + :return: The decorator that automatically caches your data. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(self, *args, **kwargs): + + # Get the cache object first: + cache_obj = getattr(self, attr_name) + + # We first use the name of the function and the inputs given to it to generate a key for Redis with the + # simple hashing and shortening by way of base64 strings: + inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) + sha256_hash = hashlib.sha256() + sha256_hash.update(inputs_given.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Now we check if we have the value in cache: + try: response = await cache_obj.get(base64_key, raise_exception = True) + + # If the key doesn't exist, we pass through the function and store the results. + except: + response = await func(self, *args, **kwargs) + await cache_obj.set(key = base64_key, value = response, expiry = expiry) + + # Return the response from the wrapped function. + return response + + return wrapper + + return decorator + + # ***************************************************************************************************************** # ***** **** # *** CLASSES *** diff --git a/database/__pycache__/async_mysql_v2.cpython-310.pyc b/database/__pycache__/async_mysql_v2.cpython-310.pyc new file mode 100644 index 0000000..b97ae9b Binary files /dev/null and b/database/__pycache__/async_mysql_v2.cpython-310.pyc differ diff --git a/database/async_mongo_v2.py b/database/async_mongo_v2.py index dc9955a..663ed3a 100644 --- a/database/async_mongo_v2.py +++ b/database/async_mongo_v2.py @@ -1691,42 +1691,43 @@ if __name__ == "__main__": async def main(): # Create an instance of the database connector: - my_fs = AsyncMongoStorage( - connection_string = constants.MONGO_FILE_CONNECTION_STRING, - database_name = constants.MONGO_FILE_DATABASE_NAME, + my_db = AsyncMongo( + connection_string = constants.MONGO_DATA_CONNECTION_STRING, + database_name = constants.MONGO_DATA_DATABASE_NAME, max_connections = 10, debug = True ) # Connect to the database: - await my_fs.connect() + await my_db.connect() - # Keep performing the changes in batches till you have corrections to make: - while True: + # # Get the documents to migrate: + # documents = await my_db.find_many( + # collection = "scriptData", + # filter = {}, + # limit = 50, + # projection = {"_id": False} + # ) + # # print(json.to_string(documents, default = str)) + # + # # Adjust them: + # adjusted_documents = [] + # for document in documents: + # script_id = document.pop("scriptId") + # adjusted_document = { + # "scriptId": script_id, + # "desc": "no desc", + # "content": document + # } + # adjusted_documents.append(adjusted_document) + # print(json.to_string(adjusted_documents, default = str)) + # + # # Insert the adjusted ones to the new collection: + # response = await my_db.insert_many( + # collection = "_scriptData", + # documents = adjusted_documents + # ) + # print("RESPONSE:", response) - # Find all the files that have their metadata as a string: - files = await my_fs.find_many( - filter = { - "metadata": {"$type": "string"} - }, - limit = 10 - ) - - print(my_fs.to_json_string(files)) - break - - # # If no matches were found: - # if not files: break - # - # # Fix the metadata file-by-file: - # for file in files: - # file_id = file["_id"] - # success = await my_fs.replace_metadata_for_one( - # filter = {"_id": file_id}, - # replacement = json.from_string(file["metadata"]) - # ) - # print(file_id, ":", success) - - print("Fixes done!") asyncio.run(main()) diff --git a/database/async_mysql_v2.py b/database/async_mysql_v2.py index 911bfaf..fc977c6 100644 --- a/database/async_mysql_v2.py +++ b/database/async_mysql_v2.py @@ -96,7 +96,8 @@ class AsyncMySQL: """ A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as - JSON-like structures (list or dict). + JSON-like structures (list or dict). The format for the results was very specific to our use case for serving + Bicree's requirement. This may not serve your requirement at all. :param pool_size: The number of connections to maintain n a pool. :param args: Any arguments to pass. Not used. :param kwargs: Pass the connection configuration from here. @@ -207,8 +208,6 @@ class AsyncMySQL: await asyncio.sleep(backoff_seconds) backoff_seconds = backoff_seconds * backoff_multiplier - print("LEN:", len(results)) - # If the results are blank: if len(results) == 0: return { "status": results[0][0]["status"], @@ -238,9 +237,6 @@ class AsyncMySQL: formatted_results["seconds"] = time.perf_counter() - start_ts # Done here: - print(f"{procedure_name}:") - print(json.to_string(formatted_results)) - print("\n") if return_exception: return formatted_results, exception else: return formatted_results @@ -274,12 +270,20 @@ if __name__ == "__main__": :return: None. """ + # cred_json = { + # "host": "del.ditscentre.in", + # "user": "bicree", + # "port": 3306, + # "password": "9c3b2808a4aa281129d399fe09e69b53", + # "database": "bicree" + # } + cred_json = { "host": "del.ditscentre.in", - "user": "bicree", + "user": "caOffice", "port": 3306, - "password": "9c3b2808a4aa281129d399fe09e69b53", - "database": "bicree" + "password": "jstArchon", + "database": "caOffice" } db_conn = AsyncMySQL( @@ -299,10 +303,16 @@ if __name__ == "__main__": # ) # ) + # result = await db_conn.call_procedure_and_get_json( + # procedure_name = "listSummary", + # procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",) + # ) + result = await db_conn.call_procedure_and_get_json( - procedure_name = "listSummary", - procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",) + procedure_name = "campaign_activity_report", + procedure_args = (10000000,) ) + print("RESULT:", json.to_string(result, default = str)) start_time = time.time() diff --git a/image/scanner/scanner.py b/image/scanner/scanner.py index 424ddd1..28fe69b 100644 --- a/image/scanner/scanner.py +++ b/image/scanner/scanner.py @@ -387,7 +387,7 @@ if __name__ == "__main__": import time my_scanner = DocumentScanner( - layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], ocr_languages = ["en"] diff --git a/image/scanner/scanner_v2.py b/image/scanner/scanner_v2.py index 2a5b9fc..fc63627 100644 --- a/image/scanner/scanner_v2.py +++ b/image/scanner/scanner_v2.py @@ -353,7 +353,7 @@ if __name__ == "__main__": import time my_scanner = DocumentScanner( - layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", + layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt", whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10], # whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10], ocr_languages = ["en"]