Squashed 'utils_v2/' changes from ef9630d..271426c
271426c (20241010) Can now cache class methods by accessing a caching object from the class itself! 606b02a Merge commit 'b0e0760fc7bb6336debf621860d1286575a0b8cc' 3d2a723 (20241009) Work in progress. 58266e3 (20241009) 'last decorator' bug-fixed 53e7a39 (20241009) Work in progress. dbfc17f (20241009) 'last decorator' bug-fixed 11875e6 (20241008) logging decorator and model improved. git-subtree-dir: utils_v2 git-subtree-split: 271426c19bea8de5edec7ecb79ee04368c47b02f
This commit is contained in:
@@ -11,7 +11,7 @@
|
||||
OBJECTIVE:
|
||||
|
||||
To provide an easy way to assess images for blurriness.
|
||||
Tried and implemented using HuggingFace models.
|
||||
Tried and implemented using HuggingFace behaviour_models.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
@@ -193,7 +193,7 @@ if __name__ == "__main__":
|
||||
|
||||
async def main():
|
||||
image = r"/home/developer/Downloads/low-res-check.png"
|
||||
assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/image_classification/BlurOrBokeh")
|
||||
assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh")
|
||||
usable = await assessor.is_ok(image)
|
||||
classes = await assessor.classify(image)
|
||||
print("IS OKAY:", usable)
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
OBJECTIVE:
|
||||
|
||||
To provide an easy way to assess images for adult content.
|
||||
Tried and implemented using HuggingFace models.
|
||||
Tried and implemented using HuggingFace behaviour_models.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
|
||||
@@ -13,12 +13,12 @@
|
||||
To provide an easy way to get masks from dichotomous image segmentation.
|
||||
|
||||
This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait".
|
||||
You may experiment with other models too, but make sure that model is made for "dichotomous" behaviour. This
|
||||
You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This
|
||||
means that the model should have only two classes like "foreground", and "background". The specified model was
|
||||
trained for implementing portrait mode style blurring of backgrounds.
|
||||
|
||||
The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not
|
||||
support drop-in replacement for other models, please be aware about this.
|
||||
support drop-in replacement for other behaviour_models, please be aware about this.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To provide a class to detect objects in images using YOLO models.
|
||||
To provide a class to detect objects in images using YOLO behaviour_models.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
@@ -166,7 +166,7 @@ class YoloDetect:
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "models", "yolov8x.pt")
|
||||
# model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt")
|
||||
model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt"
|
||||
# sample_image_path = r"/home/developer/Downloads/2_cats.jpg"
|
||||
sample_image_path = r"/home/developer/Downloads/flattened_image.jpg"
|
||||
|
||||
Binary file not shown.
Binary file not shown.
+47
-96
@@ -46,13 +46,8 @@ from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.security import sanitizers
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.api.log import APILogModel
|
||||
from utils_v2.api.response import ResponseModel
|
||||
from utils_v2.api.metrics_prometheus import (
|
||||
TOTAL_REQUEST_COUNT,
|
||||
LIVE_REQUEST_COUNT,
|
||||
REQUEST_LATENCY,
|
||||
MetricsAPI
|
||||
)
|
||||
|
||||
# To work with date and time:
|
||||
import time
|
||||
@@ -60,11 +55,9 @@ import datetime
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
import distro
|
||||
import socket
|
||||
import platform
|
||||
import os
|
||||
|
||||
# For Pydantic data-models:
|
||||
# For Pydantic data-behaviour_models:
|
||||
import pydantic
|
||||
|
||||
# For hashing and shortening the hash:
|
||||
@@ -90,11 +83,6 @@ import asyncio
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Info for logging that will stay constant during runtime:
|
||||
SERVER_HOSTNAME = socket.gethostname()
|
||||
PLATFORM_INFO = platform.uname()
|
||||
HOST_OS = distro.name(True)
|
||||
|
||||
# Chars to choose from for random strings:
|
||||
ALPHANUMERIC_CHARS = string.ascii_letters + string.digits
|
||||
|
||||
@@ -359,7 +347,7 @@ def set_api_version(api_version):
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -417,7 +405,7 @@ def read_input(
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -556,7 +544,7 @@ def validate_input(
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -572,6 +560,8 @@ def validate_input(
|
||||
def log_request_to_mongo(
|
||||
attr_name,
|
||||
collection: str = "logs",
|
||||
api_version: str = None,
|
||||
project: str = None,
|
||||
log_type: str = None,
|
||||
operation: str = None,
|
||||
log_input: bool = True,
|
||||
@@ -586,6 +576,8 @@ def log_request_to_mongo(
|
||||
:param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the
|
||||
scope of 'current_app'.
|
||||
:param collection: The name of the collection to write the log into.
|
||||
:param api_version: The version code of the API endpoint that is being logged.
|
||||
:param project: The name of the project that the endpoint was built for.
|
||||
:param log_type: A hint to identify what the log was for.
|
||||
:param operation: A hint to identify what was action was being performed.
|
||||
:param log_input: Whether, or not, you would like to log the input that came in.
|
||||
@@ -602,6 +594,9 @@ def log_request_to_mongo(
|
||||
# Let the next in-line decorator know that it has been wrapped:
|
||||
kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1
|
||||
|
||||
# Set the api version as needed:
|
||||
kwargs["api_version"] = kwargs.get("api_version", api_version)
|
||||
|
||||
# Make variables and extract available info.:
|
||||
exception = None
|
||||
response = None
|
||||
@@ -635,7 +630,8 @@ def log_request_to_mongo(
|
||||
elif isinstance(response, tuple): response_to_log, http_code_to_log = response
|
||||
else: response_to_log, http_code_to_log = str(response), 200
|
||||
|
||||
# Try to get the information about the request:
|
||||
# Try to get the information about the request.
|
||||
# There will be no data in any of these if the decorator was used to catch start-up and shut-down events.
|
||||
request_method = None
|
||||
request_url = None
|
||||
request_route = None
|
||||
@@ -653,46 +649,42 @@ def log_request_to_mongo(
|
||||
except: pass
|
||||
|
||||
# Construct the log:
|
||||
# for k in sensitive_keys: kwargs.get("inbound_headers", {}).pop(k, None)
|
||||
# for k in sensitive_keys: kwargs.get("inbound_data", {}).pop(k, None)
|
||||
log_json = {
|
||||
"hostname": SERVER_HOSTNAME,
|
||||
"os": f"{HOST_OS}",
|
||||
"cpu": f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})",
|
||||
"logId": kwargs.get("log_id"),
|
||||
"logChain": kwargs.get("inbound_headers", {}).get("X-Log-Chain"),
|
||||
"log": log_type,
|
||||
"operation": operation,
|
||||
"apiVer": kwargs.get("api_version"),
|
||||
"method": request_method,
|
||||
"url": request_url,
|
||||
"route": request_route,
|
||||
"ts": request_ts,
|
||||
"tat": time.perf_counter() - start_ts,
|
||||
"cpuTime": time.process_time() - cpu_start_ts,
|
||||
"headers": kwargs.get("inbound_headers"),
|
||||
"data": kwargs.get("inbound_data") if log_input else "not logged",
|
||||
"files": {
|
||||
api_log = APILogModel(
|
||||
project = project,
|
||||
log = log_type,
|
||||
operation = operation,
|
||||
apiVer = kwargs.get("api_version"),
|
||||
logId = kwargs.get("log_id"),
|
||||
logChain = kwargs.get("inbound_headers", {}).get("X-Log-Chain"),
|
||||
method = request_method,
|
||||
url = request_url,
|
||||
route = request_route,
|
||||
ts = request_ts,
|
||||
tat = time.perf_counter() - start_ts,
|
||||
cpuTime = time.process_time() - cpu_start_ts,
|
||||
headers = kwargs.get("inbound_headers"),
|
||||
data = kwargs.get("inbound_data") if log_input else "not logged",
|
||||
files = {
|
||||
k: {
|
||||
"name": v["name"],
|
||||
"size": v["size"]
|
||||
} for k, v in kwargs.get("inbound_files", {}).items()
|
||||
},
|
||||
"exception": None if exception is None else describe_exception(exception),
|
||||
"response": response_to_log,
|
||||
"httpCode": http_code_to_log
|
||||
}
|
||||
exception = None if exception is None else describe_exception(exception),
|
||||
response = response_to_log,
|
||||
httpCode = http_code_to_log
|
||||
)
|
||||
|
||||
# Write the log:
|
||||
app_attr = getattr(current_app, attr_name)
|
||||
inserted_id = await app_attr.insert_one(
|
||||
collection = collection,
|
||||
document = log_json
|
||||
document = api_log.model_dump()
|
||||
)
|
||||
|
||||
# Return the response from the wrapped function.
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -709,7 +701,9 @@ def should_not_be_under_maintenance(attr_name):
|
||||
|
||||
"""
|
||||
Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create
|
||||
a boolean variable within the scope of the 'current_app' for this to work.
|
||||
a boolean variable within the scope of the 'current_app' for this to work. An alternate to this is to set the value
|
||||
in an environment variable named 'IS_UNDER_MAINTENANCE' to a string value of either 'True' or 'False' for
|
||||
multi-worker deployments.
|
||||
:param attr_name: The name of the boolean variable that will hold the information about the app being under
|
||||
maintenance. If its value is True at the time of checking, the incoming request will be rejected.
|
||||
:return: The decorator factory.
|
||||
@@ -726,14 +720,16 @@ def should_not_be_under_maintenance(attr_name):
|
||||
# Get the attribute and check if it indicates that the app is under maintenance,
|
||||
# call the wrapped function if not under maintenance:
|
||||
app_attr = getattr(current_app, attr_name)
|
||||
if app_attr: response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart()
|
||||
env_attr = True if os.environ.get("IS_UNDER_MAINTENANCE", "False").lower() == "true" else False
|
||||
if app_attr or env_attr:
|
||||
response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart()
|
||||
else:
|
||||
response = await func(*args, **kwargs)
|
||||
kwargs["decorator_count"] -= 1
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -780,7 +776,7 @@ def only_whitelisted_ips(attr_name):
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -888,7 +884,7 @@ def limit_rate(
|
||||
|
||||
# Return the response from the function call:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -927,7 +923,7 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None):
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
kwargs["decorator_count"] == 0 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
@@ -950,51 +946,6 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None):
|
||||
return decorator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def measure_metrics_for_prometheus():
|
||||
|
||||
"""
|
||||
Use this decorator to automatically measure metrics for using in Prometheus.
|
||||
:return: The decorator factory.
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
|
||||
# Let the next in-line decorator know that it has been wrapped:
|
||||
kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1
|
||||
|
||||
# We use the context of the measurement class:
|
||||
async with MetricsAPI(
|
||||
method = f"{request.method}",
|
||||
endpoint = str(request.url_rule.rule)
|
||||
) as metrics:
|
||||
|
||||
# Invoke the wrapped function:
|
||||
response = await func(*args, **kwargs)
|
||||
kwargs["decorator_count"] -= 1
|
||||
|
||||
# Interpret the HTTP code:
|
||||
if isinstance(response, ResponseModel): _, metrics.http_code = response.for_quart()
|
||||
elif isinstance(response, tuple): metrics.http_code = response[1]
|
||||
else: metrics.http_code = 200
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
|
||||
@@ -3,15 +3,14 @@
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
Sharvil J Daiya
|
||||
|
||||
DATE:
|
||||
|
||||
Saturday, 28th Sept., 2024
|
||||
Thursday, 12th Sept., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To have one place from where several metrics are measured using easy to use context managers.
|
||||
To have a structure to the response sent from the API calls.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
@@ -31,16 +30,21 @@
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
# System-level activities:
|
||||
import distro
|
||||
import socket
|
||||
import platform
|
||||
|
||||
# To measure time:
|
||||
import time
|
||||
# For data-modelling:
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Any, Optional, List, Literal
|
||||
|
||||
# To capture metrics for Prometheus:
|
||||
from prometheus_client import Counter, Summary, Gauge
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# My utils:
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -50,31 +54,11 @@ from prometheus_client import Counter, Summary, Gauge
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Define the Prometheus metrics.
|
||||
# FOR THE MICROSERVICE AS A WHOLE:
|
||||
WORKER_COUNT = Counter(
|
||||
name = "ms_workers_active_total",
|
||||
documentation = "The number of threads for the microservice being monitored.",
|
||||
labelnames = ["project_name", "service_name", "host_name"]
|
||||
)
|
||||
|
||||
# Define the Prometheus metrics.
|
||||
# FOR INDIVIDUAL API ENDPOINTS:
|
||||
REQUEST_LATENCY = Summary(
|
||||
name = "http_request_latency_seconds",
|
||||
documentation = "Latency of HTTP requests in seconds.",
|
||||
labelnames = ["method", "endpoint", "http_status"]
|
||||
)
|
||||
TOTAL_REQUEST_COUNT = Counter(
|
||||
name = "http_requests_total",
|
||||
documentation = "Total HTTP requests.",
|
||||
labelnames = ["method", "endpoint", "http_status"]
|
||||
)
|
||||
LIVE_REQUEST_COUNT = Gauge(
|
||||
name = "http_requests_live_total",
|
||||
documentation = "To check if an API endpoint is being served right now.",
|
||||
labelnames = ["endpoint"]
|
||||
)
|
||||
# Info for logging that will stay constant during runtime:
|
||||
SERVER_HOSTNAME = str(socket.gethostname())
|
||||
PLATFORM_INFO = platform.uname()
|
||||
HOST_OS = str(distro.name(True))
|
||||
HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -94,59 +78,39 @@ LIVE_REQUEST_COUNT = Gauge(
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class MetricsAPI:
|
||||
class APILogModel(BaseModel):
|
||||
|
||||
"""
|
||||
Use this class through its context manager to automatically measure all the metrics in one place.
|
||||
This was originally created to measure the performance of API endpoints made in Quart, but it should work with
|
||||
other frameworks as well.
|
||||
"""
|
||||
# To identify the machine the code is running on.
|
||||
# DO NOT MODIFY THESE:
|
||||
hostname: str = SERVER_HOSTNAME
|
||||
os: str = HOST_OS
|
||||
cpu: str = HOST_CPU
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
method = None,
|
||||
endpoint = None,
|
||||
raise_exception = False
|
||||
):
|
||||
# To identify the project and actions:
|
||||
project: Optional[str] = None
|
||||
log: str
|
||||
operation: str
|
||||
apiVer: Optional[str] = None
|
||||
logId: Optional[str] = None
|
||||
logChain: Optional[str] = None
|
||||
|
||||
# Make provisions for things to note.
|
||||
# NOTE: THESE MUST BE SET FROM OUTSIDE:
|
||||
self.method = method
|
||||
self.endpoint = endpoint
|
||||
self.http_code = None
|
||||
self.__raise_exception = raise_exception
|
||||
# Timing metrics:
|
||||
ts: datetime.datetime
|
||||
tat: float
|
||||
cpuTime: float
|
||||
|
||||
async def __aenter__(self):
|
||||
# To understand the request that came in:
|
||||
method: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
route: Optional[str] = None
|
||||
headers: Optional[Any] = None
|
||||
data: Optional[Any] = None
|
||||
files: Optional[Any] = None
|
||||
|
||||
# Note down the start time immediately:
|
||||
self.__start_ts = time.perf_counter()
|
||||
self.__cpu_start_ts = time.process_time()
|
||||
|
||||
# Note down the metrics:
|
||||
LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1)
|
||||
|
||||
# Setup done:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
|
||||
# Note down the metrics:
|
||||
LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1)
|
||||
|
||||
TOTAL_REQUEST_COUNT.labels(
|
||||
self.method,
|
||||
self.endpoint,
|
||||
self.http_code
|
||||
).inc()
|
||||
|
||||
REQUEST_LATENCY.labels(
|
||||
self.method,
|
||||
self.endpoint,
|
||||
self.http_code
|
||||
).observe(time.perf_counter() - self.__start_ts)
|
||||
|
||||
# Handle the exception as per the user's preference:
|
||||
return False if self.__raise_exception else True
|
||||
# To understand the output that went out:
|
||||
exception: Optional[Any] = None
|
||||
response: Optional[Any] = None
|
||||
httpCode: Optional[int] = None
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -168,31 +132,8 @@ class MetricsAPI:
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
from prometheus_client import generate_latest
|
||||
my_log = APILogModel(
|
||||
log = "internal"
|
||||
)
|
||||
|
||||
async def simulate_endpoint():
|
||||
|
||||
async with MetricsAPI(
|
||||
method = random.choice(["GET", "POST"]),
|
||||
endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}"
|
||||
) as metrics:
|
||||
|
||||
# Simulate some action on some endpoint:
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Note down the values:
|
||||
metrics.http_code = 200
|
||||
|
||||
async def main():
|
||||
|
||||
print("Simulating endpoints...")
|
||||
tasks = [simulate_endpoint() for _ in range(250)]
|
||||
await asyncio.gather(*tasks)
|
||||
print("Done!")
|
||||
|
||||
print("METRICS:")
|
||||
print(generate_latest().decode())
|
||||
|
||||
asyncio.run(main())
|
||||
print(my_log)
|
||||
@@ -30,9 +30,11 @@
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# For data-modelling:
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, Optional, List
|
||||
|
||||
# My utils:
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
|
||||
|
||||
@@ -64,6 +66,11 @@ from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
|
||||
class ResponseModel(BaseModel):
|
||||
|
||||
"""
|
||||
A model for how the response should be when developing API endpoints.
|
||||
"""
|
||||
|
||||
# The fields that you want in your response:
|
||||
status_code: StatusCodes
|
||||
message: Optional[str | List] = None
|
||||
data: Optional[Any] = None
|
||||
@@ -74,6 +81,11 @@ class ResponseModel(BaseModel):
|
||||
|
||||
def for_quart(self):
|
||||
|
||||
"""
|
||||
Call this when you are using either Flask or Quart as your framework.
|
||||
:return: The output as expected by Flask and Quart.
|
||||
"""
|
||||
|
||||
# Construct the basic structure:
|
||||
response_dict = {
|
||||
"status": 1 if self.status_code.value[0] else 0,
|
||||
|
||||
BIN
Binary file not shown.
Binary file not shown.
Vendored
+46
@@ -123,6 +123,52 @@ def cache_it(cache = None, expiry = 120):
|
||||
return decorator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cache_class_methods(attr_name, expiry = 120):
|
||||
|
||||
"""
|
||||
This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
|
||||
If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
|
||||
cache instead of going through the whole function again.
|
||||
:param attr_name: The name of the variable that has an instance of "AsyncRedisCache".
|
||||
:param expiry: The time in seconds after which the cached data must be cleared.
|
||||
:return: The decorator that automatically caches your data.
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(self, *args, **kwargs):
|
||||
|
||||
# Get the cache object first:
|
||||
cache_obj = getattr(self, attr_name)
|
||||
|
||||
# We first use the name of the function and the inputs given to it to generate a key for Redis with the
|
||||
# simple hashing and shortening by way of base64 strings:
|
||||
inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
|
||||
sha256_hash = hashlib.sha256()
|
||||
sha256_hash.update(inputs_given.encode("utf-8"))
|
||||
hashed_key = sha256_hash.digest()
|
||||
base64_key = base64.b64encode(hashed_key).decode("utf-8")
|
||||
|
||||
# Now we check if we have the value in cache:
|
||||
try: response = await cache_obj.get(base64_key, raise_exception = True)
|
||||
|
||||
# If the key doesn't exist, we pass through the function and store the results.
|
||||
except:
|
||||
response = await func(self, *args, **kwargs)
|
||||
await cache_obj.set(key = base64_key, value = response, expiry = expiry)
|
||||
|
||||
# Return the response from the wrapped function.
|
||||
return response
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** CLASSES ***
|
||||
|
||||
Binary file not shown.
+30
-29
@@ -1691,42 +1691,43 @@ if __name__ == "__main__":
|
||||
async def main():
|
||||
|
||||
# Create an instance of the database connector:
|
||||
my_fs = AsyncMongoStorage(
|
||||
connection_string = constants.MONGO_FILE_CONNECTION_STRING,
|
||||
database_name = constants.MONGO_FILE_DATABASE_NAME,
|
||||
my_db = AsyncMongo(
|
||||
connection_string = constants.MONGO_DATA_CONNECTION_STRING,
|
||||
database_name = constants.MONGO_DATA_DATABASE_NAME,
|
||||
max_connections = 10,
|
||||
debug = True
|
||||
)
|
||||
|
||||
# Connect to the database:
|
||||
await my_fs.connect()
|
||||
await my_db.connect()
|
||||
|
||||
# Keep performing the changes in batches till you have corrections to make:
|
||||
while True:
|
||||
|
||||
# Find all the files that have their metadata as a string:
|
||||
files = await my_fs.find_many(
|
||||
filter = {
|
||||
"metadata": {"$type": "string"}
|
||||
},
|
||||
limit = 10
|
||||
)
|
||||
|
||||
print(my_fs.to_json_string(files))
|
||||
break
|
||||
|
||||
# # If no matches were found:
|
||||
# if not files: break
|
||||
#
|
||||
# # Fix the metadata file-by-file:
|
||||
# for file in files:
|
||||
# file_id = file["_id"]
|
||||
# success = await my_fs.replace_metadata_for_one(
|
||||
# filter = {"_id": file_id},
|
||||
# replacement = json.from_string(file["metadata"])
|
||||
# # Get the documents to migrate:
|
||||
# documents = await my_db.find_many(
|
||||
# collection = "scriptData",
|
||||
# filter = {},
|
||||
# limit = 50,
|
||||
# projection = {"_id": False}
|
||||
# )
|
||||
# print(file_id, ":", success)
|
||||
# # print(json.to_string(documents, default = str))
|
||||
#
|
||||
# # Adjust them:
|
||||
# adjusted_documents = []
|
||||
# for document in documents:
|
||||
# script_id = document.pop("scriptId")
|
||||
# adjusted_document = {
|
||||
# "scriptId": script_id,
|
||||
# "desc": "no desc",
|
||||
# "content": document
|
||||
# }
|
||||
# adjusted_documents.append(adjusted_document)
|
||||
# print(json.to_string(adjusted_documents, default = str))
|
||||
#
|
||||
# # Insert the adjusted ones to the new collection:
|
||||
# response = await my_db.insert_many(
|
||||
# collection = "_scriptData",
|
||||
# documents = adjusted_documents
|
||||
# )
|
||||
# print("RESPONSE:", response)
|
||||
|
||||
print("Fixes done!")
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
+21
-11
@@ -96,7 +96,8 @@ class AsyncMySQL:
|
||||
|
||||
"""
|
||||
A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as
|
||||
JSON-like structures (list or dict).
|
||||
JSON-like structures (list or dict). The format for the results was very specific to our use case for serving
|
||||
Bicree's requirement. This may not serve your requirement at all.
|
||||
:param pool_size: The number of connections to maintain n a pool.
|
||||
:param args: Any arguments to pass. Not used.
|
||||
:param kwargs: Pass the connection configuration from here.
|
||||
@@ -207,8 +208,6 @@ class AsyncMySQL:
|
||||
await asyncio.sleep(backoff_seconds)
|
||||
backoff_seconds = backoff_seconds * backoff_multiplier
|
||||
|
||||
print("LEN:", len(results))
|
||||
|
||||
# If the results are blank:
|
||||
if len(results) == 0: return {
|
||||
"status": results[0][0]["status"],
|
||||
@@ -238,9 +237,6 @@ class AsyncMySQL:
|
||||
formatted_results["seconds"] = time.perf_counter() - start_ts
|
||||
|
||||
# Done here:
|
||||
print(f"{procedure_name}:")
|
||||
print(json.to_string(formatted_results))
|
||||
print("\n")
|
||||
if return_exception: return formatted_results, exception
|
||||
else: return formatted_results
|
||||
|
||||
@@ -274,12 +270,20 @@ if __name__ == "__main__":
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# cred_json = {
|
||||
# "host": "del.ditscentre.in",
|
||||
# "user": "bicree",
|
||||
# "port": 3306,
|
||||
# "password": "9c3b2808a4aa281129d399fe09e69b53",
|
||||
# "database": "bicree"
|
||||
# }
|
||||
|
||||
cred_json = {
|
||||
"host": "del.ditscentre.in",
|
||||
"user": "bicree",
|
||||
"user": "caOffice",
|
||||
"port": 3306,
|
||||
"password": "9c3b2808a4aa281129d399fe09e69b53",
|
||||
"database": "bicree"
|
||||
"password": "jstArchon",
|
||||
"database": "caOffice"
|
||||
}
|
||||
|
||||
db_conn = AsyncMySQL(
|
||||
@@ -299,10 +303,16 @@ if __name__ == "__main__":
|
||||
# )
|
||||
# )
|
||||
|
||||
# result = await db_conn.call_procedure_and_get_json(
|
||||
# procedure_name = "listSummary",
|
||||
# procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",)
|
||||
# )
|
||||
|
||||
result = await db_conn.call_procedure_and_get_json(
|
||||
procedure_name = "listSummary",
|
||||
procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",)
|
||||
procedure_name = "campaign_activity_report",
|
||||
procedure_args = (10000000,)
|
||||
)
|
||||
print("RESULT:", json.to_string(result, default = str))
|
||||
|
||||
|
||||
start_time = time.time()
|
||||
|
||||
@@ -387,7 +387,7 @@ if __name__ == "__main__":
|
||||
import time
|
||||
|
||||
my_scanner = DocumentScanner(
|
||||
layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
|
||||
layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
|
||||
whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
|
||||
# whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
ocr_languages = ["en"]
|
||||
|
||||
@@ -353,7 +353,7 @@ if __name__ == "__main__":
|
||||
import time
|
||||
|
||||
my_scanner = DocumentScanner(
|
||||
layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
|
||||
layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
|
||||
whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
|
||||
# whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
|
||||
ocr_languages = ["en"]
|
||||
|
||||
Reference in New Issue
Block a user