Resetting utils subtree.

This commit is contained in:
2024-12-25 11:45:58 +05:30
parent bedf7da07d
commit 7005ab5a8d
172 changed files with 856 additions and 136761 deletions
View File
View File
View File
+394
View File
@@ -0,0 +1,394 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Created: Wednesday, 18th Sept., 2024
Updated: Tuesday, 8th Oct., 2024
OBJECTIVE:
To be able to fetch logs for rapid issue resolution.
REFERENCES:
N/A
DOWNLOADS:
N/A
NOTES:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For using Quart:
from quart import Blueprint, current_app
# My utils:
from utils_v2.string import json
from utils_v2.api.codes import StatusCodes, HttpCodes
from utils_v2.api.response import ResponseModel
from utils_v2.api.async_quart import (
set_api_version,
read_input,
log_request_to_mongo,
should_not_be_under_maintenance,
only_whitelisted_ips,
limit_rate,
validate_input,
handle_cancelled_request
)
# Data models:
from utils_v2.api.log import APILogModel
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Related to Quart:
logs_bp = Blueprint("int_logs", __name__)
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
@logs_bp.record_once
def init(blueprint_setup_state):
# This gets called when the blueprint is registered.
# Consider this to be a one-time setup for the whole blueprint:
pass
# ---------------------------------------------------------------------------------------------------------------------
@logs_bp.route("/get/id/<log_id>", methods = ["POST", "GET"])
@set_api_version(api_version = "2.1.0")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@only_whitelisted_ips(attr_name = "whitelisted_ips")
@handle_cancelled_request()
async def get_log(
log_id,
**kwargs
):
"""
To get the log from its log id.
:param log_id: An identifier (string) for the log to fetch.
"""
fetched_log = await current_app.mongo.find_one(
collection = "logs",
filter = {"logId": log_id},
projection = {"_id": False}
)
if not fetched_log: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "no such log"
)
else: return ResponseModel(
status_code = StatusCodes.OK,
message = f"log found",
data = {"total": 1, "fetched": 1, "logs": fetched_log}
)
# ---------------------------------------------------------------------------------------------------------------------
@logs_bp.route("/get/exception/id/<log_id>", methods = ["POST", "GET"])
@set_api_version(api_version = "2.1.0")
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@only_whitelisted_ips(attr_name = "whitelisted_ips")
async def get_exception_from_log(
log_id,
**kwargs
):
"""
To get the log's exception from its log id.
:param log_id: An identifier (string) for the log to fetch.
"""
fetched_log = await current_app.mongo.find_one(
collection = "logs",
filter = {"logId": log_id},
projection = {
"_id": False,
"logId": True,
"log": True,
"operation": True,
"ts": True,
"exception": True
}
)
if not fetched_log: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "no such log"
)
else: return ResponseModel(
status_code = StatusCodes.OK,
message = f"log found",
data = {"total": 1, "fetched": 1, "logs": fetched_log}
)
# ---------------------------------------------------------------------------------------------------------------------
@logs_bp.route("/get/chain/<log_chain>", methods = ["POST", "GET"])
@set_api_version(api_version = "2.1.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@only_whitelisted_ips(attr_name = "whitelisted_ips")
async def get_log_chain(
log_chain,
inbound_headers: dict = None,
inbound_data: dict = None,
inbound_files: dict = None,
**kwargs
):
"""
To get the series of logs from its chain identifier.
:param log_chain: An identifier (string) for the log chain to fetch.
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
:param kwargs: Any number of extra inputs supplied by the decorators.
"""
total_count = await current_app.mongo.count(
collection = "logs",
filter = {"logChain": log_chain}
)
fetched_logs = await current_app.mongo.find_many(
collection = "logs",
filter = {"logChain": log_chain},
projection = {"_id": False},
sort = inbound_data.get("sort", {"_id": -1}),
limit = inbound_data.get("limit", 25),
skip = inbound_data.get("skip", 0)
)
fetched_count = len(fetched_logs)
if not fetched_logs: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "no such log chain"
)
else: return ResponseModel(
status_code = StatusCodes.OK,
message = f"{fetched_count} logs fetched",
data = {"total": total_count, "fetched": fetched_count, "logs": fetched_logs}
)
# ---------------------------------------------------------------------------------------------------------------------
@logs_bp.route("/get/exception/chain/<log_chain>", methods = ["POST", "GET"])
@set_api_version(api_version = "2.1.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@only_whitelisted_ips(attr_name = "whitelisted_ips")
async def get_exceptions_from_log_chain(
log_chain,
inbound_headers: dict = None,
inbound_data: dict = None,
inbound_files: dict = None,
**kwargs
):
"""
To get the series of log exceptions from its chain identifier.
:param log_chain: An identifier (string) for the log chain to fetch.
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
:param kwargs: Any number of extra inputs supplied by the decorators.
"""
total_count = await current_app.mongo.count(
collection = "logs",
filter = {"logChain": log_chain}
)
fetched_logs = await current_app.mongo.find_many(
collection = "logs",
filter = {"logChain": log_chain},
projection = {
"_id": False,
"log": True,
"operation": True,
"ts": True,
"exception": True
},
sort = inbound_data.get("sort", {"_id": -1}),
limit = inbound_data.get("limit", 25),
skip = inbound_data.get("skip", 0)
)
fetched_count = len(fetched_logs)
if not fetched_logs: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "no such log chain"
)
else: return ResponseModel(
status_code = StatusCodes.OK,
message = f"{fetched_count} logs fetched",
data = {"total": total_count, "fetched": fetched_count, "logs": fetched_logs}
)
# ---------------------------------------------------------------------------------------------------------------------
@logs_bp.route("/get/filter", methods = ["POST", "GET"])
@set_api_version(api_version = "2.1.0")
@read_input(sanitize_headers = False, sanitize_data = False)
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@only_whitelisted_ips(attr_name = "whitelisted_ips")
async def get_logs_by_filter(
inbound_headers: dict = None,
inbound_data: dict = None,
inbound_files: dict = None,
**kwargs
):
"""
To fetch logs by custom filters:
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
:param kwargs: Any number of extra inputs supplied by the decorators.
"""
total_count = await current_app.mongo.count(
collection = "logs",
filter = inbound_data["filter"]
)
fetched_logs = await current_app.mongo.find_many(
collection = "logs",
filter = inbound_data["filter"],
projection = inbound_data.get("projection", {"_id": False}),
sort = inbound_data.get("sort", {"_id": -1}),
limit = inbound_data.get("limit", 25),
skip = inbound_data.get("skip", 0)
)
fetched_count = len(fetched_logs)
if not fetched_logs: return ResponseModel(
status_code = StatusCodes.FAILED,
http_code = HttpCodes.NOT_FOUND,
message = "no matching logs"
)
else: return ResponseModel(
status_code = StatusCodes.OK,
message = f"{len(fetched_logs)} / {total_count} logs fetched",
data = {"total": total_count, "fetched": fetched_count, "logs": fetched_logs}
)
# ---------------------------------------------------------------------------------------------------------------------
@logs_bp.route("/set/api", methods = ["POST"])
@set_api_version(api_version = "2.1.0")
@read_input(sanitize_headers = True, sanitize_data = True)
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
@only_whitelisted_ips(attr_name = "whitelisted_ips")
@validate_input(data_validator = lambda x: APILogModel(**x))
async def set_api_log(
inbound_headers: dict = None,
inbound_data: dict | APILogModel = None,
inbound_files: dict = None,
**kwargs
):
"""
To set logs from internal whitelisted IPs.
:param inbound_headers: auto-extracted by the decorators.
:param inbound_data: auto-extracted by the decorators.
:param inbound_files: auto-extracted by the decorators.
:param kwargs: Any number of extra inputs supplied by the decorators.
"""
inserted_id = await current_app.mongo.insert_one(
collection = "logs",
document = inbound_data.model_dump()
)
return ResponseModel(
status_code = StatusCodes.OK if inserted_id else StatusCodes.FAILED,
http_code = HttpCodes.SUCCESS if inserted_id else HttpCodes.BAD_REQUEST
)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
+370
View File
@@ -0,0 +1,370 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 8th Oct., 2024
OBJECTIVE:
This is the central location for the Quart module.
We define the app here, and import and attach all blueprints here.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For system level activities:
import gc
import os
import psutil
# For using Quart:
from quart import Quart, request, current_app, redirect
from quart_cors import cors
# Common:
from shared import constants
# My utils:
from utils_v2.string import json
from utils_v2.api import async_quart
from utils_v2.date_time import date_time
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.api.async_quart import (
set_api_version,
read_input,
log_request_to_mongo,
should_not_be_under_maintenance,
only_whitelisted_ips,
limit_rate,
validate_input,
handle_cancelled_request
)
# For debugging:
from icecream import IceCreamDebugger
# All the blueprints:
from api.cred_data.blueprint import cred_and_data_bp
from api.logs.blueprint import logs_bp
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Quart related:
MODULE_BASE = "internal"
APP_VERSION = "2.0.0"
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# The Quart app:
app = Quart(__name__)
app = cors(app)
app.register_blueprint(cred_and_data_bp, url_prefix = f"/{MODULE_BASE}")
app.register_blueprint(logs_bp, url_prefix = f"/{MODULE_BASE}/logs")
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
@app.before_serving
@set_api_version(api_version = APP_VERSION)
@log_request_to_mongo(
attr_name = "mongo",
log_type = MODULE_BASE,
operation = "apiStart",
log_input = True,
log_output = True
)
async def app_startup(**kwargs):
"""
To initialize the variables that you would like to use in this module.
WARNING: ALL VARIABLES WILL BE INITIALIZED 'n' NUMBER OF TIMES, WHERE 'n' IS THE COUNT OF WORKERS DEPLOYED.
SO, IF YOU WANT TO CONNECT TO A DATABASE AND YOU ALLOW A POOL-SIZE OF 10 AND IF YOU DEPLOY 4 WORKERS, YOU WILL END
UP WITH 40 CONNECTIONS TO THE DATABASE.
:return: None.
"""
# Safe-halt mechanism for upgrades (for a single-worker run):
current_app.is_under_maintenance = False
# Debugging:
current_app.printer = IceCreamDebugger(prefix = f"{MODULE_BASE} (Q) | ", includeContext = True)
if os.environ["DEBUG"] == "True": current_app.printer.disable()
# To connect to Mongo:
current_app.mongo = AsyncMongo(
connection_string = constants.MONGO_DATA_CONNECTION_STRING,
database_name = constants.MONGO_DATA_DATABASE_NAME,
max_connections = 10,
debug = True,
debug_only_errors = True
)
await current_app.mongo.connect()
# Get the script data:
script_id = os.environ["SCRIPT_ID"]
current_app.script_data = (await current_app.mongo.find_one(
collection = "_scriptData",
filter = {"scriptId": script_id}
))["content"]
# Pick the important stuff:
current_app.whitelisted_ips = current_app.script_data["whitelistedIps"]
# Done here!
print("Worker ready!")
# ---------------------------------------------------------------------------------------------------------------------
@app.after_serving
@set_api_version(api_version = APP_VERSION)
@log_request_to_mongo(
attr_name = "mongo",
log_type = MODULE_BASE,
operation = "apiStop",
log_input = True,
log_output = True
)
async def app_shutdown(**kwargs):
"""
This is called when "app.shutdown()" is called.
:return: None.
"""
message = "Shutting down..."
current_app.printer(message)
# ---------------------------------------------------------------------------------------------------------------------
@app.before_request
async def enforce_https():
"""
Here we enforce HTTPS (secure) requests and effectively reject unsecure requests. We make exemptions for local
network requests for regular testing.
:return: A redirect to the secure version if the incoming request is not secure.
"""
# If the connection is not secure:
if request.scheme != "https":
# Check if it is a local IP. If not, redirect:
exempt_ips = ["0.0.0.0", "127.0.0.1"]
if not (
request.remote_addr.startswith("192.168.") or
request.remote_addr in exempt_ips
): return redirect(request.url.replace("http", "https"))
# ---------------------------------------------------------------------------------------------------------------------
@app.route(f"/", methods = ["GET", "POST"])
@app.route(f"/{MODULE_BASE}", methods = ["GET", "POST"])
async def root():
"""
To check if the service is running or not.
Use this to monitor the service from your "watchman" script.
:return: only "ok"
"""
return "ok"
# ---------------------------------------------------------------------------------------------------------------------
@app.route(f"/metrics/memory", methods = ["GET", "POST"])
@app.route(f"/{MODULE_BASE}/metrics/memory", methods = ["GET", "POST"])
async def metrics():
"""
TO measure the metrics of the app.
:return: A JSON of the metrics.
"""
# Figure pout the parent process.
# This is important for multi-worker environments:
parent_pid = os.getppid()
parent_process = psutil.Process(parent_pid)
parent_mem_info = parent_process.memory_info()
# Figure out all the children of the parent process:
child_pids = [child.pid for child in parent_process.children()]
child_processes = [psutil.Process(child_pid) for child_pid in child_pids]
child_stats = []
for pid, process in zip(child_pids, child_processes):
mem_info = process.memory_info()
child_stats.append({
"pid": pid,
"mem": round(mem_info.rss / (1024 ** 2), 2)
})
# Construct the response:
metrics_json = {
"parent": {
"pid": parent_pid,
"mem": round(parent_mem_info.rss / (1024 ** 2), 2)
},
"children": child_stats,
"unit": {
"mem": "MB"
},
"ts": date_time.get_current_ist_date_time(as_string = True)
}
# Done here:
return metrics_json
# ---------------------------------------------------------------------------------------------------------------------
@app.route(f"/{MODULE_BASE}/debug/<action>", methods = ["POST", "GET"])
async def change_debug(action):
"""
Enable or disable debugging for the entire microservice.
WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS.
:param action: "enable" to allow debugging on the terminal, or "disable".
:return: "enabled"/"disabled" if successful, else "ok"
"""
# Enable or disable debugging only if the password matches:
action = action.lower()
if action == "enable": current_app.printer.enable()
elif action == "disable": current_app.printer.disable()
return "ok"
# ---------------------------------------------------------------------------------------------------------------------
@app.route(f"/{MODULE_BASE}/maintenance/<action>", methods = ["POST", "GET"])
async def change_maintenance(action):
"""
Enable or disable debugging for the entire microservice.
WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS.
:param action: "enable" to stop taking new requests on the API, or "disable".
:return: "enabled"/"disabled" if successful, else "ok"
"""
# Enable or disable debugging only if the password matches:
action = action.lower()
if action == "enable":
current_app.is_under_maintenance = True
os.environ["IS_UNDER_MAINTENANCE"] = "True"
elif action == "disable":
current_app.is_under_maintenance = False
os.environ["IS_UNDER_MAINTENANCE"] = "False"
return "ok"
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
# To get args from the terminal:
import argparse
# To run the ASGI:
import uvicorn
from multiprocessing import freeze_support
# Get the config from the command-line:
parser = argparse.ArgumentParser(description = f"Microservice for '{MODULE_BASE}' API.")
parser.add_argument(
"--workers",
type = int,
help = "The no. of threads to spin up for this instance!",
default = 2
)
parser.add_argument(
"--host",
type = str,
help = "The host for the app. e.g.: '0.0.0.0' or '127.0.0.1'.",
default = "127.0.0.1"
)
parser.add_argument(
"--port",
type = int,
help = "The port no. to bind the app to.",
default = 8080
)
parser.add_argument(
"--script-id",
type = str,
help = "The id of this script (will affect the loaded config)."
)
parser.add_argument(
"--debug",
action = "store_true",
help = "Whether, or not, you want to see debugging messages in the terminal.",
default = False
)
args = parser.parse_args()
# Note down the config;
os.environ["SCRIPT_ID"] = args.script_id
os.environ["DEBUG"] = str(args.debug)
# Run the gateway:
freeze_support()
uvicorn.run(
app = "main:app",
workers = args.workers,
host = args.host,
port = args.port
)