Resetting utils subtree.
This commit is contained in:
+370
@@ -0,0 +1,370 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Tuesday, 8th Oct., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
This is the central location for the Quart module.
|
||||
We define the app here, and import and attach all blueprints here.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For system level activities:
|
||||
import gc
|
||||
import os
|
||||
import psutil
|
||||
|
||||
# For using Quart:
|
||||
from quart import Quart, request, current_app, redirect
|
||||
from quart_cors import cors
|
||||
|
||||
# Common:
|
||||
from shared import constants
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.api import async_quart
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
from utils_v2.api.async_quart import (
|
||||
set_api_version,
|
||||
read_input,
|
||||
log_request_to_mongo,
|
||||
should_not_be_under_maintenance,
|
||||
only_whitelisted_ips,
|
||||
limit_rate,
|
||||
validate_input,
|
||||
handle_cancelled_request
|
||||
)
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
# All the blueprints:
|
||||
from api.cred_data.blueprint import cred_and_data_bp
|
||||
from api.logs.blueprint import logs_bp
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Quart related:
|
||||
MODULE_BASE = "internal"
|
||||
APP_VERSION = "2.0.0"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# The Quart app:
|
||||
app = Quart(__name__)
|
||||
app = cors(app)
|
||||
app.register_blueprint(cred_and_data_bp, url_prefix = f"/{MODULE_BASE}")
|
||||
app.register_blueprint(logs_bp, url_prefix = f"/{MODULE_BASE}/logs")
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
@app.before_serving
|
||||
@set_api_version(api_version = APP_VERSION)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = MODULE_BASE,
|
||||
operation = "apiStart",
|
||||
log_input = True,
|
||||
log_output = True
|
||||
)
|
||||
async def app_startup(**kwargs):
|
||||
|
||||
"""
|
||||
To initialize the variables that you would like to use in this module.
|
||||
WARNING: ALL VARIABLES WILL BE INITIALIZED 'n' NUMBER OF TIMES, WHERE 'n' IS THE COUNT OF WORKERS DEPLOYED.
|
||||
SO, IF YOU WANT TO CONNECT TO A DATABASE AND YOU ALLOW A POOL-SIZE OF 10 AND IF YOU DEPLOY 4 WORKERS, YOU WILL END
|
||||
UP WITH 40 CONNECTIONS TO THE DATABASE.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# Safe-halt mechanism for upgrades (for a single-worker run):
|
||||
current_app.is_under_maintenance = False
|
||||
|
||||
# Debugging:
|
||||
current_app.printer = IceCreamDebugger(prefix = f"{MODULE_BASE} (Q) | ", includeContext = True)
|
||||
if os.environ["DEBUG"] == "True": current_app.printer.disable()
|
||||
|
||||
# To connect to Mongo:
|
||||
current_app.mongo = AsyncMongo(
|
||||
connection_string = constants.MONGO_DATA_CONNECTION_STRING,
|
||||
database_name = constants.MONGO_DATA_DATABASE_NAME,
|
||||
max_connections = 10,
|
||||
debug = True,
|
||||
debug_only_errors = True
|
||||
)
|
||||
await current_app.mongo.connect()
|
||||
|
||||
# Get the script data:
|
||||
script_id = os.environ["SCRIPT_ID"]
|
||||
current_app.script_data = (await current_app.mongo.find_one(
|
||||
collection = "_scriptData",
|
||||
filter = {"scriptId": script_id}
|
||||
))["content"]
|
||||
|
||||
# Pick the important stuff:
|
||||
current_app.whitelisted_ips = current_app.script_data["whitelistedIps"]
|
||||
|
||||
# Done here!
|
||||
print("Worker ready!")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.after_serving
|
||||
@set_api_version(api_version = APP_VERSION)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = MODULE_BASE,
|
||||
operation = "apiStop",
|
||||
log_input = True,
|
||||
log_output = True
|
||||
)
|
||||
async def app_shutdown(**kwargs):
|
||||
|
||||
"""
|
||||
This is called when "app.shutdown()" is called.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
message = "Shutting down..."
|
||||
current_app.printer(message)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.before_request
|
||||
async def enforce_https():
|
||||
|
||||
"""
|
||||
Here we enforce HTTPS (secure) requests and effectively reject unsecure requests. We make exemptions for local
|
||||
network requests for regular testing.
|
||||
:return: A redirect to the secure version if the incoming request is not secure.
|
||||
"""
|
||||
|
||||
# If the connection is not secure:
|
||||
if request.scheme != "https":
|
||||
|
||||
# Check if it is a local IP. If not, redirect:
|
||||
exempt_ips = ["0.0.0.0", "127.0.0.1"]
|
||||
if not (
|
||||
request.remote_addr.startswith("192.168.") or
|
||||
request.remote_addr in exempt_ips
|
||||
): return redirect(request.url.replace("http", "https"))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/", methods = ["GET", "POST"])
|
||||
@app.route(f"/{MODULE_BASE}", methods = ["GET", "POST"])
|
||||
async def root():
|
||||
|
||||
"""
|
||||
To check if the service is running or not.
|
||||
Use this to monitor the service from your "watchman" script.
|
||||
:return: only "ok"
|
||||
"""
|
||||
|
||||
return "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/metrics/memory", methods = ["GET", "POST"])
|
||||
@app.route(f"/{MODULE_BASE}/metrics/memory", methods = ["GET", "POST"])
|
||||
async def metrics():
|
||||
|
||||
"""
|
||||
TO measure the metrics of the app.
|
||||
:return: A JSON of the metrics.
|
||||
"""
|
||||
|
||||
# Figure pout the parent process.
|
||||
# This is important for multi-worker environments:
|
||||
parent_pid = os.getppid()
|
||||
parent_process = psutil.Process(parent_pid)
|
||||
parent_mem_info = parent_process.memory_info()
|
||||
|
||||
# Figure out all the children of the parent process:
|
||||
child_pids = [child.pid for child in parent_process.children()]
|
||||
child_processes = [psutil.Process(child_pid) for child_pid in child_pids]
|
||||
child_stats = []
|
||||
for pid, process in zip(child_pids, child_processes):
|
||||
mem_info = process.memory_info()
|
||||
child_stats.append({
|
||||
"pid": pid,
|
||||
"mem": round(mem_info.rss / (1024 ** 2), 2)
|
||||
})
|
||||
|
||||
# Construct the response:
|
||||
metrics_json = {
|
||||
"parent": {
|
||||
"pid": parent_pid,
|
||||
"mem": round(parent_mem_info.rss / (1024 ** 2), 2)
|
||||
},
|
||||
"children": child_stats,
|
||||
"unit": {
|
||||
"mem": "MB"
|
||||
},
|
||||
"ts": date_time.get_current_ist_date_time(as_string = True)
|
||||
}
|
||||
|
||||
# Done here:
|
||||
return metrics_json
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/{MODULE_BASE}/debug/<action>", methods = ["POST", "GET"])
|
||||
async def change_debug(action):
|
||||
|
||||
"""
|
||||
Enable or disable debugging for the entire microservice.
|
||||
WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS.
|
||||
:param action: "enable" to allow debugging on the terminal, or "disable".
|
||||
:return: "enabled"/"disabled" if successful, else "ok"
|
||||
"""
|
||||
|
||||
# Enable or disable debugging only if the password matches:
|
||||
action = action.lower()
|
||||
if action == "enable": current_app.printer.enable()
|
||||
elif action == "disable": current_app.printer.disable()
|
||||
return "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/{MODULE_BASE}/maintenance/<action>", methods = ["POST", "GET"])
|
||||
async def change_maintenance(action):
|
||||
|
||||
"""
|
||||
Enable or disable debugging for the entire microservice.
|
||||
WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS.
|
||||
:param action: "enable" to stop taking new requests on the API, or "disable".
|
||||
:return: "enabled"/"disabled" if successful, else "ok"
|
||||
"""
|
||||
|
||||
# Enable or disable debugging only if the password matches:
|
||||
action = action.lower()
|
||||
if action == "enable":
|
||||
current_app.is_under_maintenance = True
|
||||
os.environ["IS_UNDER_MAINTENANCE"] = "True"
|
||||
elif action == "disable":
|
||||
current_app.is_under_maintenance = False
|
||||
os.environ["IS_UNDER_MAINTENANCE"] = "False"
|
||||
return "ok"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# To get args from the terminal:
|
||||
import argparse
|
||||
|
||||
# To run the ASGI:
|
||||
import uvicorn
|
||||
from multiprocessing import freeze_support
|
||||
|
||||
# Get the config from the command-line:
|
||||
parser = argparse.ArgumentParser(description = f"Microservice for '{MODULE_BASE}' API.")
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type = int,
|
||||
help = "The no. of threads to spin up for this instance!",
|
||||
default = 2
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type = str,
|
||||
help = "The host for the app. e.g.: '0.0.0.0' or '127.0.0.1'.",
|
||||
default = "127.0.0.1"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type = int,
|
||||
help = "The port no. to bind the app to.",
|
||||
default = 8080
|
||||
)
|
||||
parser.add_argument(
|
||||
"--script-id",
|
||||
type = str,
|
||||
help = "The id of this script (will affect the loaded config)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
action = "store_true",
|
||||
help = "Whether, or not, you want to see debugging messages in the terminal.",
|
||||
default = False
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Note down the config;
|
||||
os.environ["SCRIPT_ID"] = args.script_id
|
||||
os.environ["DEBUG"] = str(args.debug)
|
||||
|
||||
# Run the gateway:
|
||||
freeze_support()
|
||||
uvicorn.run(
|
||||
app = "main:app",
|
||||
workers = args.workers,
|
||||
host = args.host,
|
||||
port = args.port
|
||||
)
|
||||
Reference in New Issue
Block a user