""" AUTHOR: Khushal P Soonderji DATE: Tuesday, 8th Oct., 2024 OBJECTIVE: This is the central location for the Quart module. We define the app here, and import and attach all blueprints here. REFERENCES: N/A DOWNLOADS: N/A """ # ***************************************************************************************************************** # ***** **** # *** IMPORT *** # ***** **** # ***************************************************************************************************************** # To make sibling directories accessible for imports: import sys sys.path.append(".") sys.path.append("..") # For system level activities: import gc import os import psutil # For using Quart: from quart import Quart, request, current_app, redirect from quart_cors import cors # Common: from shared import constants # My utils: from utils_v2.string import json from utils_v2.api import async_quart from utils_v2.date_time import date_time from utils_v2.database.async_mongo_v2 import AsyncMongo from utils_v2.cache.async_redis_cache_v3 import AsyncRedisCache from utils_v2.api.async_quart import ( set_api_version, read_input, log_request_to_mongo, should_not_be_under_maintenance, only_whitelisted_ips, limit_rate, validate_input, handle_cancelled_request ) # For debugging: from icecream import IceCreamDebugger # Controllers: from controllers.servers.server import CoreServerController # All the blueprints: from api_v2.blueprints.cred_and_data.blueprint import cred_and_data_bp from api_v2.blueprints.logs.blueprint import logs_bp from api_v2.blueprints.otp.blueprint import otp_bp from api_v2.blueprints.servers.blueprint import servers_bp # ***************************************************************************************************************** # ***** **** # *** MACROS / ONE-TIME INIT *** # ***** **** # ***************************************************************************************************************** # Quart related: MODULE_BASE = "internal" APP_VERSION = "2.2.0" # ***************************************************************************************************************** # ***** **** # *** VARIABLES *** # ***** **** # ***************************************************************************************************************** # The Quart app: app = Quart(__name__) app = cors(app) app.register_blueprint(cred_and_data_bp, url_prefix = f"/{MODULE_BASE}") app.register_blueprint(logs_bp, url_prefix = f"/{MODULE_BASE}/logs") app.register_blueprint(otp_bp, url_prefix = f"/{MODULE_BASE}/otp") app.register_blueprint(servers_bp, url_prefix = f"/{MODULE_BASE}/servers") # ***************************************************************************************************************** # ***** **** # *** FUNCTIONS *** # ***** **** # ***************************************************************************************************************** @app.before_serving @set_api_version(api_version = APP_VERSION) @log_request_to_mongo( attr_name = "mongo", log_type = MODULE_BASE, operation = "apiStart", log_input = True, log_output = True ) async def app_startup(**kwargs): """ To initialize the variables that you would like to use in this module. WARNING: ALL VARIABLES WILL BE INITIALIZED 'n' NUMBER OF TIMES, WHERE 'n' IS THE COUNT OF WORKERS DEPLOYED. SO, IF YOU WANT TO CONNECT TO A DATABASE AND YOU ALLOW A POOL-SIZE OF 10 AND IF YOU DEPLOY 4 WORKERS, YOU WILL END UP WITH 40 CONNECTIONS TO THE DATABASE. :return: None. """ # ┳ • • ┓• # ┃┏┓┓╋┓┏┓┃┓┓┏┓ # ┻┛┗┗┗┗┗┻┗┗┗┗ # Safe-halt mechanism for upgrades (for a single-worker run): current_app.is_under_maintenance = False # ┳┓ ┓ • # ┃┃┏┓┣┓┓┏┏┓┏┓┓┏┓┏┓ # ┻┛┗ ┗┛┗┻┗┫┗┫┗┛┗┗┫ # ┛ ┛ ┛ # Debugging: enable_debugging = True if os.environ["DEBUG"].strip().lower() == "true" else False current_app.printer = IceCreamDebugger(prefix = f"{MODULE_BASE} (Q) | ", includeContext = True) if not enable_debugging: current_app.printer.disable() current_app.printer("initializing worker.") # ┳┳┓ ┳┓┳┓ # ┃┃┃┏┓┏┓┏┓┏┓┃┃┣┫ # ┛ ┗┗┛┛┗┗┫┗┛┻┛┻┛ # ┛ # To connect to Mongo: current_app.mongo = AsyncMongo( connection_string = constants.MONGO_DATA_CONNECTION_STRING, database_name = constants.MONGO_DATA_DATABASE_NAME, max_connections = 10, debug = True, debug_only_errors = True ) await current_app.mongo.connect() current_app.logs_mongo = current_app.mongo current_app.printer("MongoDB ready.") # ┏┓ ┓ ┓ ┳┓ # ┃ ┏┓┏┓┏┫ ┏┓┏┓┏┫ ┃┃┏┓╋┏┓ # ┗┛┛ ┗ ┗┻ ┗┻┛┗┗┻ ┻┛┗┻┗┗┻ # Get the script data: script_id = os.environ["SCRIPT_ID"] script_cred = (await current_app.mongo.find_one( collection = "_scriptCred", filter = {"scriptId": script_id} ))["content"] current_app.script_data = (await current_app.mongo.find_one( collection = "_scriptData", filter = {"scriptId": script_id} ))["content"] current_app.printer("Cred and Data ready.") # ┳┓ ┓• ┏┓ ┓ # ┣┫┏┓┏┫┓┏ ┃ ┏┓┏┣┓┏┓ # ┛┗┗ ┗┻┗┛ ┗┛┗┻┗┛┗┗ current_app.otp_redis = AsyncRedisCache( connection_string = script_cred["redisCache"]["general"]["sentinelJson"], debug = enable_debugging, debug_prefix = "OTP Cache | " ) current_app.printer("Redis ready.") # ┏┓ ┓┓ # ┃ ┏┓┏┓╋┏┓┏┓┃┃┏┓┏┓┏ # ┗┛┗┛┛┗┗┛ ┗┛┗┗┗ ┛ ┛ current_app.server_controller = CoreServerController() current_app.printer("Controllers ready.") # ┳┳┓• # ┃┃┃┓┏┏ # ┛ ┗┗┛┗• # Pick the important stuff: current_app.whitelisted_ips = current_app.script_data["whitelistedIps"] # Done here! print("Worker ready!") # --------------------------------------------------------------------------------------------------------------------- @app.after_serving @set_api_version(api_version = APP_VERSION) @log_request_to_mongo( attr_name = "mongo", log_type = MODULE_BASE, operation = "apiStop", log_input = True, log_output = True ) async def app_shutdown(**kwargs): """ This is called when "app.shutdown()" is called. :return: None. """ message = "Shutting down..." current_app.printer(message) # --------------------------------------------------------------------------------------------------------------------- @app.before_request async def enforce_https(): """ Here we enforce HTTPS (secure) requests and effectively reject unsecure requests. We make exemptions for local network requests for regular testing. :return: A redirect to the secure version if the incoming request is not secure. """ # If the connection is not secure: if request.scheme != "https": # Check if it is a local IP. If not, redirect: exempt_ips = ["0.0.0.0", "127.0.0.1"] if not ( request.remote_addr.startswith("192.168.") or request.remote_addr in exempt_ips ): return redirect(request.url.replace("http", "https")) # --------------------------------------------------------------------------------------------------------------------- @app.route(f"/", methods = ["GET", "POST"]) @app.route(f"/{MODULE_BASE}", methods = ["GET", "POST"]) async def root(): """ To check if the service is running or not. Use this to monitor the service from your "watchman" script. :return: only "ok" """ return "ok" # --------------------------------------------------------------------------------------------------------------------- @app.route(f"/metrics/memory", methods = ["GET", "POST"]) @app.route(f"/{MODULE_BASE}/metrics/memory", methods = ["GET", "POST"]) async def metrics(): """ TO measure the metrics of the app. :return: A JSON of the metrics. """ # Figure pout the parent process. # This is important for multi-worker environments: parent_pid = os.getppid() parent_process = psutil.Process(parent_pid) parent_mem_info = parent_process.memory_info() # Figure out all the children of the parent process: child_pids = [child.pid for child in parent_process.children()] child_processes = [psutil.Process(child_pid) for child_pid in child_pids] child_stats = [] for pid, process in zip(child_pids, child_processes): mem_info = process.memory_info() child_stats.append({ "pid": pid, "mem": round(mem_info.rss / (1024 ** 2), 2) }) # Construct the response: metrics_json = { "parent": { "pid": parent_pid, "mem": round(parent_mem_info.rss / (1024 ** 2), 2) }, "children": child_stats, "unit": { "mem": "MB" }, "ts": date_time.get_current_ist_date_time(as_string = True) } # Done here: return metrics_json # --------------------------------------------------------------------------------------------------------------------- @app.route(f"/{MODULE_BASE}/debug/", methods = ["POST", "GET"]) async def change_debug(action): """ Enable or disable debugging for the entire microservice. WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS. :param action: "enable" to allow debugging on the terminal, or "disable". :return: "enabled"/"disabled" if successful, else "ok" """ # Enable or disable debugging only if the password matches: action = action.lower() if action == "enable": current_app.printer.enable() elif action == "disable": current_app.printer.disable() return "ok" # --------------------------------------------------------------------------------------------------------------------- @app.route(f"/{MODULE_BASE}/maintenance/", methods = ["POST", "GET"]) async def change_maintenance(action): """ Enable or disable debugging for the entire microservice. WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS. :param action: "enable" to stop taking new requests on the API, or "disable". :return: "enabled"/"disabled" if successful, else "ok" """ # Enable or disable debugging only if the password matches: action = action.lower() if action == "enable": current_app.is_under_maintenance = True os.environ["IS_UNDER_MAINTENANCE"] = "True" elif action == "disable": current_app.is_under_maintenance = False os.environ["IS_UNDER_MAINTENANCE"] = "False" return "ok" # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** # ***** **** # ***************************************************************************************************************** if __name__ == "__main__": # To get args from the terminal: import argparse # To run the ASGI: import uvicorn from multiprocessing import freeze_support # Get the config from the command-line: parser = argparse.ArgumentParser(description = f"Microservice for '{MODULE_BASE}' API.") parser.add_argument( "--workers", type = int, help = "The no. of threads to spin up for this instance!", default = 2 ) parser.add_argument( "--host", type = str, help = "The host for the app. e.g.: '0.0.0.0' or '127.0.0.1'.", default = "127.0.0.1" ) parser.add_argument( "--port", type = int, help = "The port no. to bind the app to.", default = 8080 ) parser.add_argument( "--script-id", type = str, help = "The id of this script (will affect the loaded config)." ) parser.add_argument( "--debug", action = "store_true", help = "Whether, or not, you want to see debugging messages in the terminal.", default = False ) args = parser.parse_args() # Note down the config; os.environ["SCRIPT_ID"] = args.script_id os.environ["DEBUG"] = str(args.debug) # Run the gateway: freeze_support() uvicorn.run( app = "main:app", workers = args.workers, host = args.host, port = args.port )