diff --git a/README_GIT.md b/README_GIT.md new file mode 100644 index 0000000..8cd438a --- /dev/null +++ b/README_GIT.md @@ -0,0 +1,83 @@ + +# GIT CHEATSHEET +## By Sharvil Sir 🙏 (20240927) +Perform these steps on creating a new project. + +_**ASSUMPTION:** You are already in your project directory in the command line terminal._ + +--- + +### STEP 0. +#### Initialize Git in your new project: +Do this when creating a new project directory. +Do this **ONLY ONCE**. +```commandline +git init +``` + +### STEP 1. +#### Create a `.gitignore` file: +Add files/directories in it which you don't want to sync to git. +Update this as frequently as your project needs you to. Start with this: +```commandline +/.venv/ +/.idea/ +**/__pycache__/ +*.pem +``` + +### STEP 2. +#### Create a repository on Gitea (web UI): +Do this **ONLY ONCE**. +Open the following URL: https://wtt.ditscentre.in/ and sign in. +- Ensure that the owner is `ditscentre` (img 0). +- Give your repository a name. Preferably keep it the same as the project name that you made locally (img 0). +- Type in a brief description of your project (img 0). + +![img 0](https://nexcom.ditscentre.in/utils/files/small/download/66f66a54196705ee2f25d28e) + +- Ensure that the default branch is `master` (img 1). +- Create the repository (img 1). + +![img 1](https://nexcom.ditscentre.in/utils/files/small/download/66f66a54196705ee2f25d28f) + +### STEP 3. +#### Add files and directories to the staging area: +The following command adds everything to the staging area. +The '.' is important, it refers to the current directory. +```commandline +git add . +``` + +### STEP 4. +#### Commit the changes to local git: +This step commits all the added (staged) changes to the local git instance with the provided comment. +```commandline +git commit -m "you comment here..." +``` + +### STEP 5. +#### We push the local commits to the repository: +The format of the command is: +``` +git push -u +``` + +--- + +## For Python Devs: How to add a common subtree (like `utils_v2`) + +The format of the command is: +``` +git subtree add --prefix= --squash +``` +Before you start: ensure that you have committed all pending changes. Then run the following command **ONLY ONCE**: +```commandline +git subtree add --prefix=utils_v2 https://wtt.ditscentre.in/ditscentre/utils_v2.git master --squash +``` +This will create a new directory named `utils_v2` in your project. +**Do NOT change the files in it.** Just import and use them in your scripts. +When you need to get the latest updates to these files, you must run a modification of the previous command: +```commandline +git subtree pull --prefix=utils_v2 https://wtt.ditscentre.in/ditscentre/utils_v2.git master --squash +``` \ No newline at end of file diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/cred_data/__init__.py b/api/cred_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/cred_data/blueprint.py b/api/cred_data/blueprint.py new file mode 100644 index 0000000..cf2ddb3 --- /dev/null +++ b/api/cred_data/blueprint.py @@ -0,0 +1,406 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Wednesday, 28th Aug., 2024 + + OBJECTIVE: + + To be able to fetch setup credentials and data for any project. + This could include things like default values, URLs to assets, etc. + While 'data' and 'cred' can have anything held in them, the idea behind giving two services is for the user of + this service to be able to organise his setup variables. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For using Quart: +from quart import Blueprint, current_app + +# My utils: +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.response import ResponseModel +from utils_v2.api.async_quart import ( + read_input, + log_request_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input +) + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Related to Quart: +cred_and_data_bp = Blueprint("int_cnd", __name__) +get_api_version = "2.0.0" +set_api_version = "2.0.0" +update_api_version = "2.0.0" +delete_api_version = "2.0.0" + +# related to the operations of this blueprint: +JSON_TYPE_INFO = { + "cred": { + "collection": "scriptCred" + }, + "data": { + "collection": "scriptData" + } +} + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@cred_and_data_bp.record_once +def init(blueprint_setup_state): + + # This gets called when the blueprint is registered. + # Consider this to be a one-time setup for the whole blueprint: + pass + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//set", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "set", + api_version = set_api_version, + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def set_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To set the credentials for a particular script. If the document exists, it will be overwritten. If the document + doesn't exist, it will be created. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart_utils.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart_utils.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Pre-process the inbound data: + inbound_data = inbound_data or {} + inbound_data["scriptId"] = inbound_headers["X-Script-Id"] + + # Make an attempt to set the credentials: + success = await current_app.mongo.replace_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + replacement = inbound_data, + upsert = True + ) + + # Return the response: + if success: return ResponseModel(api_version = set_api_version, status_code = StatusCodes.OK) + else: return ResponseModel(api_version = set_api_version, status_code = StatusCodes.FAILED) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = set_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//get", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "get", + api_version = get_api_version, + log_input = True, + log_output = False +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def get_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To retrieve the credentials stored for a specific script. The script's id can be anything set by the programmers. + The idea is to have only the script's id stored in the script, and everything else is fetched from the database. + This means that we get to store and update everything from one central location. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to retrieve the credentials: + cred_json = await current_app.mongo.find_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + projection = {"_id": False, "scriptId": False} + ) + + # In case no result was found: + if cred_json is None: + return ResponseModel( + api_version = get_api_version, + status_code = StatusCodes.FAILED, + message = "invalid script id" + ) + + # Successfully retrieved: + return ResponseModel( + api_version = get_api_version, + status_code = StatusCodes.OK, + data = cred_json + ) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = get_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//update", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "update", + api_version = update_api_version, + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def update_cred( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To update values of certain fields for a credentials document. It only updates existing values, does NOT add a new + document if the document doesn't already exist. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart_utils.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart_utils.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Pre-process the inbound data: + inbound_data = inbound_data or {} + + # Prepare the update JSON: + update_json = {} + if inbound_data.get("unset"): update_json["$unset"] = current_app.mongo.dict_to_dot_notation(inbound_data["unset"]) + if inbound_data.get("set"): update_json["$set"] = current_app.mongo.dict_to_dot_notation(inbound_data["set"]) + + # Make an attempt to set the credentials: + success = await current_app.mongo.update_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + update = update_json, + upsert = False + ) + + # Return the response: + if success: return ResponseModel(api_version = update_api_version, status_code = StatusCodes.OK) + else: return ResponseModel(api_version = update_api_version, status_code = StatusCodes.FAILED) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = update_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//delete", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "delete", + api_version = delete_api_version, + log_input = True, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def delete_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To delete a document for a particular script id. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to set the credentials: + success = await current_app.mongo.delete_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + ) + + # Return the response: + if success: return ResponseModel(api_version = delete_api_version, status_code = StatusCodes.OK) + else: return ResponseModel(api_version = delete_api_version, status_code = StatusCodes.FAILED) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = delete_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/api/logs/__init__.py b/api/logs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/logs/blueprint.py b/api/logs/blueprint.py new file mode 100644 index 0000000..9fa4d30 --- /dev/null +++ b/api/logs/blueprint.py @@ -0,0 +1,381 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Created: Wednesday, 28th Aug., 2024 + Updated: Tuesday, 8th Oct. 2024 + + OBJECTIVE: + + To be able to fetch setup credentials and data for any project. This could include things like default values, + URLs to assets, etc. While 'data' and 'cred' can be used interchangeably, the idea behind giving two services is + for the user of this service to be able to organise his setup variables. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For using Quart: +from quart import Blueprint, current_app + +# My utils: +from utils_v2.string import json +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.response import ResponseModel +from utils_v2.api.async_quart import ( + set_api_version, + read_input, + log_request_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input, + handle_cancelled_request +) + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Related to Quart: +cred_and_data_bp = Blueprint("int_cnd", __name__) + +# related to the operations of this blueprint: +JSON_TYPE_INFO = { + "cred": { + "collection": "_scriptCred" + }, + "data": { + "collection": "_scriptData" + } +} + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@cred_and_data_bp.record_once +def init(blueprint_setup_state): + + # This gets called when the blueprint is registered. + # Consider this to be a one-time setup for the whole blueprint: + pass + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//set", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "set", + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id", "X-Script-Desc"]) +@handle_cancelled_request() +async def set_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To set the credentials for a particular script. If the document exists, it will be overwritten. If the document + doesn't exist, it will be created. Ideally use this for only the first time setup. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Construct the document: + description = inbound_headers.get("X-Script-Desc", "") + if len(description) > 200: description = description[:200] + document = { + "scriptId": inbound_headers["X-Script-Id"], + "desc": description, + "content": inbound_data + } + + # Make an attempt to set the credentials: + success = await current_app.mongo.replace_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + replacement = document, + upsert = True, + raise_exception = True + ) + + # Return the response: + if success: return ResponseModel(status_code = StatusCodes.OK) + else: return ResponseModel(status_code = StatusCodes.FAILED) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//get", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "get", + log_input = True, + log_output = False +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@handle_cancelled_request() +async def get_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To retrieve the credentials stored for a specific script. The script's id can be anything set by the programmers. + The idea is to have only the script's id stored in the script, and everything else is fetched from the database. + This means that we get to store and update everything from one central location. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to retrieve the credentials: + cred_json = await current_app.mongo.find_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + projection = {"_id": False, "scriptId": False}, + raise_exception = True + ) + + # In case no result was found: + if cred_json is None: + return ResponseModel( + status_code = StatusCodes.FAILED, + message = "invalid script id" + ) + + # Successfully retrieved: + return ResponseModel( + status_code = StatusCodes.OK, + data = cred_json["content"] + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//update", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "update", + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@handle_cancelled_request() +async def update_cred( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To update values of certain fields for a credentials document. It only updates existing values, does NOT add a new + document if the document doesn't already exist. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Pre-process the inbound data: + inbound_data = inbound_data or {} + + # Prepare the update JSON. Pre-process the fields to set and unset. + # Our actual data/cred are held inside a field called "content", so we must wrap the request in that: + update_json = {} + if inbound_data.get("unset"): + update_json["$unset"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["unset"]}) + if inbound_data.get("set"): + update_json["$set"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["set"]}) + + # Make an attempt to set the credentials: + success = await current_app.mongo.update_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + update = update_json, + upsert = False, + raise_exception = True + ) + + # Return the response: + if success: return ResponseModel(status_code = StatusCodes.OK) + else: return ResponseModel(status_code = StatusCodes.FAILED) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//delete", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "delete", + log_input = True, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def delete_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To delete a document for a particular script id. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to set the credentials: + success = await current_app.mongo.delete_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + ) + + # Return the response: + if success: return ResponseModel(status_code = StatusCodes.OK) + else: return ResponseModel(status_code = StatusCodes.FAILED) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000..0edcc28 --- /dev/null +++ b/api/main.py @@ -0,0 +1,301 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Wednesday, 28th Aug., 2024 + + OBJECTIVE: + + This is the central location for the Quart module. + We define the app here, and import and attach all blueprints here. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For system level activities: +import gc +import os + +# For using Quart: +from quart import Quart, request, current_app +from quart_cors import cors + +# To make REST-API calls: +import httpx + +# My utils: +from utils_v2.string import json +from utils_v2.api import async_quart +from utils_v2.database.async_mongo_v2 import AsyncMongo +from utils_v2.api.async_quart import ( + read_input, + log_request_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input +) + +# For debugging: +from icecream import IceCreamDebugger + +# Other blueprints: +from api.llm.chat_completion import llm_chat_bp + +# LangChain-related: +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.messages import HumanMessage, AIMessage, SystemMessage +from langchain_openai import ChatOpenAI + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Quart related: +MODULE_BASE = "ai" +APP_VERSION = "1.0.0" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# The Quart app: +app = Quart(__name__) +app = cors(app) +app.register_blueprint(llm_chat_bp, url_prefix = f"/{MODULE_BASE}/llm") + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@app.before_serving +@log_request_to_mongo( + attr_name = "mongo", + log_type = MODULE_BASE, + operation = "apiStart", + api_version = APP_VERSION, + log_input = True, + log_output = True +) +async def app_startup(): + + """ + To initialize the variables that you would like to use in this module. + WARNING: ALL VARIABLES WILL BE INITIALIZED 'n' NUMBER OF TIMES, WHERE 'n' IS THE COUNT OF WORKERS DEPLOYED. + SO, IF YOU WANT TO CONNECT TO A DATABASE AND YOU ALLOW A POOL-SIZE OF 10 AND IF YOU DEPLOY 4 WORKERS, YOU WILL END + UP WITH 40 CONNECTIONS TO THE DATABASE. + :return: None. + """ + + # Safe-halt mechanism for upgrades (for a single-worker run): + current_app.is_under_maintenance = False + + # Debugging: + current_app.printer = IceCreamDebugger(prefix = f"{MODULE_BASE} (Q) | ", includeContext = True) + if os.environ["DEBUG"] == "True": current_app.printer.disable() + + # To make API calls: + max_connections = 5 + limits = httpx.Limits( + max_keepalive_connections = max_connections, + max_connections = max_connections, + keepalive_expiry = 3600 + ) + current_app.http_client = httpx.AsyncClient(limits = limits) + + # Get the credentials and data for this script: + script_id = os.environ.get("SCRIPT_ID") + response = await current_app.http_client.get( + url = r"https://nexcom.ditscentre.in/internal/cred/get", + headers = {"X-Script-Id": script_id} + ) + script_cred = response.json().get("data") + response = await current_app.http_client.get( + url = r"https://nexcom.ditscentre.in/internal/data/get", + headers = {"X-Script-Id": script_id} + ) + current_app.script_data = response.json().get("data") + + # To connect to Mongo: + current_app.mongo = AsyncMongo( + connection_string = script_cred["mongoDb"]["dataDb"]["connectionString"], + database_name = script_cred["mongoDb"]["dataDb"]["dbName"], + max_connections = script_cred["mongoDb"]["dataDb"]["poolSize"], + debug = True if os.environ["DEBUG"] == "True" else False + ) + + # Remove unwanted/sensitive variables from RAM: + del script_cred + gc.collect() + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.after_serving +@log_request_to_mongo( + attr_name = "mongo", + log_type = MODULE_BASE, + operation = "apiStop", + api_version = APP_VERSION, + log_input = True, + log_output = True +) +async def app_shutdown(): + + """ + This is called when "app.shutdown()" is called. + :return: None. + """ + + message = "Shutting down..." + current_app.printer(message) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.route(f"/", methods = ["GET", "POST"]) +@app.route(f"/{MODULE_BASE}", methods = ["GET", "POST"]) +async def root(): + + """ + To check if the service is running or not. + Use this to monitor the service from your "watchman" script. + :return: only "ok" + """ + + return "ok" + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.route(f"/{MODULE_BASE}/debug/", methods = ["POST", "GET"]) +async def change_debug(action): + + """ + Enable or disable debugging for the entire microservice. + WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS. + :param action: "enable" to allow debugging on the terminal, or "disable". + :return: "enabled"/"disabled" if successful, else "ok" + """ + + # Enable or disable debugging only if the password matches: + action = action.lower() + if action == "enable": current_app.printer.enable() + elif action == "disable": current_app.printer.disable() + return "ok" + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.route(f"/{MODULE_BASE}/maintenance/", methods = ["POST", "GET"]) +async def change_maintenance(action): + + """ + Enable or disable debugging for the entire microservice. + WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS. + :param action: "enable" to stop taking new requests on the API, or "disable". + :return: "enabled"/"disabled" if successful, else "ok" + """ + + # Enable or disable debugging only if the password matches: + action = action.lower() + if action == "enable": current_app.is_under_maintenance = True + elif action == "disable": current_app.is_under_maintenance = False + return "ok" + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + # To get args from the terminal: + import argparse + + # To run the ASGI: + import uvicorn + from multiprocessing import freeze_support + + # Get the config from the command-line: + parser = argparse.ArgumentParser(description = f"Microservice for '{MODULE_BASE}' API.") + parser.add_argument( + "--workers", + type = int, + help = "The no. of threads to spin up for this instance!", + default = 2 + ) + parser.add_argument( + "--host", + type = str, + help = "The host for the app. e.g.: '0.0.0.0' or '127.0.0.1'.", + default = "127.0.0.1" + ) + parser.add_argument( + "--script-id", + type = str, + help = "The id of this script (will affect the loaded config)." + ) + parser.add_argument( + "--debug", + action = "store_true", + help = "Whether, or not, you want to see debugging messages in the terminal.", + default = False + ) + args = parser.parse_args() + + # Note down the config; + os.environ["SCRIPT_ID"] = args.script_id + os.environ["DEBUG"] = str(args.debug) + + # Run the gateway: + freeze_support() + uvicorn.run( + app = "main:app", + workers = args.workers, + host = args.host, + port = 8080 + ) diff --git a/cred/README_CERTS.md b/cred/README_CERTS.md new file mode 100644 index 0000000..b68709a --- /dev/null +++ b/cred/README_CERTS.md @@ -0,0 +1,6 @@ +# INSTRUCTIONS TO RENEW CERTIFICATES +**Date:** 2024-09-18 + +1. Go to [jcdev.ditscentre.in/jcdev](http://jcdev.ditscentre.in/jcdev/) +2. Navigate to the respective folders (e.g., `kafka`, `mongo`, etc.). +3. Download and replace the certificates to ensure they have the same names as those already present in the `cred` folder. diff --git a/cred/mongo/mongo_data_connection_string.txt b/cred/mongo/mongo_data_connection_string.txt new file mode 100644 index 0000000..7865109 --- /dev/null +++ b/cred/mongo/mongo_data_connection_string.txt @@ -0,0 +1 @@ +mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external \ No newline at end of file diff --git a/cred/mongo/mongo_file_connection_string.txt b/cred/mongo/mongo_file_connection_string.txt new file mode 100644 index 0000000..7865109 --- /dev/null +++ b/cred/mongo/mongo_file_connection_string.txt @@ -0,0 +1 @@ +mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external \ No newline at end of file diff --git a/cred/mongo/mongo_keys_connection_string.txt b/cred/mongo/mongo_keys_connection_string.txt new file mode 100644 index 0000000..7865109 --- /dev/null +++ b/cred/mongo/mongo_keys_connection_string.txt @@ -0,0 +1 @@ +mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external \ No newline at end of file diff --git a/shared/__init__.py b/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared/constants.py b/shared/constants.py new file mode 100644 index 0000000..ebfaba2 --- /dev/null +++ b/shared/constants.py @@ -0,0 +1,100 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 17th Aug., 2024 + + OBJECTIVE: + + To hold constant that will be shared throughout the project. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import os + +# My utils: +from utils_v2.system import files + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Directories: +PROJECT_DIRECTORY = files.get_parent_directory(files.get_parent_directory(files.get_cwd())) +CREDENTIALS_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "cred") +MONGO_CREDENTIALS_DIRECTORY = os.path.join(CREDENTIALS_DIRECTORY, "mongo") +MODELS_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "models") + +# Files: +MONGO_KEYS_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_connection_string.txt") +MONGO_KEYS_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_ca.pem") +MONGO_KEYS_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_cert.pem") +MONGO_DATA_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_connection_string.txt") +MONGO_DATA_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_ca.pem") +MONGO_DATA_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_cert.pem") +MONGO_FILE_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_connection_string.txt") +MONGO_FILE_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_ca.pem") +MONGO_FILE_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_cert.pem") + +# MongoDB Connection Strings: +MONGO_KEYS_CONNECTION_STRING = files.read_file( + MONGO_KEYS_CONNECTION_STRING_FILE, + mode = "r" +).format( + mongo_ca = MONGO_KEYS_CA_FILE.replace("/", "%2F"), + mongo_cert = MONGO_KEYS_CERT_FILE.replace("/", "%2F") +) +MONGO_DATA_CONNECTION_STRING = files.read_file( + MONGO_DATA_CONNECTION_STRING_FILE, + mode = "r" +).format( + mongo_ca = MONGO_DATA_CA_FILE.replace("/", "%2F"), + mongo_cert = MONGO_DATA_CERT_FILE.replace("/", "%2F") +) +MONGO_FILE_CONNECTION_STRING = files.read_file( + MONGO_FILE_CONNECTION_STRING_FILE, + mode = "r" +).format( + mongo_ca = MONGO_FILE_CA_FILE.replace("/", "%2F"), + mongo_cert = MONGO_FILE_CERT_FILE.replace("/", "%2F") +) + +# MongoDB Databases: +MONGO_DATA_DATABASE_NAME = "converse" +MONGO_KEYS_DATABASE_NAME = "converse" +MONGO_FILE_DATABASE_NAME = "converseStore" + +# Mongo Config: +MONGO_DATA_POOL_SIZE = 10 +MONGO_KEYS_POOL_SIZE = 10 +MONGO_FILE_POOL_SIZE = 10 diff --git a/utils_v2/api/__pycache__/__init__.cpython-310.pyc b/utils_v2/api/__pycache__/__init__.cpython-310.pyc index 2ee960f..4981456 100644 Binary files a/utils_v2/api/__pycache__/__init__.cpython-310.pyc and b/utils_v2/api/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/api/__pycache__/async_quart.cpython-310.pyc b/utils_v2/api/__pycache__/async_quart.cpython-310.pyc index e948687..d0c6850 100644 Binary files a/utils_v2/api/__pycache__/async_quart.cpython-310.pyc and b/utils_v2/api/__pycache__/async_quart.cpython-310.pyc differ diff --git a/utils_v2/api/__pycache__/codes.cpython-310.pyc b/utils_v2/api/__pycache__/codes.cpython-310.pyc index b7c160c..343b95a 100644 Binary files a/utils_v2/api/__pycache__/codes.cpython-310.pyc and b/utils_v2/api/__pycache__/codes.cpython-310.pyc differ diff --git a/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc b/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc deleted file mode 100644 index 2970917..0000000 Binary files a/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc and /dev/null differ diff --git a/utils_v2/api/__pycache__/response.cpython-310.pyc b/utils_v2/api/__pycache__/response.cpython-310.pyc index bd3d7e5..65cf988 100644 Binary files a/utils_v2/api/__pycache__/response.cpython-310.pyc and b/utils_v2/api/__pycache__/response.cpython-310.pyc differ diff --git a/utils_v2/api/async_quart.py b/utils_v2/api/async_quart.py index 0c29f69..ab2ff39 100644 --- a/utils_v2/api/async_quart.py +++ b/utils_v2/api/async_quart.py @@ -46,13 +46,8 @@ from utils_v2.string import json from utils_v2.date_time import date_time from utils_v2.security import sanitizers from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.log import APILogModel from utils_v2.api.response import ResponseModel -from utils_v2.api.metrics_prometheus import ( - TOTAL_REQUEST_COUNT, - LIVE_REQUEST_COUNT, - REQUEST_LATENCY, - MetricsAPI -) # To work with date and time: import time @@ -60,9 +55,6 @@ import datetime # System-level activities: import io -import distro -import socket -import platform # For Pydantic data-models: import pydantic @@ -90,11 +82,6 @@ import asyncio # ***************************************************************************************************************** -# Info for logging that will stay constant during runtime: -SERVER_HOSTNAME = socket.gethostname() -PLATFORM_INFO = platform.uname() -HOST_OS = distro.name(True) - # Chars to choose from for random strings: ALPHANUMERIC_CHARS = string.ascii_letters + string.digits @@ -572,6 +559,8 @@ def validate_input( def log_request_to_mongo( attr_name, collection: str = "logs", + api_version: str = None, + project: str = None, log_type: str = None, operation: str = None, log_input: bool = True, @@ -586,6 +575,8 @@ def log_request_to_mongo( :param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the scope of 'current_app'. :param collection: The name of the collection to write the log into. + :param api_version: The version code of the API endpoint that is being logged. + :param project: The name of the project that the endpoint was built for. :param log_type: A hint to identify what the log was for. :param operation: A hint to identify what was action was being performed. :param log_input: Whether, or not, you would like to log the input that came in. @@ -602,6 +593,9 @@ def log_request_to_mongo( # Let the next in-line decorator know that it has been wrapped: kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + # Set the api version as needed: + kwargs["api_version"] = kwargs.get("api_version", api_version) + # Make variables and extract available info.: exception = None response = None @@ -635,7 +629,8 @@ def log_request_to_mongo( elif isinstance(response, tuple): response_to_log, http_code_to_log = response else: response_to_log, http_code_to_log = str(response), 200 - # Try to get the information about the request: + # Try to get the information about the request. + # There will be no data in any of these if the decorator was used to catch start-up and shut-down events. request_method = None request_url = None request_route = None @@ -653,41 +648,37 @@ def log_request_to_mongo( except: pass # Construct the log: - # for k in sensitive_keys: kwargs.get("inbound_headers", {}).pop(k, None) - # for k in sensitive_keys: kwargs.get("inbound_data", {}).pop(k, None) - log_json = { - "hostname": SERVER_HOSTNAME, - "os": f"{HOST_OS}", - "cpu": f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})", - "logId": kwargs.get("log_id"), - "logChain": kwargs.get("inbound_headers", {}).get("X-Log-Chain"), - "log": log_type, - "operation": operation, - "apiVer": kwargs.get("api_version"), - "method": request_method, - "url": request_url, - "route": request_route, - "ts": request_ts, - "tat": time.perf_counter() - start_ts, - "cpuTime": time.process_time() - cpu_start_ts, - "headers": kwargs.get("inbound_headers"), - "data": kwargs.get("inbound_data") if log_input else "not logged", - "files": { + api_log = APILogModel( + project = project, + log = log_type, + operation = operation, + apiVer = kwargs.get("api_version"), + logId = kwargs.get("log_id"), + logChain = kwargs.get("inbound_headers", {}).get("X-Log-Chain"), + method = request_method, + url = request_url, + route = request_route, + ts = request_ts, + tat = time.perf_counter() - start_ts, + cpuTime = time.process_time() - cpu_start_ts, + headers = kwargs.get("inbound_headers"), + data = kwargs.get("inbound_data") if log_input else "not logged", + files = { k: { "name": v["name"], "size": v["size"] } for k, v in kwargs.get("inbound_files", {}).items() }, - "exception": None if exception is None else describe_exception(exception), - "response": response_to_log, - "httpCode": http_code_to_log - } + exception = None if exception is None else describe_exception(exception), + response = response_to_log, + httpCode = http_code_to_log + ) # Write the log: app_attr = getattr(current_app, attr_name) inserted_id = await app_attr.insert_one( collection = collection, - document = log_json + document = api_log.model_dump() ) # Return the response from the wrapped function. @@ -950,51 +941,6 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): return decorator -# --------------------------------------------------------------------------------------------------------------------- - - -def measure_metrics_for_prometheus(): - - """ - Use this decorator to automatically measure metrics for using in Prometheus. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # We use the context of the measurement class: - async with MetricsAPI( - method = f"{request.method}", - endpoint = str(request.url_rule.rule) - ) as metrics: - - # Invoke the wrapped function: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Interpret the HTTP code: - if isinstance(response, ResponseModel): _, metrics.http_code = response.for_quart() - elif isinstance(response, tuple): metrics.http_code = response[1] - else: metrics.http_code = 200 - - # Done here: - if ( - kwargs["decorator_count"] == 1 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** diff --git a/utils_v2/api/metrics_prometheus.py b/utils_v2/api/log.py similarity index 51% rename from utils_v2/api/metrics_prometheus.py rename to utils_v2/api/log.py index f7e9e3e..9f5acef 100644 --- a/utils_v2/api/metrics_prometheus.py +++ b/utils_v2/api/log.py @@ -3,15 +3,14 @@ AUTHOR: Khushal P Soonderji - Sharvil J Daiya DATE: - Saturday, 28th Sept., 2024 + Thursday, 12th Sept., 2024 OBJECTIVE: - To have one place from where several metrics are measured using easy to use context managers. + To have a structure to the response sent from the API calls. REFERENCES: @@ -31,16 +30,21 @@ # ***************************************************************************************************************** -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") +# System-level activities: +import distro +import socket +import platform -# To measure time: -import time +# For data-modelling: +from pydantic import BaseModel, Field +from typing import Any, Optional, List, Literal -# To capture metrics for Prometheus: -from prometheus_client import Counter, Summary, Gauge +# To work with date and time: +import datetime + +# My utils: +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.date_time import date_time # ***************************************************************************************************************** @@ -50,31 +54,11 @@ from prometheus_client import Counter, Summary, Gauge # ***************************************************************************************************************** -# Define the Prometheus metrics. -# FOR THE MICROSERVICE AS A WHOLE: -WORKER_COUNT = Counter( - name = "ms_workers_active_total", - documentation = "The number of threads for the microservice being monitored.", - labelnames = ["project_name", "service_name", "host_name"] -) - -# Define the Prometheus metrics. -# FOR INDIVIDUAL API ENDPOINTS: -REQUEST_LATENCY = Summary( - name = "http_request_latency_seconds", - documentation = "Latency of HTTP requests in seconds.", - labelnames = ["method", "endpoint", "http_status"] -) -TOTAL_REQUEST_COUNT = Counter( - name = "http_requests_total", - documentation = "Total HTTP requests.", - labelnames = ["method", "endpoint", "http_status"] -) -LIVE_REQUEST_COUNT = Gauge( - name = "http_requests_live_total", - documentation = "To check if an API endpoint is being served right now.", - labelnames = ["endpoint"] -) +# Info for logging that will stay constant during runtime: +SERVER_HOSTNAME = str(socket.gethostname()) +PLATFORM_INFO = platform.uname() +HOST_OS = str(distro.name(True)) +HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" # ***************************************************************************************************************** @@ -94,59 +78,39 @@ LIVE_REQUEST_COUNT = Gauge( # ***************************************************************************************************************** -class MetricsAPI: +class APILogModel(BaseModel): - """ - Use this class through its context manager to automatically measure all the metrics in one place. - This was originally created to measure the performance of API endpoints made in Quart, but it should work with - other frameworks as well. - """ + # To identify the machine the code is running on. + # DO NOT MODIFY THESE: + hostname: str = SERVER_HOSTNAME + os: str = HOST_OS + cpu: str = HOST_CPU - def __init__( - self, - method = None, - endpoint = None, - raise_exception = False - ): + # To identify the project and actions: + project: Optional[str] = None + log: str + operation: str + apiVer: Optional[str] = None + logId: Optional[str] = None + logChain: Optional[str] = None - # Make provisions for things to note. - # NOTE: THESE MUST BE SET FROM OUTSIDE: - self.method = method - self.endpoint = endpoint - self.http_code = None - self.__raise_exception = raise_exception + # Timing metrics: + ts: datetime.datetime + tat: float + cpuTime: float - async def __aenter__(self): + # To understand the request that came in: + method: Optional[str] = None + url: Optional[str] = None + route: Optional[str] = None + headers: Optional[Any] = None + data: Optional[Any] = None + files: Optional[Any] = None - # Note down the start time immediately: - self.__start_ts = time.perf_counter() - self.__cpu_start_ts = time.process_time() - - # Note down the metrics: - LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1) - - # Setup done: - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - - # Note down the metrics: - LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1) - - TOTAL_REQUEST_COUNT.labels( - self.method, - self.endpoint, - self.http_code - ).inc() - - REQUEST_LATENCY.labels( - self.method, - self.endpoint, - self.http_code - ).observe(time.perf_counter() - self.__start_ts) - - # Handle the exception as per the user's preference: - return False if self.__raise_exception else True + # To understand the output that went out: + exception: Optional[Any] = None + response: Optional[Any] = None + httpCode: Optional[int] = None # ***************************************************************************************************************** @@ -168,31 +132,8 @@ class MetricsAPI: if __name__ == "__main__": - import asyncio - import random - from prometheus_client import generate_latest + my_log = APILogModel( + log = "internal" + ) - async def simulate_endpoint(): - - async with MetricsAPI( - method = random.choice(["GET", "POST"]), - endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}" - ) as metrics: - - # Simulate some action on some endpoint: - await asyncio.sleep(1.0) - - # Note down the values: - metrics.http_code = 200 - - async def main(): - - print("Simulating endpoints...") - tasks = [simulate_endpoint() for _ in range(250)] - await asyncio.gather(*tasks) - print("Done!") - - print("METRICS:") - print(generate_latest().decode()) - - asyncio.run(main()) + print(my_log) diff --git a/utils_v2/api/response.py b/utils_v2/api/response.py index 3c51696..02e3d72 100644 --- a/utils_v2/api/response.py +++ b/utils_v2/api/response.py @@ -30,9 +30,11 @@ # ***************************************************************************************************************** +# For data-modelling: from pydantic import BaseModel from typing import Any, Optional, List +# My utils: from utils_v2.api.codes import StatusCodes, HttpCodes @@ -64,6 +66,11 @@ from utils_v2.api.codes import StatusCodes, HttpCodes class ResponseModel(BaseModel): + """ + A model for how the response should be when developing API endpoints. + """ + + # The fields that you want in your response: status_code: StatusCodes message: Optional[str | List] = None data: Optional[Any] = None @@ -74,6 +81,11 @@ class ResponseModel(BaseModel): def for_quart(self): + """ + Call this when you are using either Flask or Quart as your framework. + :return: The output as expected by Flask and Quart. + """ + # Construct the basic structure: response_dict = { "status": 1 if self.status_code.value[0] else 0, diff --git a/utils_v2/database/__pycache__/__init__.cpython-310.pyc b/utils_v2/database/__pycache__/__init__.cpython-310.pyc index 9b36a88..a9d214a 100644 Binary files a/utils_v2/database/__pycache__/__init__.cpython-310.pyc and b/utils_v2/database/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc b/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc index fccfeaa..2fe27b8 100644 Binary files a/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc and b/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc differ diff --git a/utils_v2/database/async_mongo_v2.py b/utils_v2/database/async_mongo_v2.py index dc9955a..663ed3a 100644 --- a/utils_v2/database/async_mongo_v2.py +++ b/utils_v2/database/async_mongo_v2.py @@ -1691,42 +1691,43 @@ if __name__ == "__main__": async def main(): # Create an instance of the database connector: - my_fs = AsyncMongoStorage( - connection_string = constants.MONGO_FILE_CONNECTION_STRING, - database_name = constants.MONGO_FILE_DATABASE_NAME, + my_db = AsyncMongo( + connection_string = constants.MONGO_DATA_CONNECTION_STRING, + database_name = constants.MONGO_DATA_DATABASE_NAME, max_connections = 10, debug = True ) # Connect to the database: - await my_fs.connect() + await my_db.connect() - # Keep performing the changes in batches till you have corrections to make: - while True: + # # Get the documents to migrate: + # documents = await my_db.find_many( + # collection = "scriptData", + # filter = {}, + # limit = 50, + # projection = {"_id": False} + # ) + # # print(json.to_string(documents, default = str)) + # + # # Adjust them: + # adjusted_documents = [] + # for document in documents: + # script_id = document.pop("scriptId") + # adjusted_document = { + # "scriptId": script_id, + # "desc": "no desc", + # "content": document + # } + # adjusted_documents.append(adjusted_document) + # print(json.to_string(adjusted_documents, default = str)) + # + # # Insert the adjusted ones to the new collection: + # response = await my_db.insert_many( + # collection = "_scriptData", + # documents = adjusted_documents + # ) + # print("RESPONSE:", response) - # Find all the files that have their metadata as a string: - files = await my_fs.find_many( - filter = { - "metadata": {"$type": "string"} - }, - limit = 10 - ) - - print(my_fs.to_json_string(files)) - break - - # # If no matches were found: - # if not files: break - # - # # Fix the metadata file-by-file: - # for file in files: - # file_id = file["_id"] - # success = await my_fs.replace_metadata_for_one( - # filter = {"_id": file_id}, - # replacement = json.from_string(file["metadata"]) - # ) - # print(file_id, ":", success) - - print("Fixes done!") asyncio.run(main()) diff --git a/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc b/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc index 98444f4..b18eeed 100644 Binary files a/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc and b/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc b/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc index 33397e9..46643f1 100644 Binary files a/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc and b/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc differ diff --git a/utils_v2/security/__pycache__/__init__.cpython-310.pyc b/utils_v2/security/__pycache__/__init__.cpython-310.pyc index 20af223..e6d54d5 100644 Binary files a/utils_v2/security/__pycache__/__init__.cpython-310.pyc and b/utils_v2/security/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc b/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc index 7193fe2..4768d00 100644 Binary files a/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc and b/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc differ diff --git a/utils_v2/string/__pycache__/__init__.cpython-310.pyc b/utils_v2/string/__pycache__/__init__.cpython-310.pyc index adc32f1..6e9484d 100644 Binary files a/utils_v2/string/__pycache__/__init__.cpython-310.pyc and b/utils_v2/string/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/string/__pycache__/json.cpython-310.pyc b/utils_v2/string/__pycache__/json.cpython-310.pyc index 5b01bd8..98048fa 100644 Binary files a/utils_v2/string/__pycache__/json.cpython-310.pyc and b/utils_v2/string/__pycache__/json.cpython-310.pyc differ diff --git a/utils_v2/string/__pycache__/regex.cpython-310.pyc b/utils_v2/string/__pycache__/regex.cpython-310.pyc index ad9ed17..1a93e0b 100644 Binary files a/utils_v2/string/__pycache__/regex.cpython-310.pyc and b/utils_v2/string/__pycache__/regex.cpython-310.pyc differ diff --git a/utils_v2/system/__pycache__/__init__.cpython-310.pyc b/utils_v2/system/__pycache__/__init__.cpython-310.pyc index dac9ac6..403cde5 100644 Binary files a/utils_v2/system/__pycache__/__init__.cpython-310.pyc and b/utils_v2/system/__pycache__/__init__.cpython-310.pyc differ diff --git a/utils_v2/system/__pycache__/files.cpython-310.pyc b/utils_v2/system/__pycache__/files.cpython-310.pyc index 3755755..f3e7a71 100644 Binary files a/utils_v2/system/__pycache__/files.cpython-310.pyc and b/utils_v2/system/__pycache__/files.cpython-310.pyc differ