From d9117f2a9299362330ea1afd2d0eb059ab426dcb Mon Sep 17 00:00:00 2001 From: khushal Date: Tue, 8 Oct 2024 15:36:20 +0530 Subject: [PATCH] (20241008) logging decorator and model improved. --- README_GIT.md | 83 ++++ api/__init__.py | 0 api/cred_data/__init__.py | 0 api/cred_data/blueprint.py | 406 ++++++++++++++++++ api/logs/__init__.py | 0 api/logs/blueprint.py | 381 ++++++++++++++++ api/main.py | 301 +++++++++++++ cred/README_CERTS.md | 6 + cred/mongo/mongo_data_connection_string.txt | 1 + cred/mongo/mongo_file_connection_string.txt | 1 + cred/mongo/mongo_keys_connection_string.txt | 1 + shared/__init__.py | 0 shared/constants.py | 100 +++++ .../api/__pycache__/__init__.cpython-310.pyc | Bin 162 -> 162 bytes .../__pycache__/async_quart.cpython-310.pyc | Bin 23924 -> 22943 bytes .../api/__pycache__/codes.cpython-310.pyc | Bin 3708 -> 3708 bytes .../metrics_prometheus.cpython-310.pyc | Bin 3424 -> 0 bytes .../api/__pycache__/response.cpython-310.pyc | Bin 1632 -> 1873 bytes utils_v2/api/async_quart.py | 116 ++--- .../api/{metrics_prometheus.py => log.py} | 163 +++---- utils_v2/api/response.py | 12 + .../__pycache__/__init__.cpython-310.pyc | Bin 167 -> 167 bytes .../async_mongo_v2.cpython-310.pyc | Bin 40214 -> 40075 bytes utils_v2/database/async_mongo_v2.py | 61 +-- .../__pycache__/__init__.cpython-310.pyc | Bin 168 -> 168 bytes .../__pycache__/date_time.cpython-310.pyc | Bin 5816 -> 5816 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 167 -> 167 bytes .../__pycache__/sanitizers.cpython-310.pyc | Bin 1929 -> 1929 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 165 -> 165 bytes .../string/__pycache__/json.cpython-310.pyc | Bin 3542 -> 3542 bytes .../string/__pycache__/regex.cpython-310.pyc | Bin 6844 -> 6844 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 165 -> 165 bytes .../system/__pycache__/files.cpython-310.pyc | Bin 5493 -> 5493 bytes 33 files changed, 1406 insertions(+), 226 deletions(-) create mode 100644 README_GIT.md create mode 100644 api/__init__.py create mode 100644 api/cred_data/__init__.py create mode 100644 api/cred_data/blueprint.py create mode 100644 api/logs/__init__.py create mode 100644 api/logs/blueprint.py create mode 100644 api/main.py create mode 100644 cred/README_CERTS.md create mode 100644 cred/mongo/mongo_data_connection_string.txt create mode 100644 cred/mongo/mongo_file_connection_string.txt create mode 100644 cred/mongo/mongo_keys_connection_string.txt create mode 100644 shared/__init__.py create mode 100644 shared/constants.py delete mode 100644 utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc rename utils_v2/api/{metrics_prometheus.py => log.py} (51%) diff --git a/README_GIT.md b/README_GIT.md new file mode 100644 index 0000000..8cd438a --- /dev/null +++ b/README_GIT.md @@ -0,0 +1,83 @@ + +# GIT CHEATSHEET +## By Sharvil Sir 🙏 (20240927) +Perform these steps on creating a new project. + +_**ASSUMPTION:** You are already in your project directory in the command line terminal._ + +--- + +### STEP 0. +#### Initialize Git in your new project: +Do this when creating a new project directory. +Do this **ONLY ONCE**. +```commandline +git init +``` + +### STEP 1. +#### Create a `.gitignore` file: +Add files/directories in it which you don't want to sync to git. +Update this as frequently as your project needs you to. Start with this: +```commandline +/.venv/ +/.idea/ +**/__pycache__/ +*.pem +``` + +### STEP 2. +#### Create a repository on Gitea (web UI): +Do this **ONLY ONCE**. +Open the following URL: https://wtt.ditscentre.in/ and sign in. +- Ensure that the owner is `ditscentre` (img 0). +- Give your repository a name. Preferably keep it the same as the project name that you made locally (img 0). +- Type in a brief description of your project (img 0). + +![img 0](https://nexcom.ditscentre.in/utils/files/small/download/66f66a54196705ee2f25d28e) + +- Ensure that the default branch is `master` (img 1). +- Create the repository (img 1). + +![img 1](https://nexcom.ditscentre.in/utils/files/small/download/66f66a54196705ee2f25d28f) + +### STEP 3. +#### Add files and directories to the staging area: +The following command adds everything to the staging area. +The '.' is important, it refers to the current directory. +```commandline +git add . +``` + +### STEP 4. +#### Commit the changes to local git: +This step commits all the added (staged) changes to the local git instance with the provided comment. +```commandline +git commit -m "you comment here..." +``` + +### STEP 5. +#### We push the local commits to the repository: +The format of the command is: +``` +git push -u +``` + +--- + +## For Python Devs: How to add a common subtree (like `utils_v2`) + +The format of the command is: +``` +git subtree add --prefix= --squash +``` +Before you start: ensure that you have committed all pending changes. Then run the following command **ONLY ONCE**: +```commandline +git subtree add --prefix=utils_v2 https://wtt.ditscentre.in/ditscentre/utils_v2.git master --squash +``` +This will create a new directory named `utils_v2` in your project. +**Do NOT change the files in it.** Just import and use them in your scripts. +When you need to get the latest updates to these files, you must run a modification of the previous command: +```commandline +git subtree pull --prefix=utils_v2 https://wtt.ditscentre.in/ditscentre/utils_v2.git master --squash +``` \ No newline at end of file diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/cred_data/__init__.py b/api/cred_data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/cred_data/blueprint.py b/api/cred_data/blueprint.py new file mode 100644 index 0000000..cf2ddb3 --- /dev/null +++ b/api/cred_data/blueprint.py @@ -0,0 +1,406 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Wednesday, 28th Aug., 2024 + + OBJECTIVE: + + To be able to fetch setup credentials and data for any project. + This could include things like default values, URLs to assets, etc. + While 'data' and 'cred' can have anything held in them, the idea behind giving two services is for the user of + this service to be able to organise his setup variables. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For using Quart: +from quart import Blueprint, current_app + +# My utils: +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.response import ResponseModel +from utils_v2.api.async_quart import ( + read_input, + log_request_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input +) + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Related to Quart: +cred_and_data_bp = Blueprint("int_cnd", __name__) +get_api_version = "2.0.0" +set_api_version = "2.0.0" +update_api_version = "2.0.0" +delete_api_version = "2.0.0" + +# related to the operations of this blueprint: +JSON_TYPE_INFO = { + "cred": { + "collection": "scriptCred" + }, + "data": { + "collection": "scriptData" + } +} + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@cred_and_data_bp.record_once +def init(blueprint_setup_state): + + # This gets called when the blueprint is registered. + # Consider this to be a one-time setup for the whole blueprint: + pass + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//set", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "set", + api_version = set_api_version, + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def set_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To set the credentials for a particular script. If the document exists, it will be overwritten. If the document + doesn't exist, it will be created. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart_utils.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart_utils.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Pre-process the inbound data: + inbound_data = inbound_data or {} + inbound_data["scriptId"] = inbound_headers["X-Script-Id"] + + # Make an attempt to set the credentials: + success = await current_app.mongo.replace_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + replacement = inbound_data, + upsert = True + ) + + # Return the response: + if success: return ResponseModel(api_version = set_api_version, status_code = StatusCodes.OK) + else: return ResponseModel(api_version = set_api_version, status_code = StatusCodes.FAILED) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = set_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//get", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "get", + api_version = get_api_version, + log_input = True, + log_output = False +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def get_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To retrieve the credentials stored for a specific script. The script's id can be anything set by the programmers. + The idea is to have only the script's id stored in the script, and everything else is fetched from the database. + This means that we get to store and update everything from one central location. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to retrieve the credentials: + cred_json = await current_app.mongo.find_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + projection = {"_id": False, "scriptId": False} + ) + + # In case no result was found: + if cred_json is None: + return ResponseModel( + api_version = get_api_version, + status_code = StatusCodes.FAILED, + message = "invalid script id" + ) + + # Successfully retrieved: + return ResponseModel( + api_version = get_api_version, + status_code = StatusCodes.OK, + data = cred_json + ) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = get_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//update", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "update", + api_version = update_api_version, + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def update_cred( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To update values of certain fields for a credentials document. It only updates existing values, does NOT add a new + document if the document doesn't already exist. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart_utils.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart_utils.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart_utils.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Pre-process the inbound data: + inbound_data = inbound_data or {} + + # Prepare the update JSON: + update_json = {} + if inbound_data.get("unset"): update_json["$unset"] = current_app.mongo.dict_to_dot_notation(inbound_data["unset"]) + if inbound_data.get("set"): update_json["$set"] = current_app.mongo.dict_to_dot_notation(inbound_data["set"]) + + # Make an attempt to set the credentials: + success = await current_app.mongo.update_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + update = update_json, + upsert = False + ) + + # Return the response: + if success: return ResponseModel(api_version = update_api_version, status_code = StatusCodes.OK) + else: return ResponseModel(api_version = update_api_version, status_code = StatusCodes.FAILED) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = update_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//delete", methods = ["POST", "GET"]) +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + log_type = "internalCredData", + operation = "delete", + api_version = delete_api_version, + log_input = True, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def delete_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + log_id: str = None +): + + """ + To delete a document for a particular script id. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_data: auto-extracted by the decorators from 'async_quart.py'. + :param inbound_files: auto-extracted by the decorators from 'async_quart.py'. + :param log_id: An identifier for the logs (if logging is enabled). + :return: A standard response structure from the function in 'async_quart.py'. + """ + + try: + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid url segment '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to set the credentials: + success = await current_app.mongo.delete_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + ) + + # Return the response: + if success: return ResponseModel(api_version = delete_api_version, status_code = StatusCodes.OK) + else: return ResponseModel(api_version = delete_api_version, status_code = StatusCodes.FAILED) + + # In case the client terminates the connection prematurely: + except asyncio.CancelledError as exception: + current_app.printer(exception) + return ResponseModel( + api_version = delete_api_version, + status_code = StatusCodes.CLIENT_CLOSED_REQUEST + ) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/api/logs/__init__.py b/api/logs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/logs/blueprint.py b/api/logs/blueprint.py new file mode 100644 index 0000000..9fa4d30 --- /dev/null +++ b/api/logs/blueprint.py @@ -0,0 +1,381 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Created: Wednesday, 28th Aug., 2024 + Updated: Tuesday, 8th Oct. 2024 + + OBJECTIVE: + + To be able to fetch setup credentials and data for any project. This could include things like default values, + URLs to assets, etc. While 'data' and 'cred' can be used interchangeably, the idea behind giving two services is + for the user of this service to be able to organise his setup variables. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + + NOTES: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For using Quart: +from quart import Blueprint, current_app + +# My utils: +from utils_v2.string import json +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.response import ResponseModel +from utils_v2.api.async_quart import ( + set_api_version, + read_input, + log_request_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input, + handle_cancelled_request +) + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Related to Quart: +cred_and_data_bp = Blueprint("int_cnd", __name__) + +# related to the operations of this blueprint: +JSON_TYPE_INFO = { + "cred": { + "collection": "_scriptCred" + }, + "data": { + "collection": "_scriptData" + } +} + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@cred_and_data_bp.record_once +def init(blueprint_setup_state): + + # This gets called when the blueprint is registered. + # Consider this to be a one-time setup for the whole blueprint: + pass + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//set", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "set", + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id", "X-Script-Desc"]) +@handle_cancelled_request() +async def set_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To set the credentials for a particular script. If the document exists, it will be overwritten. If the document + doesn't exist, it will be created. Ideally use this for only the first time setup. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Construct the document: + description = inbound_headers.get("X-Script-Desc", "") + if len(description) > 200: description = description[:200] + document = { + "scriptId": inbound_headers["X-Script-Id"], + "desc": description, + "content": inbound_data + } + + # Make an attempt to set the credentials: + success = await current_app.mongo.replace_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + replacement = document, + upsert = True, + raise_exception = True + ) + + # Return the response: + if success: return ResponseModel(status_code = StatusCodes.OK) + else: return ResponseModel(status_code = StatusCodes.FAILED) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//get", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "get", + log_input = True, + log_output = False +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@handle_cancelled_request() +async def get_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To retrieve the credentials stored for a specific script. The script's id can be anything set by the programmers. + The idea is to have only the script's id stored in the script, and everything else is fetched from the database. + This means that we get to store and update everything from one central location. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to retrieve the credentials: + cred_json = await current_app.mongo.find_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + projection = {"_id": False, "scriptId": False}, + raise_exception = True + ) + + # In case no result was found: + if cred_json is None: + return ResponseModel( + status_code = StatusCodes.FAILED, + message = "invalid script id" + ) + + # Successfully retrieved: + return ResponseModel( + status_code = StatusCodes.OK, + data = cred_json["content"] + ) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//update", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "update", + log_input = False, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@handle_cancelled_request() +async def update_cred( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To update values of certain fields for a credentials document. It only updates existing values, does NOT add a new + document if the document doesn't already exist. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Pre-process the inbound data: + inbound_data = inbound_data or {} + + # Prepare the update JSON. Pre-process the fields to set and unset. + # Our actual data/cred are held inside a field called "content", so we must wrap the request in that: + update_json = {} + if inbound_data.get("unset"): + update_json["$unset"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["unset"]}) + if inbound_data.get("set"): + update_json["$set"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["set"]}) + + # Make an attempt to set the credentials: + success = await current_app.mongo.update_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + update = update_json, + upsert = False, + raise_exception = True + ) + + # Return the response: + if success: return ResponseModel(status_code = StatusCodes.OK) + else: return ResponseModel(status_code = StatusCodes.FAILED) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@cred_and_data_bp.route("//delete", methods = ["POST", "GET"]) +@set_api_version(api_version = "2.1.0") +@read_input(sanitize_headers = True, sanitize_data = True) +@log_request_to_mongo( + attr_name = "mongo", + project = "internal", + log_type = "credData", + operation = "delete", + log_input = True, + log_output = True +) +@should_not_be_under_maintenance(attr_name = "is_under_maintenance") +@only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(mandatory_header_keys = ["X-Script-Id"]) +async def delete_data( + json_type: str = None, + inbound_headers: dict = None, + inbound_data: dict = None, + inbound_files: dict = None, + **kwargs +): + + """ + To delete a document for a particular script id. + :param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'. + :param inbound_headers: auto-extracted by the decorators. + :param inbound_data: auto-extracted by the decorators. + :param inbound_files: auto-extracted by the decorators. + :param kwargs: Any number of extra inputs supplied by the decorators. + :return: A standard response structure. + """ + + # If an invalid choice was made: + if json_type not in JSON_TYPE_INFO.keys(): + return ResponseModel( + status_code = StatusCodes.FAILED, + message = f"invalid path '{json_type}'", + http_code = HttpCodes.BAD_REQUEST + ) + + # Make an attempt to set the credentials: + success = await current_app.mongo.delete_one( + collection = JSON_TYPE_INFO[json_type]["collection"], + filter = {"scriptId": inbound_headers["X-Script-Id"]}, + ) + + # Return the response: + if success: return ResponseModel(status_code = StatusCodes.OK) + else: return ResponseModel(status_code = StatusCodes.FAILED) + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000..0edcc28 --- /dev/null +++ b/api/main.py @@ -0,0 +1,301 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Wednesday, 28th Aug., 2024 + + OBJECTIVE: + + This is the central location for the Quart module. + We define the app here, and import and attach all blueprints here. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For system level activities: +import gc +import os + +# For using Quart: +from quart import Quart, request, current_app +from quart_cors import cors + +# To make REST-API calls: +import httpx + +# My utils: +from utils_v2.string import json +from utils_v2.api import async_quart +from utils_v2.database.async_mongo_v2 import AsyncMongo +from utils_v2.api.async_quart import ( + read_input, + log_request_to_mongo, + should_not_be_under_maintenance, + only_whitelisted_ips, + limit_rate, + validate_input +) + +# For debugging: +from icecream import IceCreamDebugger + +# Other blueprints: +from api.llm.chat_completion import llm_chat_bp + +# LangChain-related: +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.messages import HumanMessage, AIMessage, SystemMessage +from langchain_openai import ChatOpenAI + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Quart related: +MODULE_BASE = "ai" +APP_VERSION = "1.0.0" + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# The Quart app: +app = Quart(__name__) +app = cors(app) +app.register_blueprint(llm_chat_bp, url_prefix = f"/{MODULE_BASE}/llm") + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +@app.before_serving +@log_request_to_mongo( + attr_name = "mongo", + log_type = MODULE_BASE, + operation = "apiStart", + api_version = APP_VERSION, + log_input = True, + log_output = True +) +async def app_startup(): + + """ + To initialize the variables that you would like to use in this module. + WARNING: ALL VARIABLES WILL BE INITIALIZED 'n' NUMBER OF TIMES, WHERE 'n' IS THE COUNT OF WORKERS DEPLOYED. + SO, IF YOU WANT TO CONNECT TO A DATABASE AND YOU ALLOW A POOL-SIZE OF 10 AND IF YOU DEPLOY 4 WORKERS, YOU WILL END + UP WITH 40 CONNECTIONS TO THE DATABASE. + :return: None. + """ + + # Safe-halt mechanism for upgrades (for a single-worker run): + current_app.is_under_maintenance = False + + # Debugging: + current_app.printer = IceCreamDebugger(prefix = f"{MODULE_BASE} (Q) | ", includeContext = True) + if os.environ["DEBUG"] == "True": current_app.printer.disable() + + # To make API calls: + max_connections = 5 + limits = httpx.Limits( + max_keepalive_connections = max_connections, + max_connections = max_connections, + keepalive_expiry = 3600 + ) + current_app.http_client = httpx.AsyncClient(limits = limits) + + # Get the credentials and data for this script: + script_id = os.environ.get("SCRIPT_ID") + response = await current_app.http_client.get( + url = r"https://nexcom.ditscentre.in/internal/cred/get", + headers = {"X-Script-Id": script_id} + ) + script_cred = response.json().get("data") + response = await current_app.http_client.get( + url = r"https://nexcom.ditscentre.in/internal/data/get", + headers = {"X-Script-Id": script_id} + ) + current_app.script_data = response.json().get("data") + + # To connect to Mongo: + current_app.mongo = AsyncMongo( + connection_string = script_cred["mongoDb"]["dataDb"]["connectionString"], + database_name = script_cred["mongoDb"]["dataDb"]["dbName"], + max_connections = script_cred["mongoDb"]["dataDb"]["poolSize"], + debug = True if os.environ["DEBUG"] == "True" else False + ) + + # Remove unwanted/sensitive variables from RAM: + del script_cred + gc.collect() + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.after_serving +@log_request_to_mongo( + attr_name = "mongo", + log_type = MODULE_BASE, + operation = "apiStop", + api_version = APP_VERSION, + log_input = True, + log_output = True +) +async def app_shutdown(): + + """ + This is called when "app.shutdown()" is called. + :return: None. + """ + + message = "Shutting down..." + current_app.printer(message) + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.route(f"/", methods = ["GET", "POST"]) +@app.route(f"/{MODULE_BASE}", methods = ["GET", "POST"]) +async def root(): + + """ + To check if the service is running or not. + Use this to monitor the service from your "watchman" script. + :return: only "ok" + """ + + return "ok" + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.route(f"/{MODULE_BASE}/debug/", methods = ["POST", "GET"]) +async def change_debug(action): + + """ + Enable or disable debugging for the entire microservice. + WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS. + :param action: "enable" to allow debugging on the terminal, or "disable". + :return: "enabled"/"disabled" if successful, else "ok" + """ + + # Enable or disable debugging only if the password matches: + action = action.lower() + if action == "enable": current_app.printer.enable() + elif action == "disable": current_app.printer.disable() + return "ok" + + +# --------------------------------------------------------------------------------------------------------------------- + + +@app.route(f"/{MODULE_BASE}/maintenance/", methods = ["POST", "GET"]) +async def change_maintenance(action): + + """ + Enable or disable debugging for the entire microservice. + WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS. + :param action: "enable" to stop taking new requests on the API, or "disable". + :return: "enabled"/"disabled" if successful, else "ok" + """ + + # Enable or disable debugging only if the password matches: + action = action.lower() + if action == "enable": current_app.is_under_maintenance = True + elif action == "disable": current_app.is_under_maintenance = False + return "ok" + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + # To get args from the terminal: + import argparse + + # To run the ASGI: + import uvicorn + from multiprocessing import freeze_support + + # Get the config from the command-line: + parser = argparse.ArgumentParser(description = f"Microservice for '{MODULE_BASE}' API.") + parser.add_argument( + "--workers", + type = int, + help = "The no. of threads to spin up for this instance!", + default = 2 + ) + parser.add_argument( + "--host", + type = str, + help = "The host for the app. e.g.: '0.0.0.0' or '127.0.0.1'.", + default = "127.0.0.1" + ) + parser.add_argument( + "--script-id", + type = str, + help = "The id of this script (will affect the loaded config)." + ) + parser.add_argument( + "--debug", + action = "store_true", + help = "Whether, or not, you want to see debugging messages in the terminal.", + default = False + ) + args = parser.parse_args() + + # Note down the config; + os.environ["SCRIPT_ID"] = args.script_id + os.environ["DEBUG"] = str(args.debug) + + # Run the gateway: + freeze_support() + uvicorn.run( + app = "main:app", + workers = args.workers, + host = args.host, + port = 8080 + ) diff --git a/cred/README_CERTS.md b/cred/README_CERTS.md new file mode 100644 index 0000000..b68709a --- /dev/null +++ b/cred/README_CERTS.md @@ -0,0 +1,6 @@ +# INSTRUCTIONS TO RENEW CERTIFICATES +**Date:** 2024-09-18 + +1. Go to [jcdev.ditscentre.in/jcdev](http://jcdev.ditscentre.in/jcdev/) +2. Navigate to the respective folders (e.g., `kafka`, `mongo`, etc.). +3. Download and replace the certificates to ensure they have the same names as those already present in the `cred` folder. diff --git a/cred/mongo/mongo_data_connection_string.txt b/cred/mongo/mongo_data_connection_string.txt new file mode 100644 index 0000000..7865109 --- /dev/null +++ b/cred/mongo/mongo_data_connection_string.txt @@ -0,0 +1 @@ +mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external \ No newline at end of file diff --git a/cred/mongo/mongo_file_connection_string.txt b/cred/mongo/mongo_file_connection_string.txt new file mode 100644 index 0000000..7865109 --- /dev/null +++ b/cred/mongo/mongo_file_connection_string.txt @@ -0,0 +1 @@ +mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external \ No newline at end of file diff --git a/cred/mongo/mongo_keys_connection_string.txt b/cred/mongo/mongo_keys_connection_string.txt new file mode 100644 index 0000000..7865109 --- /dev/null +++ b/cred/mongo/mongo_keys_connection_string.txt @@ -0,0 +1 @@ +mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external \ No newline at end of file diff --git a/shared/__init__.py b/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared/constants.py b/shared/constants.py new file mode 100644 index 0000000..ebfaba2 --- /dev/null +++ b/shared/constants.py @@ -0,0 +1,100 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 17th Aug., 2024 + + OBJECTIVE: + + To hold constant that will be shared throughout the project. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import os + +# My utils: +from utils_v2.system import files + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# Directories: +PROJECT_DIRECTORY = files.get_parent_directory(files.get_parent_directory(files.get_cwd())) +CREDENTIALS_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "cred") +MONGO_CREDENTIALS_DIRECTORY = os.path.join(CREDENTIALS_DIRECTORY, "mongo") +MODELS_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "models") + +# Files: +MONGO_KEYS_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_connection_string.txt") +MONGO_KEYS_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_ca.pem") +MONGO_KEYS_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_cert.pem") +MONGO_DATA_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_connection_string.txt") +MONGO_DATA_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_ca.pem") +MONGO_DATA_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_cert.pem") +MONGO_FILE_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_connection_string.txt") +MONGO_FILE_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_ca.pem") +MONGO_FILE_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_cert.pem") + +# MongoDB Connection Strings: +MONGO_KEYS_CONNECTION_STRING = files.read_file( + MONGO_KEYS_CONNECTION_STRING_FILE, + mode = "r" +).format( + mongo_ca = MONGO_KEYS_CA_FILE.replace("/", "%2F"), + mongo_cert = MONGO_KEYS_CERT_FILE.replace("/", "%2F") +) +MONGO_DATA_CONNECTION_STRING = files.read_file( + MONGO_DATA_CONNECTION_STRING_FILE, + mode = "r" +).format( + mongo_ca = MONGO_DATA_CA_FILE.replace("/", "%2F"), + mongo_cert = MONGO_DATA_CERT_FILE.replace("/", "%2F") +) +MONGO_FILE_CONNECTION_STRING = files.read_file( + MONGO_FILE_CONNECTION_STRING_FILE, + mode = "r" +).format( + mongo_ca = MONGO_FILE_CA_FILE.replace("/", "%2F"), + mongo_cert = MONGO_FILE_CERT_FILE.replace("/", "%2F") +) + +# MongoDB Databases: +MONGO_DATA_DATABASE_NAME = "converse" +MONGO_KEYS_DATABASE_NAME = "converse" +MONGO_FILE_DATABASE_NAME = "converseStore" + +# Mongo Config: +MONGO_DATA_POOL_SIZE = 10 +MONGO_KEYS_POOL_SIZE = 10 +MONGO_FILE_POOL_SIZE = 10 diff --git a/utils_v2/api/__pycache__/__init__.cpython-310.pyc b/utils_v2/api/__pycache__/__init__.cpython-310.pyc index 2ee960ff44a6f2f17800ea484d37a1b069cffd19..4981456b201e85562378d41e375e61c9b15e839c 100644 GIT binary patch delta 31 lcmZ3)xQLNEpO=@50SNXt{hr93$dQ>>l3J9Pm@~0N4*-T_3AO+L delta 31 lcmZ3)xQLNEpO=@50SL~m|2~mBks~oZJ-)O!wP<3A9sr3;3Eltz diff --git a/utils_v2/api/__pycache__/async_quart.cpython-310.pyc b/utils_v2/api/__pycache__/async_quart.cpython-310.pyc index e9486874e7560511e1f288c275407e0121c12002..d0c6850ad6b20a749024265744fa07517e55a39a 100644 GIT binary patch delta 5121 zcmaJ_Yj7LY72dnMlBKm|%Z?wi{8Ic#qQsBbdH6}}#34;cZ~`PGHDOdmckRfQCFN?B zhmq^F$;9RnAlzXJEtI%WXecdA#gs>wDJ|0}ZU2PFv~*hv6gnNC!@EgoL+Cko{ZiWL zYWCZ^=bm%!J?EZ#&Ru=>EO{V98scuZCc#Dis18-%D@kwT%fhPwUV7n9zDG_qBz#1o zglf^EfgUJmBZC=`%9&P#b1$VE_EH3s7tRorVP|lH_baP4b;(m z>VaoHEzspbnHJJT$JK!)PL1}`B7G@PifPGl+rTnfO1;1rpk>qt&*gMIt)x}Q$-oL) zP5nS=aF#pE=?1!()&RB9sW~S!0Qk{9T1)GI*yOCYD1g8WJsar~y_qhBdl}pTxR=Ad zLSIQ6;BJHocxV%Ch6%LLm9zzhX{D`n6+BltS3B3tXj==KuB2`HYTB-^p&j~C+6j^l zAMK*uAi9=rqF2y$Fo8C`o%W2#?9|MYIuV%quB&r7oS$s6e^aCk9X*|eZ z(#q_O$Xl8DNPC4$2)U(5ScVWlIW?*4BWwvaUEw$w5Ho_tk%S(wvr1$_8Pgq!?Te>k zG>C&F*>db?zz(MvH5nPxFP~4LV9lBkIm`RpwWKXG>?18GJwOuHfJ^#mdZl*3^A3Y!2&YtLMc1&2aSFIJ=bU?Y0b5ipDNl= z(TdEm__z2n2<(bLb%5rxpv|s_AgXB+YPTf zGsDFmWz81UCA3@xt=ajh5knunYj+% z#lqHngx7Npu}~tRvu(Vi@`hz8RCy5K3=!<`5G29sqP9e>hy)MnEC~y`9{q>A%H>pj?50dpRiU~fhV0NGu{eO4KO4~d^$-Oy zR)t+_uuI}!^A_`?HQQ^%!js@-UNxQFLwZaM@kBx<5L;3>G9>6XJ5Kvf(-&b3oZ!HUw(M31$bG3W@hoE&vW;Y}C7K9@R zXut5u5x%Xi$?}w9^$X5aC)-nOoIkl@ z#p*ftj5$Xm$z&up)ai#9xC6D`31B++8%81+hCCT4WWpnEzNn#zY~ULj){w3Ia6`+| z5y5!n?8dWvfab$C!DwPhge&>E2Csb(n+E@&p#&1Ct#S8dPC3g*8sBLaPO-83k@vd@ z4wc(JemzDL zai~K;2pRrJ(!WoK#qz`OLpp8J{BU(9LM$}upP5>xeLZiBWM$Jrzr2+NM={yty zeW^&)@IxY)SC%=~@(>B+!rU>>C6@yj(v)hgLUJ2^*BrBB7vsO1+b?5<#h$CNV;B zI&h+*)yE}4^@{;Zr=^YHYv@~?mZ_AIK&pvKF~#tvWkIh=qhxK`$;s_xJl`lYd{g9@ zJnl(*hRdhqbpGir71I)}J1K9GsI*xc2YVtk7Fn16O$}w593pT=wv^@mL1ag^*S?cK-ogY zuIbw!io}G_Sni(2ZSkQ-LE(*UOFccPY9)fmPxYL(ZC>4s?=1i@)?C}l;horp3BUr8 zJ-gxc+x*Qoy%n=Rv+@qKOb2GPK1^sb`@SI1csMl*=OZB~f@7G{{m>-BblCQ9Z_7*zH5u&4V44)uuLz zhtD8QjV2(q&gaRQ40%V#GO~+5-Em!^A9X4RL(I&Z83qgDJ{Q(l z_4W*o^C-e22)G7fN!SmuB|IgZgmtJNW}kc-8uHO1)uZrlbhb>WDp83{9PsnNtt!}B zLNp>@a@e5fOI4OHsfzrCrpgyJoBZ!WyZkRTNB&%O$p2KG@s&%1Kvf4E%oXPR67 zv@lOTDt}_n&r|Ie<&TSs;q863M1D^#Ro~UhaV$G=UCPUWvA%T3yrpYHzbFDuVI+GQ zr;Urn{=a3}g<%_7W(O&|?B`E)HCCYE0d0Xcv8EWS>=FL=F0W_4NW6l?0$$Z!mfwtO z@Z3>RsYckVaFnWaL{XCx`VD`{SCzTB``3hw@{=1%`S7}v%d)plfQV`ua9Gvz1=Nc* z0Ya;~r@9McNfgX8>VgvQNKd3(c!^j{+1CG(f6>z?OrBLT zZ{2ICzku)Ut;;_ey>D11{kT2boH6^p5cNkA>=dX7xGpyelu4`? z!OfF>t+jLJvEL!__Xyc$e8gYuiGvAJrX>$ z9}dZQUf1FLlSsa^z@lpQdgd1!U$v8X=384=$UWJ{z-PyksK#!c7DdW|92FUj80;+^ zPUN(Auq6`FN>2KHWIM0uU$R)_!adM6=7KY=?iW&>{xQ`biTPpC-{Rr^mWs!K;oUFc z3RGLJn$H){^C$ZE6wbwiX$!-bm;rwCZLcDfcWm#h@q#oQLBKNhGi)kwA{N;P9B(BI ze)smiH5e_XTZro{2xppM1swsWSs02&qNXcm<&fHE9ew#2|fQyMfPfTaPKv3H;RCsA4Lo*WQ7)?Sty+k zR~5&=gk@^N%|Z0|A_cZ3p0czW7jY4vtoAZH;muX$PL>8TY|n?q_Wa-)Q|`)McoPnA z;-g-q7bpY?5U}>?Oh}{atuLelT-mjPyuh1x`Ae}T1Qu>o+@oxe5ALcd`X5{LR({{E z^z0Geg+{m$vYm1>Z@<3VI=(7azu=s=K5x!?@BUDX!oLCGP%Ny&iImR%B`|@$z!a6I zsdX$L1WZRTI0_Xu7-SPTX?Z8ZV1?(-fUO@d$Vd+X1)Do}diw)8?9cGZ-mqw!`U5iX zbnyEJ?q1|c8Ifo*c(Anz3Q!YPbr)~Cp}bpsL7qja^9bS}hRwRpCMd}a1_9_ zLnyehQhrL*reo70aU4IiA#YW zgqR0z7ja-QRK(AEVwqH2x%`v8?aO7TWaL!g-H+9@rt)f_I zBKN%oA&^Yd0(0N)zJ2fQ?%TI--`%J4eqHK+{ z8ufE+^HQYUXqfAOveT%W3(>N95?O1soLvXc`f4fC^{7O>wEUVx%Z=`sjpH+Iktcsk(;8U1uE zJnK%wpkBJ3cEO-KX*b;fs&~;x=td}ayEnNvFR1MS6(e+uF+jH(gLIp*nf8LDMz_;G zklaJ}(jBxP1|6b1=`N5RHb&^~bMnUttIYf4d*np<5yQ-+Q>Jk+O^rlIwO5Rfjt?Ht zkBl5XIx;q{4~-rTkK0uT_8l8pMyn{Y92guQ2@jpLJqHbo#bf5+p?%?T-mJdptb!e8 zRs3}|q(!Hu^sH$xeot*9AK?v-$oiF{cPRw0SqbnYJf;euNnV`nXB_=CTZ@Es2n^`pagH?d z`xOo4Xuy!8FsU89+238ZNRChVYsrsz*55?-<)8N}WNWFjR1*fSh(%$~bmLOY$XM}o z%1vNXWKgbYY=8c(%4%82-X^wUS}Y{9UC?JcfnT>y|LR;5akDVjR$d?2)361Zm83bv z#&Et)h6J_>A;(z?1CZR6C|Gu`3ba7JG zMpFE2e?xw#E+jV(BUe3w7r_L8T~9Opd^XCg+X}1YPdC)87Ibwon~D`?Ax!PM$fB8y z!A|n8HcWJ0Lix)8w~0^#g&+x5H#K5u7PZpsIDfQp^8ii>7Kdut41$>BZ(*&li>6`e z(M(*wU@#MQF^opTqvdt0b?ga=fBSHK{%;!ZDedP`26`pN0PKpQE1xl<6lB>N?CRxr z*2J5|P7{5Z9c_32l#wznW!SD4c&=G@3X|;PKWMJ)>OkQ>geg?+VA0f+vB+E&c4YxV z3cJKRTG}cKwnj3V;s&2+ITf6S+HINr4pMI-Ei9rG`UPjnEKe8N zZ4ZN0>G4!1Yh5f_#8;uVv1jtbt?xPgOO3&<@ZIay^%N`sI;P#pxM{{yQ++|$*juQT z1K92}mX*NxVwAPl(3_-&|hK~(sL+NGV4p&6yD}rn2b>{?H7xt4Xji%?DN28 ztBLfKX?tSnM8b$+Sm9sj-nr8SUB#aU6>KZ;m<<5erR(Ind|kP&UUyt~PP*XMm#=9v zuAFNO+$H)g{$%Tc{Pc!0W9f;bd?Bssi@()Yf|H14zA`OY%V9S z@!#yM!vDo+>~b&%acf2QI(T8*^r_7blOQ+-+ZN{Cd6m%c(Kf%kkj5=KGD>qzR-?Dl3Wg_AK zq;&q}^Ph`Da$>FGgqB;?FGx`OteUg6FG#ck$~q|hFRDPT6l1#P3Dm1ZeMMZFmth_O zL0fsl0X?g3I035*^Aq%%oCheixiY}IoENZuHirruEM-oemvW989JCQy4rn<=Yjt4& zp`hu80(nEdpakbhick&sA<>~N*Cau06%(nSmxjUOoaAIGWhIc=M5UBsHRR;0B&X%v zwC$71OJt_PI?`yZhRu;@{5k*GrW+(zLECT2`y?toCe6S)DXEpHBGQ$KqHRIlM8Uc< zN=||8?+CL+N0tBG)~2#&VcQGw@MHYJ*6N8X*v6x4leM;XCT&`&XwtCdv}r4`Om@*L-VuT*K2I(MH!`Z%wfh5Q-(04$n0h4 zv^AQJWs{J8go13Cztp?e`6*Q9<#&2(X2k_0E`yID5#x&Ou(Fwi!NN$u`chq)6 ze2N*SnP#>t8I7HZr!<2#qVfnovAuBx1i?~7D+Dh7-1bg#g8%vUv02P(Y~@0f37hL@ z&2-Av;2PAUmc{I{IHaY_g4h?QAWH2m{V^mWccLnX2x_yJV~yV{9|$PZ)}^nB=)BS|LD&#u>zx*FsrB79VA^&@z zTE;Qn^)%vGGVwm}xnQghHJK0Y*fT6*vx|KXCypD;e(kcBM(LxtEGGy!qLNigDusPYq5B_jK(v${0Ou}d(z=z_lj+n{x*x|-p}VG5jkC8Q+i7&CaTq1~ zCgFD)oAQ6$|20A~{EK_4`O~{@&MrEP$VSBF2$K?*;afmr7}g6Z5~gl;V0iR+SO=fd z4-Uec!$^2AJTxNwR@{uY)kQ93qbS~ekOK&FiL_|(W`pu?^A~r=>xEHCforMR-r{Qp zh8j?)sHJ3$f5HI*{K`OU#aD|f_j=S?_71L`r1Rey_{YL(U+G=C(rM@EC=JFl>^V>o z@-AO5D3jO_f|vj9(54nVmxY`E`&jz{!s2yE(tZAup+qe%gNa!!UBu7$`QhG`LgI%H zWfx}u<>A_j_ZQ8+iM^Zu=i%=<3CsWK{&jL>>BQkp;Wf};I}2%^2xJfoeskClpqA_FIk+mkfdLcR;;RE4zrwG3jJQ1!V@A2#5jSa7W#E(CO zU#O{ImQo;kgMT-CqUxbr*p3*yJXv6)U8AeX6n}iQuNeaxBn|NLz+T6y0{N_P4={2e zxbRm;hqj<=*yTdpU^-+WSwnaVdmcJMZo4X)NTe_7@yXKr4ErA6c&M)HePAerlb>LX zBm66ZSpCvX8RnM`J+({tmd3mUn1maaV@>G!-&8E!xjnr9@U~MztTc)kRLBY~LbFg> za!ntOAr7iXY>p1^07e7&yLbcgG<%7c)XC}q^Yd>VZX~n$_YZ$4?^wFmwkyNp@aD?I z^$0sAE{a>Q*rLal-d`-X)8V7Em z1u#JQJZdYeg^n0B9{!*$;n^F4$Mgb-+)4K+U_2E(1YZ~o>x_{#`7`_1^102G{FO1cuLhJYB$v`8!F0010VFp(@X1Ru-Z1mV8!uL9$dR9cP_a2~}^(JyIdf`9$xG%%5Ddf_Dy8&RQcXycR+^Y7Wt!^ zlXXwZCwj`DXKzlSO06WQ3Uy$49P%X<{#TWPN+PAoHZG_)PhKR&9FstjR?Mha^5Iy? z!dQ?^&0;#`T#`<~I>1RVTd*}FMKft|9)`!c=q`S!FG?xI!F~g2Uqe{rOxVPeD9$j1 zoQG)vBn}0mv$Mi;o+B;y56R99AW&y%i`|=B^(|f z4~JarUnCff{a8@Kd&40aIQH@l$Df~FnYH2xQ@^mO8&cM8w(z}AT)=kIlD?v?N_Jx9 zppf|xDiFkW|0UMowMAlUp=LWNhUT>GG}EzjhGlD+MAQOvC)p>FasOS!`v?yZjv`zI z00H=3n3`f$xZbiUIARN#Ir}T@c4S4e_YLIVgz%=syHC{3eiZ8>;jP442ZH!?*M~I) z=d}-O`w=1tDTF5wogo=&F|>}_nngYYgA74UJ}F_}n5E#^mU7*oWih>uf` zFUs6Y8?5yw(-gAkUBD}ucwkg1(C$^>4`JC4PYuyD$Sp;FDN3v<3vER!)4-v<3SV)u dS61)y!zWicJ@=(q&CRnX+gv@KjZRs8@IQ$cq3-|y diff --git a/utils_v2/api/__pycache__/codes.cpython-310.pyc b/utils_v2/api/__pycache__/codes.cpython-310.pyc index b7c160ce6cef69a159f0d5edb101828a12618d25..343b95a84e6c004ddf41f7282f2258f95604932a 100644 GIT binary patch delta 34 ocmew(^GAj|pO=@50SNXt{ocrZmV+ZRuOzi7FEMBH1CBYY0Lgz0Jpcdz delta 34 ocmew(^GAj|pO=@50SL~m|Gts?EC)wodU||maca@#2OM))0nF|UWdHyG diff --git a/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc b/utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc deleted file mode 100644 index 29709178bcf58cf001a95fe407d73c1a69863c0a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3424 zcmZWrTW=f372X@kB~4u{$+7IZ#U@Q#5O65hNQ=OA-Ka8sNgPTM=^7!>E>=4ua^=0y zGrN*SD1Zz3H3jd zZz(KAGVY&e}Hi5V9=ffA&D zoq;EIgV6cFx$6b{UNPzJYPU5tzvHEuVBY?9=jMCq!0GT|+5~^&=DUUX`fopIt#u#% zZhB5PaR%NlcaoSp!_f1&vn`Uy*&A@d9m#jOfI<;ZMc_-v!&t;UnF-FEOa^h^L2utl z6ZE*_CvnQ3r%vR>UY`ruEVkWj-D_>O+H0-Ol&A8P*qhrISUGC-(EweJ|_t{Bk7Sy+l04j;`mY!7g{xB=y44uet;7#96e( zg_CSM=|FIgNoP9|;Bk_`7m4J66{p_fbebrMgEYbEXv6wYBs<(s-PntGUY|%7z3L#5 z#SaKtN??X1eirdK_0k}T^IGU_@sR$={MsN*hpynyGLBiHm-5)(cO?g4Ope}uR1A=g z54znAXCf$_AchH)5b0Bf83qZ62xhKL9HvXVdGh0rc=g zPfD_AlJy5pkYeP#^N~+_S(-$!hVO;p{)yu{=oIFkI5j{%b^fN>rAs^PdD2O6!XQ?& zefUm_Gh^9F1gJ%ZT)>^+@NglRZyGL%IAEM?_;Kcm^t!W^rO?!ogCq-?Lmunw1?fO- znZO|hLJ#gjKM-orfQMnBQ2U;50m74qcmr(>O(LIa2l|2LY6k`=Fa|XaOi+_)NI81D zVToBtDI?`oWqUtiJiq3i0qnZM3nX{>b028~4&1PEOY(4gYU4Y{&xmwoCdQBfKo~hhCk@AsJPaopR@&1D*uY>$iPqndrq+kJOD$Ytv$t%g0%07bpUi^^AkBD3Xfu9p{NRZEcH{Bodyh_}z7iM5n zf$Fy9`A=U(wo1r8CwDuGT@km#s5?!J0KkORs>fCO6D>`(C3$&P!R} zb;)E=!m^O~y6Zm6JXH_StGF(z1!C?hHWP0XAy=K!i~>uL;l(0|hbHGiY|SzZ+t3ZY z_MJUjudBbbW#k31>o)X!uFqo>?UbuJBU2x`LGf4ZZw4!UX1uFC#I@{!_Ls|_YN3L%S(Oq@%;BU$6w;pC>NNNg%@L~yzCDW6!^S?i%RSVNnVm6=R-k02DmCnJ9M|1 zf?kE&MHxgHRciO7{QVshH1F-un=n+E0H9=<>H%KhuXMw1nCKUmG%gR)@!vrN(-f5l*9e)dfQRyx!>zgC{qgHqG;acZbqnww9f^IG)SO-_c zTae1jeUGZ2xT!kG45&@8Oeyu`)7R``(T@Lsq@0aB_UfCLp)LFqBCKUv@Wu(es_XYn%{|0xlb!zdcX zJ48r#@g5OUT*0Rl$W4*OQ}~@ma&F%)(!m{aXn7CB((JeDdR@28CDW)=Sfk8+Uj(?~ aN~dFrv+#Fz*?wRfHqA2ZnfaRa_5T5DzkO8z diff --git a/utils_v2/api/__pycache__/response.cpython-310.pyc b/utils_v2/api/__pycache__/response.cpython-310.pyc index bd3d7e5d36062d830d265585bf33c07d4d60aea0..65cf988d17f8177eb3a2679f208d53f3e6fdf3e0 100644 GIT binary patch delta 618 zcmYLGJ#P~+7`E>#cbAW*B^^2-y+~z1Adrw)T7*<8Au&*(z;a7;#BZ9U*YnA~Bcuw- z!o+HctrB~u{saC38_EJ=;WseBdrkol_WL}3{P=n8@2xMLj%>GMM&s%$^pI^a;yla6#5FDUEp8ERW9L#6SZ=L8XEhCr zyM)&*4lUmJx4(#ph(+@~ORfm_1-sE&whvJkGS~P-3d~NQ#z12aPHF^0sbD0hP>vAP zr4IB+R+)g;2-6YfAn*-lvKZ$_u=ngSU@i(d&P#pUIxU;JOv_652V{6)K00TMpyKN0 zoK3d5c-zfG`h4A^X(KYmyTyCv zv$x$?Hu`-b2j+)2TigGuh?(bvz8U#Agw|h0*bsUcezXP}Hc)$Jt DaA>1I delta 348 zcmX|+F-yZh6vy9P?k<<4i7|~sLBXL*Q4j?;7hS~;ic3I(c-M-Ek%r=u(nZ|mz{PK1 z2I&_OoP>htw-6_NIm8p{B8BvhYW6+Lf zitK=*8P`I)TJAvy&^gw5=zPn}a)8{@;olCoE^6-Vj>l3J9Pm@~1&7yyTr3B>>a delta 31 lcmZ3^xSWwYpO=@50SL~m|2~mBks~oZJ-)O!wP<3AF#w8u3GDy? diff --git a/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc b/utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc index fccfeaa6aa18c597811bbdfb7f26b51c366a74b5..2fe27b8afa1e0cf5b136ee7ad16b5ca0b7b95e9b 100644 GIT binary patch delta 174 zcmbQXi>Z4j6JI_rFBbz4-1y9ruK#i)-+We%%)FA+qP)bM&0ATIP2pn%O0og*VIVHn znA|x_oh^kSm_d_e@|#(*5}K?<AO@J{!!glMu9$pPsiB@iJGB9uYIWgt<-s!~!}ka~-)xTGjEFI|(VXbMPb z8AyuTEwdt3zE~k8KQC2LlcVS~Nah!ixFzH3@8|9x@8;>_I{Ec1sd^A2-o-J*(aAB` zHQvwB*R{w5q<95LF>3+P{^ZemsXJX{xD`H>-8Yy#&eR9m~{Y(sulV#^* JGi$P31^{?1S!VzM diff --git a/utils_v2/database/async_mongo_v2.py b/utils_v2/database/async_mongo_v2.py index dc9955a..663ed3a 100644 --- a/utils_v2/database/async_mongo_v2.py +++ b/utils_v2/database/async_mongo_v2.py @@ -1691,42 +1691,43 @@ if __name__ == "__main__": async def main(): # Create an instance of the database connector: - my_fs = AsyncMongoStorage( - connection_string = constants.MONGO_FILE_CONNECTION_STRING, - database_name = constants.MONGO_FILE_DATABASE_NAME, + my_db = AsyncMongo( + connection_string = constants.MONGO_DATA_CONNECTION_STRING, + database_name = constants.MONGO_DATA_DATABASE_NAME, max_connections = 10, debug = True ) # Connect to the database: - await my_fs.connect() + await my_db.connect() - # Keep performing the changes in batches till you have corrections to make: - while True: + # # Get the documents to migrate: + # documents = await my_db.find_many( + # collection = "scriptData", + # filter = {}, + # limit = 50, + # projection = {"_id": False} + # ) + # # print(json.to_string(documents, default = str)) + # + # # Adjust them: + # adjusted_documents = [] + # for document in documents: + # script_id = document.pop("scriptId") + # adjusted_document = { + # "scriptId": script_id, + # "desc": "no desc", + # "content": document + # } + # adjusted_documents.append(adjusted_document) + # print(json.to_string(adjusted_documents, default = str)) + # + # # Insert the adjusted ones to the new collection: + # response = await my_db.insert_many( + # collection = "_scriptData", + # documents = adjusted_documents + # ) + # print("RESPONSE:", response) - # Find all the files that have their metadata as a string: - files = await my_fs.find_many( - filter = { - "metadata": {"$type": "string"} - }, - limit = 10 - ) - - print(my_fs.to_json_string(files)) - break - - # # If no matches were found: - # if not files: break - # - # # Fix the metadata file-by-file: - # for file in files: - # file_id = file["_id"] - # success = await my_fs.replace_metadata_for_one( - # filter = {"_id": file_id}, - # replacement = json.from_string(file["metadata"]) - # ) - # print(file_id, ":", success) - - print("Fixes done!") asyncio.run(main()) diff --git a/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc b/utils_v2/date_time/__pycache__/__init__.cpython-310.pyc index 98444f403cae52560b139e52bd008f625e615c08..b18eeed72587d41f03476a14d8d2c1161d73a892 100644 GIT binary patch delta 31 lcmZ3%xPp;8pO=@50SNXt{hr93$dQ>>l3J9Pm@~1&1OSK93CI8d delta 31 lcmZ3%xPp;8pO=@50SL~m|2~mBks~oZJ-)O!wP<3A2>^<@3Ge^_ diff --git a/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc b/utils_v2/date_time/__pycache__/date_time.cpython-310.pyc index 33397e9cff1b8b78dd0b57f12dd2861a80b5dc42..46643f1d493eea318821fd09b92b5626fe8dc1fc 100644 GIT binary patch delta 34 ocmdm?yF-^dpO=@50SNXt{ocqufsG?GuOzi7FEMBHVm46$0JcmD-2eap delta 34 ocmdm?yF-^dpO=@50SL~m|GtrX0vktSdU||maca@##cZMi0J*yh1poj5 diff --git a/utils_v2/security/__pycache__/__init__.cpython-310.pyc b/utils_v2/security/__pycache__/__init__.cpython-310.pyc index 20af223f36dc7252f9f586c066dc9cdc27368c3b..e6d54d57e6b3d330035d70deae0aadbc4c9e7fbe 100644 GIT binary patch delta 31 lcmZ3^xSWwYpO=@50SNXt{hr93$dQ>>l3J9Pm@~1&7yyTr3B>>a delta 31 lcmZ3^xSWwYpO=@50SL~m|2~mBks~oZJ-)O!wP<3AF#w8u3GDy? diff --git a/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc b/utils_v2/security/__pycache__/sanitizers.cpython-310.pyc index 7193fe2e3d29c047950995bf454e1baa73692883..4768d0027ab733438b6a9bdc9df92b3128bb5a4c 100644 GIT binary patch delta 34 ocmeC=@8svs=jG*M0D}EZzc+G+F>_?*m82HsCFX3-WL97S0G;{?$N&HU delta 34 ocmeC=@8svs=jG*M0D^Ptzi;FYW9CRqPmeDxPA%G;$*jNv0HJ9K@Bjb+ diff --git a/utils_v2/string/__pycache__/__init__.cpython-310.pyc b/utils_v2/string/__pycache__/__init__.cpython-310.pyc index adc32f16cfaab7aa767869c25f8638b352865d55..6e9484d9ba8541790d3f8e9608b1941f9bacffa2 100644 GIT binary patch delta 31 lcmZ3=xRjAQpO=@50SNXt{hr93$dQ>>l3J9Pm@~1&5CDf83BLdU delta 31 lcmZ3=xRjAQpO=@50SL~m|2~mBks~oZJ-)O!wP<3AApnX33FiO+ diff --git a/utils_v2/string/__pycache__/json.cpython-310.pyc b/utils_v2/string/__pycache__/json.cpython-310.pyc index 5b01bd8d2a5d92cf037d81e34e1f65639a9fc964..98048fa9fc1f1614b2af82cfc7960699a45dabe9 100644 GIT binary patch delta 34 ocmca6eNCD>pO=@50SNXt{ocrJ#>|nKSCU$kmzcBJlew1*0JAg-tpET3 delta 34 ocmca6eNCD>pO=@50SL~m|GtsijF}@bJw3j(IJIcACvz_s0JftF)c^nh diff --git a/utils_v2/string/__pycache__/regex.cpython-310.pyc b/utils_v2/string/__pycache__/regex.cpython-310.pyc index ad9ed17d7bf216843ff67105ae9c196465a52124..1a93e0b52d4ff05e2161ccbee6ad6f80a530d721 100644 GIT binary patch delta 34 ocmdmEy2q3|pO=@50SNXt{ocs^kAovKuOzi7FEMAcAm?g90KY&BZ2$lO delta 34 ocmdmEy2q3|pO=@50SL~m|Gts?9|uQbdU||maca?KLC)2J0K%^el>h($ diff --git a/utils_v2/system/__pycache__/__init__.cpython-310.pyc b/utils_v2/system/__pycache__/__init__.cpython-310.pyc index dac9ac61882c9c364e3a787b23fedc1f72ecf63d..403cde5605ddd4fd5b002327bbc55a9bdff565a5 100644 GIT binary patch delta 31 lcmZ3=xRjAQpO=@50SNXt{hr93$dQ>>l3J9Pm@~1&5CDf83BLdU delta 31 lcmZ3=xRjAQpO=@50SL~m|2~mBks~oZJ-)O!wP<3AApnX33FiO+ diff --git a/utils_v2/system/__pycache__/files.cpython-310.pyc b/utils_v2/system/__pycache__/files.cpython-310.pyc index 3755755f00ea3a3a830c14d7c20500f869cf3189..f3e7a716c242d55fcc9f0b65ee79f4d3480f7f5c 100644 GIT binary patch delta 34 ocmeyW^;L^IpO=@50SNXt{ocs^jfo>OuOzi7FEM8`Cv%Pv0LiHfZvX%Q delta 34 ocmeyW^;L^IpO=@50SL~m|Gts?8xu!jdU||maca?KPUajT0L>T+mjD0&