(20241008) logging decorator and model improved.
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
|
||||
# GIT CHEATSHEET
|
||||
## By Sharvil Sir 🙏 (20240927)
|
||||
Perform these steps on creating a new project.
|
||||
|
||||
_**ASSUMPTION:** You are already in your project directory in the command line terminal._
|
||||
|
||||
---
|
||||
|
||||
### STEP 0.
|
||||
#### Initialize Git in your new project:
|
||||
Do this when creating a new project directory.
|
||||
Do this **ONLY ONCE**.
|
||||
```commandline
|
||||
git init
|
||||
```
|
||||
|
||||
### STEP 1.
|
||||
#### Create a `.gitignore` file:
|
||||
Add files/directories in it which you don't want to sync to git.
|
||||
Update this as frequently as your project needs you to. Start with this:
|
||||
```commandline
|
||||
/.venv/
|
||||
/.idea/
|
||||
**/__pycache__/
|
||||
*.pem
|
||||
```
|
||||
|
||||
### STEP 2.
|
||||
#### Create a repository on Gitea (web UI):
|
||||
Do this **ONLY ONCE**.
|
||||
Open the following URL: https://wtt.ditscentre.in/ and sign in.
|
||||
- Ensure that the owner is `ditscentre` (img 0).
|
||||
- Give your repository a name. Preferably keep it the same as the project name that you made locally (img 0).
|
||||
- Type in a brief description of your project (img 0).
|
||||
|
||||
|
||||
|
||||
- Ensure that the default branch is `master` (img 1).
|
||||
- Create the repository (img 1).
|
||||
|
||||
|
||||
|
||||
### STEP 3.
|
||||
#### Add files and directories to the staging area:
|
||||
The following command adds everything to the staging area.
|
||||
The '.' is important, it refers to the current directory.
|
||||
```commandline
|
||||
git add .
|
||||
```
|
||||
|
||||
### STEP 4.
|
||||
#### Commit the changes to local git:
|
||||
This step commits all the added (staged) changes to the local git instance with the provided comment.
|
||||
```commandline
|
||||
git commit -m "you comment here..."
|
||||
```
|
||||
|
||||
### STEP 5.
|
||||
#### We push the local commits to the repository:
|
||||
The format of the command is:
|
||||
```
|
||||
git push -u <repo_url> <branch_name>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## For Python Devs: How to add a common subtree (like `utils_v2`)
|
||||
|
||||
The format of the command is:
|
||||
```
|
||||
git subtree add --prefix=<local_dir> <subtree_repo_url> <branch_name> --squash
|
||||
```
|
||||
Before you start: ensure that you have committed all pending changes. Then run the following command **ONLY ONCE**:
|
||||
```commandline
|
||||
git subtree add --prefix=utils_v2 https://wtt.ditscentre.in/ditscentre/utils_v2.git master --squash
|
||||
```
|
||||
This will create a new directory named `utils_v2` in your project.
|
||||
**Do NOT change the files in it.** Just import and use them in your scripts.
|
||||
When you need to get the latest updates to these files, you must run a modification of the previous command:
|
||||
```commandline
|
||||
git subtree pull --prefix=utils_v2 https://wtt.ditscentre.in/ditscentre/utils_v2.git master --squash
|
||||
```
|
||||
@@ -0,0 +1,406 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Wednesday, 28th Aug., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To be able to fetch setup credentials and data for any project.
|
||||
This could include things like default values, URLs to assets, etc.
|
||||
While 'data' and 'cred' can have anything held in them, the idea behind giving two services is for the user of
|
||||
this service to be able to organise his setup variables.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
NOTES:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For using Quart:
|
||||
from quart import Blueprint, current_app
|
||||
|
||||
# My utils:
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.api.response import ResponseModel
|
||||
from utils_v2.api.async_quart import (
|
||||
read_input,
|
||||
log_request_to_mongo,
|
||||
should_not_be_under_maintenance,
|
||||
only_whitelisted_ips,
|
||||
limit_rate,
|
||||
validate_input
|
||||
)
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Related to Quart:
|
||||
cred_and_data_bp = Blueprint("int_cnd", __name__)
|
||||
get_api_version = "2.0.0"
|
||||
set_api_version = "2.0.0"
|
||||
update_api_version = "2.0.0"
|
||||
delete_api_version = "2.0.0"
|
||||
|
||||
# related to the operations of this blueprint:
|
||||
JSON_TYPE_INFO = {
|
||||
"cred": {
|
||||
"collection": "scriptCred"
|
||||
},
|
||||
"data": {
|
||||
"collection": "scriptData"
|
||||
}
|
||||
}
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
@cred_and_data_bp.record_once
|
||||
def init(blueprint_setup_state):
|
||||
|
||||
# This gets called when the blueprint is registered.
|
||||
# Consider this to be a one-time setup for the whole blueprint:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/set", methods = ["POST", "GET"])
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = "internalCredData",
|
||||
operation = "set",
|
||||
api_version = set_api_version,
|
||||
log_input = False,
|
||||
log_output = True
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
async def set_data(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
log_id: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
To set the credentials for a particular script. If the document exists, it will be overwritten. If the document
|
||||
doesn't exist, it will be created.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators from 'async_quart_utils.py'.
|
||||
:param inbound_data: auto-extracted by the decorators from 'async_quart_utils.py'.
|
||||
:param inbound_files: auto-extracted by the decorators from 'async_quart_utils.py'.
|
||||
:param log_id: An identifier for the logs (if logging is enabled).
|
||||
:return: A standard response structure from the function in 'async_quart_utils.py'.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid url segment '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Pre-process the inbound data:
|
||||
inbound_data = inbound_data or {}
|
||||
inbound_data["scriptId"] = inbound_headers["X-Script-Id"]
|
||||
|
||||
# Make an attempt to set the credentials:
|
||||
success = await current_app.mongo.replace_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
replacement = inbound_data,
|
||||
upsert = True
|
||||
)
|
||||
|
||||
# Return the response:
|
||||
if success: return ResponseModel(api_version = set_api_version, status_code = StatusCodes.OK)
|
||||
else: return ResponseModel(api_version = set_api_version, status_code = StatusCodes.FAILED)
|
||||
|
||||
# In case the client terminates the connection prematurely:
|
||||
except asyncio.CancelledError as exception:
|
||||
current_app.printer(exception)
|
||||
return ResponseModel(
|
||||
api_version = set_api_version,
|
||||
status_code = StatusCodes.CLIENT_CLOSED_REQUEST
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/get", methods = ["POST", "GET"])
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = "internalCredData",
|
||||
operation = "get",
|
||||
api_version = get_api_version,
|
||||
log_input = True,
|
||||
log_output = False
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
async def get_data(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
log_id: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
To retrieve the credentials stored for a specific script. The script's id can be anything set by the programmers.
|
||||
The idea is to have only the script's id stored in the script, and everything else is fetched from the database.
|
||||
This means that we get to store and update everything from one central location.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators from 'async_quart.py'.
|
||||
:param inbound_data: auto-extracted by the decorators from 'async_quart.py'.
|
||||
:param inbound_files: auto-extracted by the decorators from 'async_quart.py'.
|
||||
:param log_id: An identifier for the logs (if logging is enabled).
|
||||
:return: A standard response structure from the function in 'async_quart.py'.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid url segment '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Make an attempt to retrieve the credentials:
|
||||
cred_json = await current_app.mongo.find_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
projection = {"_id": False, "scriptId": False}
|
||||
)
|
||||
|
||||
# In case no result was found:
|
||||
if cred_json is None:
|
||||
return ResponseModel(
|
||||
api_version = get_api_version,
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = "invalid script id"
|
||||
)
|
||||
|
||||
# Successfully retrieved:
|
||||
return ResponseModel(
|
||||
api_version = get_api_version,
|
||||
status_code = StatusCodes.OK,
|
||||
data = cred_json
|
||||
)
|
||||
|
||||
# In case the client terminates the connection prematurely:
|
||||
except asyncio.CancelledError as exception:
|
||||
current_app.printer(exception)
|
||||
return ResponseModel(
|
||||
api_version = get_api_version,
|
||||
status_code = StatusCodes.CLIENT_CLOSED_REQUEST
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/update", methods = ["POST", "GET"])
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = "internalCredData",
|
||||
operation = "update",
|
||||
api_version = update_api_version,
|
||||
log_input = False,
|
||||
log_output = True
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
async def update_cred(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
log_id: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
To update values of certain fields for a credentials document. It only updates existing values, does NOT add a new
|
||||
document if the document doesn't already exist.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators from 'async_quart_utils.py'.
|
||||
:param inbound_data: auto-extracted by the decorators from 'async_quart_utils.py'.
|
||||
:param inbound_files: auto-extracted by the decorators from 'async_quart_utils.py'.
|
||||
:param log_id: An identifier for the logs (if logging is enabled).
|
||||
:return: A standard response structure from the function in 'async_quart_utils.py'.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid url segment '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Pre-process the inbound data:
|
||||
inbound_data = inbound_data or {}
|
||||
|
||||
# Prepare the update JSON:
|
||||
update_json = {}
|
||||
if inbound_data.get("unset"): update_json["$unset"] = current_app.mongo.dict_to_dot_notation(inbound_data["unset"])
|
||||
if inbound_data.get("set"): update_json["$set"] = current_app.mongo.dict_to_dot_notation(inbound_data["set"])
|
||||
|
||||
# Make an attempt to set the credentials:
|
||||
success = await current_app.mongo.update_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
update = update_json,
|
||||
upsert = False
|
||||
)
|
||||
|
||||
# Return the response:
|
||||
if success: return ResponseModel(api_version = update_api_version, status_code = StatusCodes.OK)
|
||||
else: return ResponseModel(api_version = update_api_version, status_code = StatusCodes.FAILED)
|
||||
|
||||
# In case the client terminates the connection prematurely:
|
||||
except asyncio.CancelledError as exception:
|
||||
current_app.printer(exception)
|
||||
return ResponseModel(
|
||||
api_version = update_api_version,
|
||||
status_code = StatusCodes.CLIENT_CLOSED_REQUEST
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/delete", methods = ["POST", "GET"])
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = "internalCredData",
|
||||
operation = "delete",
|
||||
api_version = delete_api_version,
|
||||
log_input = True,
|
||||
log_output = True
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
async def delete_data(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
log_id: str = None
|
||||
):
|
||||
|
||||
"""
|
||||
To delete a document for a particular script id.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators from 'async_quart.py'.
|
||||
:param inbound_data: auto-extracted by the decorators from 'async_quart.py'.
|
||||
:param inbound_files: auto-extracted by the decorators from 'async_quart.py'.
|
||||
:param log_id: An identifier for the logs (if logging is enabled).
|
||||
:return: A standard response structure from the function in 'async_quart.py'.
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid url segment '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Make an attempt to set the credentials:
|
||||
success = await current_app.mongo.delete_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
)
|
||||
|
||||
# Return the response:
|
||||
if success: return ResponseModel(api_version = delete_api_version, status_code = StatusCodes.OK)
|
||||
else: return ResponseModel(api_version = delete_api_version, status_code = StatusCodes.FAILED)
|
||||
|
||||
# In case the client terminates the connection prematurely:
|
||||
except asyncio.CancelledError as exception:
|
||||
current_app.printer(exception)
|
||||
return ResponseModel(
|
||||
api_version = delete_api_version,
|
||||
status_code = StatusCodes.CLIENT_CLOSED_REQUEST
|
||||
)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,381 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Created: Wednesday, 28th Aug., 2024
|
||||
Updated: Tuesday, 8th Oct. 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To be able to fetch setup credentials and data for any project. This could include things like default values,
|
||||
URLs to assets, etc. While 'data' and 'cred' can be used interchangeably, the idea behind giving two services is
|
||||
for the user of this service to be able to organise his setup variables.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
NOTES:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For using Quart:
|
||||
from quart import Blueprint, current_app
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.api.response import ResponseModel
|
||||
from utils_v2.api.async_quart import (
|
||||
set_api_version,
|
||||
read_input,
|
||||
log_request_to_mongo,
|
||||
should_not_be_under_maintenance,
|
||||
only_whitelisted_ips,
|
||||
limit_rate,
|
||||
validate_input,
|
||||
handle_cancelled_request
|
||||
)
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Related to Quart:
|
||||
cred_and_data_bp = Blueprint("int_cnd", __name__)
|
||||
|
||||
# related to the operations of this blueprint:
|
||||
JSON_TYPE_INFO = {
|
||||
"cred": {
|
||||
"collection": "_scriptCred"
|
||||
},
|
||||
"data": {
|
||||
"collection": "_scriptData"
|
||||
}
|
||||
}
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
@cred_and_data_bp.record_once
|
||||
def init(blueprint_setup_state):
|
||||
|
||||
# This gets called when the blueprint is registered.
|
||||
# Consider this to be a one-time setup for the whole blueprint:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/set", methods = ["POST", "GET"])
|
||||
@set_api_version(api_version = "2.1.0")
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
project = "internal",
|
||||
log_type = "credData",
|
||||
operation = "set",
|
||||
log_input = False,
|
||||
log_output = True
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id", "X-Script-Desc"])
|
||||
@handle_cancelled_request()
|
||||
async def set_data(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
**kwargs
|
||||
):
|
||||
|
||||
"""
|
||||
To set the credentials for a particular script. If the document exists, it will be overwritten. If the document
|
||||
doesn't exist, it will be created. Ideally use this for only the first time setup.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators.
|
||||
:param inbound_data: auto-extracted by the decorators.
|
||||
:param inbound_files: auto-extracted by the decorators.
|
||||
:param kwargs: Any number of extra inputs supplied by the decorators.
|
||||
:return: A standard response structure.
|
||||
"""
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid path '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Construct the document:
|
||||
description = inbound_headers.get("X-Script-Desc", "")
|
||||
if len(description) > 200: description = description[:200]
|
||||
document = {
|
||||
"scriptId": inbound_headers["X-Script-Id"],
|
||||
"desc": description,
|
||||
"content": inbound_data
|
||||
}
|
||||
|
||||
# Make an attempt to set the credentials:
|
||||
success = await current_app.mongo.replace_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
replacement = document,
|
||||
upsert = True,
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Return the response:
|
||||
if success: return ResponseModel(status_code = StatusCodes.OK)
|
||||
else: return ResponseModel(status_code = StatusCodes.FAILED)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/get", methods = ["POST", "GET"])
|
||||
@set_api_version(api_version = "2.1.0")
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
project = "internal",
|
||||
log_type = "credData",
|
||||
operation = "get",
|
||||
log_input = True,
|
||||
log_output = False
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
@handle_cancelled_request()
|
||||
async def get_data(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
**kwargs
|
||||
):
|
||||
|
||||
"""
|
||||
To retrieve the credentials stored for a specific script. The script's id can be anything set by the programmers.
|
||||
The idea is to have only the script's id stored in the script, and everything else is fetched from the database.
|
||||
This means that we get to store and update everything from one central location.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators.
|
||||
:param inbound_data: auto-extracted by the decorators.
|
||||
:param inbound_files: auto-extracted by the decorators.
|
||||
:param kwargs: Any number of extra inputs supplied by the decorators.
|
||||
:return: A standard response structure.
|
||||
"""
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid path '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Make an attempt to retrieve the credentials:
|
||||
cred_json = await current_app.mongo.find_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
projection = {"_id": False, "scriptId": False},
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# In case no result was found:
|
||||
if cred_json is None:
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = "invalid script id"
|
||||
)
|
||||
|
||||
# Successfully retrieved:
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.OK,
|
||||
data = cred_json["content"]
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/update", methods = ["POST", "GET"])
|
||||
@set_api_version(api_version = "2.1.0")
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
project = "internal",
|
||||
log_type = "credData",
|
||||
operation = "update",
|
||||
log_input = False,
|
||||
log_output = True
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
@handle_cancelled_request()
|
||||
async def update_cred(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
**kwargs
|
||||
):
|
||||
|
||||
"""
|
||||
To update values of certain fields for a credentials document. It only updates existing values, does NOT add a new
|
||||
document if the document doesn't already exist.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators.
|
||||
:param inbound_data: auto-extracted by the decorators.
|
||||
:param inbound_files: auto-extracted by the decorators.
|
||||
:param kwargs: Any number of extra inputs supplied by the decorators.
|
||||
:return: A standard response structure.
|
||||
"""
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid path '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Pre-process the inbound data:
|
||||
inbound_data = inbound_data or {}
|
||||
|
||||
# Prepare the update JSON. Pre-process the fields to set and unset.
|
||||
# Our actual data/cred are held inside a field called "content", so we must wrap the request in that:
|
||||
update_json = {}
|
||||
if inbound_data.get("unset"):
|
||||
update_json["$unset"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["unset"]})
|
||||
if inbound_data.get("set"):
|
||||
update_json["$set"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["set"]})
|
||||
|
||||
# Make an attempt to set the credentials:
|
||||
success = await current_app.mongo.update_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
update = update_json,
|
||||
upsert = False,
|
||||
raise_exception = True
|
||||
)
|
||||
|
||||
# Return the response:
|
||||
if success: return ResponseModel(status_code = StatusCodes.OK)
|
||||
else: return ResponseModel(status_code = StatusCodes.FAILED)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@cred_and_data_bp.route("/<json_type>/delete", methods = ["POST", "GET"])
|
||||
@set_api_version(api_version = "2.1.0")
|
||||
@read_input(sanitize_headers = True, sanitize_data = True)
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
project = "internal",
|
||||
log_type = "credData",
|
||||
operation = "delete",
|
||||
log_input = True,
|
||||
log_output = True
|
||||
)
|
||||
@should_not_be_under_maintenance(attr_name = "is_under_maintenance")
|
||||
@only_whitelisted_ips(attr_name = "whitelisted_ips")
|
||||
@validate_input(mandatory_header_keys = ["X-Script-Id"])
|
||||
async def delete_data(
|
||||
json_type: str = None,
|
||||
inbound_headers: dict = None,
|
||||
inbound_data: dict = None,
|
||||
inbound_files: dict = None,
|
||||
**kwargs
|
||||
):
|
||||
|
||||
"""
|
||||
To delete a document for a particular script id.
|
||||
:param json_type: The choice from one of the fields of 'JSON_TYPE_INFO'.
|
||||
:param inbound_headers: auto-extracted by the decorators.
|
||||
:param inbound_data: auto-extracted by the decorators.
|
||||
:param inbound_files: auto-extracted by the decorators.
|
||||
:param kwargs: Any number of extra inputs supplied by the decorators.
|
||||
:return: A standard response structure.
|
||||
"""
|
||||
|
||||
# If an invalid choice was made:
|
||||
if json_type not in JSON_TYPE_INFO.keys():
|
||||
return ResponseModel(
|
||||
status_code = StatusCodes.FAILED,
|
||||
message = f"invalid path '{json_type}'",
|
||||
http_code = HttpCodes.BAD_REQUEST
|
||||
)
|
||||
|
||||
# Make an attempt to set the credentials:
|
||||
success = await current_app.mongo.delete_one(
|
||||
collection = JSON_TYPE_INFO[json_type]["collection"],
|
||||
filter = {"scriptId": inbound_headers["X-Script-Id"]},
|
||||
)
|
||||
|
||||
# Return the response:
|
||||
if success: return ResponseModel(status_code = StatusCodes.OK)
|
||||
else: return ResponseModel(status_code = StatusCodes.FAILED)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
pass
|
||||
+301
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Wednesday, 28th Aug., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
This is the central location for the Quart module.
|
||||
We define the app here, and import and attach all blueprints here.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# For system level activities:
|
||||
import gc
|
||||
import os
|
||||
|
||||
# For using Quart:
|
||||
from quart import Quart, request, current_app
|
||||
from quart_cors import cors
|
||||
|
||||
# To make REST-API calls:
|
||||
import httpx
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.api import async_quart
|
||||
from utils_v2.database.async_mongo_v2 import AsyncMongo
|
||||
from utils_v2.api.async_quart import (
|
||||
read_input,
|
||||
log_request_to_mongo,
|
||||
should_not_be_under_maintenance,
|
||||
only_whitelisted_ips,
|
||||
limit_rate,
|
||||
validate_input
|
||||
)
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
# Other blueprints:
|
||||
from api.llm.chat_completion import llm_chat_bp
|
||||
|
||||
# LangChain-related:
|
||||
from langchain_core.prompts import ChatPromptTemplate
|
||||
from langchain_core.messages import HumanMessage, AIMessage, SystemMessage
|
||||
from langchain_openai import ChatOpenAI
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Quart related:
|
||||
MODULE_BASE = "ai"
|
||||
APP_VERSION = "1.0.0"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# The Quart app:
|
||||
app = Quart(__name__)
|
||||
app = cors(app)
|
||||
app.register_blueprint(llm_chat_bp, url_prefix = f"/{MODULE_BASE}/llm")
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
@app.before_serving
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = MODULE_BASE,
|
||||
operation = "apiStart",
|
||||
api_version = APP_VERSION,
|
||||
log_input = True,
|
||||
log_output = True
|
||||
)
|
||||
async def app_startup():
|
||||
|
||||
"""
|
||||
To initialize the variables that you would like to use in this module.
|
||||
WARNING: ALL VARIABLES WILL BE INITIALIZED 'n' NUMBER OF TIMES, WHERE 'n' IS THE COUNT OF WORKERS DEPLOYED.
|
||||
SO, IF YOU WANT TO CONNECT TO A DATABASE AND YOU ALLOW A POOL-SIZE OF 10 AND IF YOU DEPLOY 4 WORKERS, YOU WILL END
|
||||
UP WITH 40 CONNECTIONS TO THE DATABASE.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
# Safe-halt mechanism for upgrades (for a single-worker run):
|
||||
current_app.is_under_maintenance = False
|
||||
|
||||
# Debugging:
|
||||
current_app.printer = IceCreamDebugger(prefix = f"{MODULE_BASE} (Q) | ", includeContext = True)
|
||||
if os.environ["DEBUG"] == "True": current_app.printer.disable()
|
||||
|
||||
# To make API calls:
|
||||
max_connections = 5
|
||||
limits = httpx.Limits(
|
||||
max_keepalive_connections = max_connections,
|
||||
max_connections = max_connections,
|
||||
keepalive_expiry = 3600
|
||||
)
|
||||
current_app.http_client = httpx.AsyncClient(limits = limits)
|
||||
|
||||
# Get the credentials and data for this script:
|
||||
script_id = os.environ.get("SCRIPT_ID")
|
||||
response = await current_app.http_client.get(
|
||||
url = r"https://nexcom.ditscentre.in/internal/cred/get",
|
||||
headers = {"X-Script-Id": script_id}
|
||||
)
|
||||
script_cred = response.json().get("data")
|
||||
response = await current_app.http_client.get(
|
||||
url = r"https://nexcom.ditscentre.in/internal/data/get",
|
||||
headers = {"X-Script-Id": script_id}
|
||||
)
|
||||
current_app.script_data = response.json().get("data")
|
||||
|
||||
# To connect to Mongo:
|
||||
current_app.mongo = AsyncMongo(
|
||||
connection_string = script_cred["mongoDb"]["dataDb"]["connectionString"],
|
||||
database_name = script_cred["mongoDb"]["dataDb"]["dbName"],
|
||||
max_connections = script_cred["mongoDb"]["dataDb"]["poolSize"],
|
||||
debug = True if os.environ["DEBUG"] == "True" else False
|
||||
)
|
||||
|
||||
# Remove unwanted/sensitive variables from RAM:
|
||||
del script_cred
|
||||
gc.collect()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.after_serving
|
||||
@log_request_to_mongo(
|
||||
attr_name = "mongo",
|
||||
log_type = MODULE_BASE,
|
||||
operation = "apiStop",
|
||||
api_version = APP_VERSION,
|
||||
log_input = True,
|
||||
log_output = True
|
||||
)
|
||||
async def app_shutdown():
|
||||
|
||||
"""
|
||||
This is called when "app.shutdown()" is called.
|
||||
:return: None.
|
||||
"""
|
||||
|
||||
message = "Shutting down..."
|
||||
current_app.printer(message)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/", methods = ["GET", "POST"])
|
||||
@app.route(f"/{MODULE_BASE}", methods = ["GET", "POST"])
|
||||
async def root():
|
||||
|
||||
"""
|
||||
To check if the service is running or not.
|
||||
Use this to monitor the service from your "watchman" script.
|
||||
:return: only "ok"
|
||||
"""
|
||||
|
||||
return "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/{MODULE_BASE}/debug/<action>", methods = ["POST", "GET"])
|
||||
async def change_debug(action):
|
||||
|
||||
"""
|
||||
Enable or disable debugging for the entire microservice.
|
||||
WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS.
|
||||
:param action: "enable" to allow debugging on the terminal, or "disable".
|
||||
:return: "enabled"/"disabled" if successful, else "ok"
|
||||
"""
|
||||
|
||||
# Enable or disable debugging only if the password matches:
|
||||
action = action.lower()
|
||||
if action == "enable": current_app.printer.enable()
|
||||
elif action == "disable": current_app.printer.disable()
|
||||
return "ok"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@app.route(f"/{MODULE_BASE}/maintenance/<action>", methods = ["POST", "GET"])
|
||||
async def change_maintenance(action):
|
||||
|
||||
"""
|
||||
Enable or disable debugging for the entire microservice.
|
||||
WARNING: NOT RECOMMENDED FOR MULTI-WORKER DEPLOYMENTS.
|
||||
:param action: "enable" to stop taking new requests on the API, or "disable".
|
||||
:return: "enabled"/"disabled" if successful, else "ok"
|
||||
"""
|
||||
|
||||
# Enable or disable debugging only if the password matches:
|
||||
action = action.lower()
|
||||
if action == "enable": current_app.is_under_maintenance = True
|
||||
elif action == "disable": current_app.is_under_maintenance = False
|
||||
return "ok"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# To get args from the terminal:
|
||||
import argparse
|
||||
|
||||
# To run the ASGI:
|
||||
import uvicorn
|
||||
from multiprocessing import freeze_support
|
||||
|
||||
# Get the config from the command-line:
|
||||
parser = argparse.ArgumentParser(description = f"Microservice for '{MODULE_BASE}' API.")
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type = int,
|
||||
help = "The no. of threads to spin up for this instance!",
|
||||
default = 2
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type = str,
|
||||
help = "The host for the app. e.g.: '0.0.0.0' or '127.0.0.1'.",
|
||||
default = "127.0.0.1"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--script-id",
|
||||
type = str,
|
||||
help = "The id of this script (will affect the loaded config)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
action = "store_true",
|
||||
help = "Whether, or not, you want to see debugging messages in the terminal.",
|
||||
default = False
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Note down the config;
|
||||
os.environ["SCRIPT_ID"] = args.script_id
|
||||
os.environ["DEBUG"] = str(args.debug)
|
||||
|
||||
# Run the gateway:
|
||||
freeze_support()
|
||||
uvicorn.run(
|
||||
app = "main:app",
|
||||
workers = args.workers,
|
||||
host = args.host,
|
||||
port = 8080
|
||||
)
|
||||
@@ -0,0 +1,6 @@
|
||||
# INSTRUCTIONS TO RENEW CERTIFICATES
|
||||
**Date:** 2024-09-18
|
||||
|
||||
1. Go to [jcdev.ditscentre.in/jcdev](http://jcdev.ditscentre.in/jcdev/)
|
||||
2. Navigate to the respective folders (e.g., `kafka`, `mongo`, etc.).
|
||||
3. Download and replace the certificates to ensure they have the same names as those already present in the `cred` folder.
|
||||
@@ -0,0 +1 @@
|
||||
mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external
|
||||
@@ -0,0 +1 @@
|
||||
mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external
|
||||
@@ -0,0 +1 @@
|
||||
mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile={mongo_ca}&tlsCertificateKeyFile={mongo_cert}&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external
|
||||
@@ -0,0 +1,100 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Saturday, 17th Aug., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To hold constant that will be shared throughout the project.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import os
|
||||
|
||||
# My utils:
|
||||
from utils_v2.system import files
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Directories:
|
||||
PROJECT_DIRECTORY = files.get_parent_directory(files.get_parent_directory(files.get_cwd()))
|
||||
CREDENTIALS_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "cred")
|
||||
MONGO_CREDENTIALS_DIRECTORY = os.path.join(CREDENTIALS_DIRECTORY, "mongo")
|
||||
MODELS_DIRECTORY = os.path.join(PROJECT_DIRECTORY, "models")
|
||||
|
||||
# Files:
|
||||
MONGO_KEYS_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_connection_string.txt")
|
||||
MONGO_KEYS_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_ca.pem")
|
||||
MONGO_KEYS_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_keys_cert.pem")
|
||||
MONGO_DATA_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_connection_string.txt")
|
||||
MONGO_DATA_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_ca.pem")
|
||||
MONGO_DATA_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_data_cert.pem")
|
||||
MONGO_FILE_CONNECTION_STRING_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_connection_string.txt")
|
||||
MONGO_FILE_CA_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_ca.pem")
|
||||
MONGO_FILE_CERT_FILE = os.path.join(MONGO_CREDENTIALS_DIRECTORY, "mongo_file_cert.pem")
|
||||
|
||||
# MongoDB Connection Strings:
|
||||
MONGO_KEYS_CONNECTION_STRING = files.read_file(
|
||||
MONGO_KEYS_CONNECTION_STRING_FILE,
|
||||
mode = "r"
|
||||
).format(
|
||||
mongo_ca = MONGO_KEYS_CA_FILE.replace("/", "%2F"),
|
||||
mongo_cert = MONGO_KEYS_CERT_FILE.replace("/", "%2F")
|
||||
)
|
||||
MONGO_DATA_CONNECTION_STRING = files.read_file(
|
||||
MONGO_DATA_CONNECTION_STRING_FILE,
|
||||
mode = "r"
|
||||
).format(
|
||||
mongo_ca = MONGO_DATA_CA_FILE.replace("/", "%2F"),
|
||||
mongo_cert = MONGO_DATA_CERT_FILE.replace("/", "%2F")
|
||||
)
|
||||
MONGO_FILE_CONNECTION_STRING = files.read_file(
|
||||
MONGO_FILE_CONNECTION_STRING_FILE,
|
||||
mode = "r"
|
||||
).format(
|
||||
mongo_ca = MONGO_FILE_CA_FILE.replace("/", "%2F"),
|
||||
mongo_cert = MONGO_FILE_CERT_FILE.replace("/", "%2F")
|
||||
)
|
||||
|
||||
# MongoDB Databases:
|
||||
MONGO_DATA_DATABASE_NAME = "converse"
|
||||
MONGO_KEYS_DATABASE_NAME = "converse"
|
||||
MONGO_FILE_DATABASE_NAME = "converseStore"
|
||||
|
||||
# Mongo Config:
|
||||
MONGO_DATA_POOL_SIZE = 10
|
||||
MONGO_KEYS_POOL_SIZE = 10
|
||||
MONGO_FILE_POOL_SIZE = 10
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+31
-85
@@ -46,13 +46,8 @@ from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.security import sanitizers
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.api.log import APILogModel
|
||||
from utils_v2.api.response import ResponseModel
|
||||
from utils_v2.api.metrics_prometheus import (
|
||||
TOTAL_REQUEST_COUNT,
|
||||
LIVE_REQUEST_COUNT,
|
||||
REQUEST_LATENCY,
|
||||
MetricsAPI
|
||||
)
|
||||
|
||||
# To work with date and time:
|
||||
import time
|
||||
@@ -60,9 +55,6 @@ import datetime
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
import distro
|
||||
import socket
|
||||
import platform
|
||||
|
||||
# For Pydantic data-models:
|
||||
import pydantic
|
||||
@@ -90,11 +82,6 @@ import asyncio
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Info for logging that will stay constant during runtime:
|
||||
SERVER_HOSTNAME = socket.gethostname()
|
||||
PLATFORM_INFO = platform.uname()
|
||||
HOST_OS = distro.name(True)
|
||||
|
||||
# Chars to choose from for random strings:
|
||||
ALPHANUMERIC_CHARS = string.ascii_letters + string.digits
|
||||
|
||||
@@ -572,6 +559,8 @@ def validate_input(
|
||||
def log_request_to_mongo(
|
||||
attr_name,
|
||||
collection: str = "logs",
|
||||
api_version: str = None,
|
||||
project: str = None,
|
||||
log_type: str = None,
|
||||
operation: str = None,
|
||||
log_input: bool = True,
|
||||
@@ -586,6 +575,8 @@ def log_request_to_mongo(
|
||||
:param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the
|
||||
scope of 'current_app'.
|
||||
:param collection: The name of the collection to write the log into.
|
||||
:param api_version: The version code of the API endpoint that is being logged.
|
||||
:param project: The name of the project that the endpoint was built for.
|
||||
:param log_type: A hint to identify what the log was for.
|
||||
:param operation: A hint to identify what was action was being performed.
|
||||
:param log_input: Whether, or not, you would like to log the input that came in.
|
||||
@@ -602,6 +593,9 @@ def log_request_to_mongo(
|
||||
# Let the next in-line decorator know that it has been wrapped:
|
||||
kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1
|
||||
|
||||
# Set the api version as needed:
|
||||
kwargs["api_version"] = kwargs.get("api_version", api_version)
|
||||
|
||||
# Make variables and extract available info.:
|
||||
exception = None
|
||||
response = None
|
||||
@@ -635,7 +629,8 @@ def log_request_to_mongo(
|
||||
elif isinstance(response, tuple): response_to_log, http_code_to_log = response
|
||||
else: response_to_log, http_code_to_log = str(response), 200
|
||||
|
||||
# Try to get the information about the request:
|
||||
# Try to get the information about the request.
|
||||
# There will be no data in any of these if the decorator was used to catch start-up and shut-down events.
|
||||
request_method = None
|
||||
request_url = None
|
||||
request_route = None
|
||||
@@ -653,41 +648,37 @@ def log_request_to_mongo(
|
||||
except: pass
|
||||
|
||||
# Construct the log:
|
||||
# for k in sensitive_keys: kwargs.get("inbound_headers", {}).pop(k, None)
|
||||
# for k in sensitive_keys: kwargs.get("inbound_data", {}).pop(k, None)
|
||||
log_json = {
|
||||
"hostname": SERVER_HOSTNAME,
|
||||
"os": f"{HOST_OS}",
|
||||
"cpu": f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})",
|
||||
"logId": kwargs.get("log_id"),
|
||||
"logChain": kwargs.get("inbound_headers", {}).get("X-Log-Chain"),
|
||||
"log": log_type,
|
||||
"operation": operation,
|
||||
"apiVer": kwargs.get("api_version"),
|
||||
"method": request_method,
|
||||
"url": request_url,
|
||||
"route": request_route,
|
||||
"ts": request_ts,
|
||||
"tat": time.perf_counter() - start_ts,
|
||||
"cpuTime": time.process_time() - cpu_start_ts,
|
||||
"headers": kwargs.get("inbound_headers"),
|
||||
"data": kwargs.get("inbound_data") if log_input else "not logged",
|
||||
"files": {
|
||||
api_log = APILogModel(
|
||||
project = project,
|
||||
log = log_type,
|
||||
operation = operation,
|
||||
apiVer = kwargs.get("api_version"),
|
||||
logId = kwargs.get("log_id"),
|
||||
logChain = kwargs.get("inbound_headers", {}).get("X-Log-Chain"),
|
||||
method = request_method,
|
||||
url = request_url,
|
||||
route = request_route,
|
||||
ts = request_ts,
|
||||
tat = time.perf_counter() - start_ts,
|
||||
cpuTime = time.process_time() - cpu_start_ts,
|
||||
headers = kwargs.get("inbound_headers"),
|
||||
data = kwargs.get("inbound_data") if log_input else "not logged",
|
||||
files = {
|
||||
k: {
|
||||
"name": v["name"],
|
||||
"size": v["size"]
|
||||
} for k, v in kwargs.get("inbound_files", {}).items()
|
||||
},
|
||||
"exception": None if exception is None else describe_exception(exception),
|
||||
"response": response_to_log,
|
||||
"httpCode": http_code_to_log
|
||||
}
|
||||
exception = None if exception is None else describe_exception(exception),
|
||||
response = response_to_log,
|
||||
httpCode = http_code_to_log
|
||||
)
|
||||
|
||||
# Write the log:
|
||||
app_attr = getattr(current_app, attr_name)
|
||||
inserted_id = await app_attr.insert_one(
|
||||
collection = collection,
|
||||
document = log_json
|
||||
document = api_log.model_dump()
|
||||
)
|
||||
|
||||
# Return the response from the wrapped function.
|
||||
@@ -950,51 +941,6 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None):
|
||||
return decorator
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def measure_metrics_for_prometheus():
|
||||
|
||||
"""
|
||||
Use this decorator to automatically measure metrics for using in Prometheus.
|
||||
:return: The decorator factory.
|
||||
"""
|
||||
|
||||
def decorator(func):
|
||||
|
||||
@wraps(func)
|
||||
async def wrapper(*args, **kwargs):
|
||||
|
||||
# Let the next in-line decorator know that it has been wrapped:
|
||||
kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1
|
||||
|
||||
# We use the context of the measurement class:
|
||||
async with MetricsAPI(
|
||||
method = f"{request.method}",
|
||||
endpoint = str(request.url_rule.rule)
|
||||
) as metrics:
|
||||
|
||||
# Invoke the wrapped function:
|
||||
response = await func(*args, **kwargs)
|
||||
kwargs["decorator_count"] -= 1
|
||||
|
||||
# Interpret the HTTP code:
|
||||
if isinstance(response, ResponseModel): _, metrics.http_code = response.for_quart()
|
||||
elif isinstance(response, tuple): metrics.http_code = response[1]
|
||||
else: metrics.http_code = 200
|
||||
|
||||
# Done here:
|
||||
if (
|
||||
kwargs["decorator_count"] == 1 and
|
||||
isinstance(response, ResponseModel)
|
||||
): response = response.for_quart()
|
||||
return response
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
|
||||
@@ -3,15 +3,14 @@
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
Sharvil J Daiya
|
||||
|
||||
DATE:
|
||||
|
||||
Saturday, 28th Sept., 2024
|
||||
Thursday, 12th Sept., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To have one place from where several metrics are measured using easy to use context managers.
|
||||
To have a structure to the response sent from the API calls.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
@@ -31,16 +30,21 @@
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
# System-level activities:
|
||||
import distro
|
||||
import socket
|
||||
import platform
|
||||
|
||||
# To measure time:
|
||||
import time
|
||||
# For data-modelling:
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Any, Optional, List, Literal
|
||||
|
||||
# To capture metrics for Prometheus:
|
||||
from prometheus_client import Counter, Summary, Gauge
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# My utils:
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -50,31 +54,11 @@ from prometheus_client import Counter, Summary, Gauge
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# Define the Prometheus metrics.
|
||||
# FOR THE MICROSERVICE AS A WHOLE:
|
||||
WORKER_COUNT = Counter(
|
||||
name = "ms_workers_active_total",
|
||||
documentation = "The number of threads for the microservice being monitored.",
|
||||
labelnames = ["project_name", "service_name", "host_name"]
|
||||
)
|
||||
|
||||
# Define the Prometheus metrics.
|
||||
# FOR INDIVIDUAL API ENDPOINTS:
|
||||
REQUEST_LATENCY = Summary(
|
||||
name = "http_request_latency_seconds",
|
||||
documentation = "Latency of HTTP requests in seconds.",
|
||||
labelnames = ["method", "endpoint", "http_status"]
|
||||
)
|
||||
TOTAL_REQUEST_COUNT = Counter(
|
||||
name = "http_requests_total",
|
||||
documentation = "Total HTTP requests.",
|
||||
labelnames = ["method", "endpoint", "http_status"]
|
||||
)
|
||||
LIVE_REQUEST_COUNT = Gauge(
|
||||
name = "http_requests_live_total",
|
||||
documentation = "To check if an API endpoint is being served right now.",
|
||||
labelnames = ["endpoint"]
|
||||
)
|
||||
# Info for logging that will stay constant during runtime:
|
||||
SERVER_HOSTNAME = str(socket.gethostname())
|
||||
PLATFORM_INFO = platform.uname()
|
||||
HOST_OS = str(distro.name(True))
|
||||
HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})"
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -94,59 +78,39 @@ LIVE_REQUEST_COUNT = Gauge(
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class MetricsAPI:
|
||||
class APILogModel(BaseModel):
|
||||
|
||||
"""
|
||||
Use this class through its context manager to automatically measure all the metrics in one place.
|
||||
This was originally created to measure the performance of API endpoints made in Quart, but it should work with
|
||||
other frameworks as well.
|
||||
"""
|
||||
# To identify the machine the code is running on.
|
||||
# DO NOT MODIFY THESE:
|
||||
hostname: str = SERVER_HOSTNAME
|
||||
os: str = HOST_OS
|
||||
cpu: str = HOST_CPU
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
method = None,
|
||||
endpoint = None,
|
||||
raise_exception = False
|
||||
):
|
||||
# To identify the project and actions:
|
||||
project: Optional[str] = None
|
||||
log: str
|
||||
operation: str
|
||||
apiVer: Optional[str] = None
|
||||
logId: Optional[str] = None
|
||||
logChain: Optional[str] = None
|
||||
|
||||
# Make provisions for things to note.
|
||||
# NOTE: THESE MUST BE SET FROM OUTSIDE:
|
||||
self.method = method
|
||||
self.endpoint = endpoint
|
||||
self.http_code = None
|
||||
self.__raise_exception = raise_exception
|
||||
# Timing metrics:
|
||||
ts: datetime.datetime
|
||||
tat: float
|
||||
cpuTime: float
|
||||
|
||||
async def __aenter__(self):
|
||||
# To understand the request that came in:
|
||||
method: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
route: Optional[str] = None
|
||||
headers: Optional[Any] = None
|
||||
data: Optional[Any] = None
|
||||
files: Optional[Any] = None
|
||||
|
||||
# Note down the start time immediately:
|
||||
self.__start_ts = time.perf_counter()
|
||||
self.__cpu_start_ts = time.process_time()
|
||||
|
||||
# Note down the metrics:
|
||||
LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1)
|
||||
|
||||
# Setup done:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_value, traceback):
|
||||
|
||||
# Note down the metrics:
|
||||
LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1)
|
||||
|
||||
TOTAL_REQUEST_COUNT.labels(
|
||||
self.method,
|
||||
self.endpoint,
|
||||
self.http_code
|
||||
).inc()
|
||||
|
||||
REQUEST_LATENCY.labels(
|
||||
self.method,
|
||||
self.endpoint,
|
||||
self.http_code
|
||||
).observe(time.perf_counter() - self.__start_ts)
|
||||
|
||||
# Handle the exception as per the user's preference:
|
||||
return False if self.__raise_exception else True
|
||||
# To understand the output that went out:
|
||||
exception: Optional[Any] = None
|
||||
response: Optional[Any] = None
|
||||
httpCode: Optional[int] = None
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
@@ -168,31 +132,8 @@ class MetricsAPI:
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
from prometheus_client import generate_latest
|
||||
my_log = APILogModel(
|
||||
log = "internal"
|
||||
)
|
||||
|
||||
async def simulate_endpoint():
|
||||
|
||||
async with MetricsAPI(
|
||||
method = random.choice(["GET", "POST"]),
|
||||
endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}"
|
||||
) as metrics:
|
||||
|
||||
# Simulate some action on some endpoint:
|
||||
await asyncio.sleep(1.0)
|
||||
|
||||
# Note down the values:
|
||||
metrics.http_code = 200
|
||||
|
||||
async def main():
|
||||
|
||||
print("Simulating endpoints...")
|
||||
tasks = [simulate_endpoint() for _ in range(250)]
|
||||
await asyncio.gather(*tasks)
|
||||
print("Done!")
|
||||
|
||||
print("METRICS:")
|
||||
print(generate_latest().decode())
|
||||
|
||||
asyncio.run(main())
|
||||
print(my_log)
|
||||
@@ -30,9 +30,11 @@
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# For data-modelling:
|
||||
from pydantic import BaseModel
|
||||
from typing import Any, Optional, List
|
||||
|
||||
# My utils:
|
||||
from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
|
||||
|
||||
@@ -64,6 +66,11 @@ from utils_v2.api.codes import StatusCodes, HttpCodes
|
||||
|
||||
class ResponseModel(BaseModel):
|
||||
|
||||
"""
|
||||
A model for how the response should be when developing API endpoints.
|
||||
"""
|
||||
|
||||
# The fields that you want in your response:
|
||||
status_code: StatusCodes
|
||||
message: Optional[str | List] = None
|
||||
data: Optional[Any] = None
|
||||
@@ -74,6 +81,11 @@ class ResponseModel(BaseModel):
|
||||
|
||||
def for_quart(self):
|
||||
|
||||
"""
|
||||
Call this when you are using either Flask or Quart as your framework.
|
||||
:return: The output as expected by Flask and Quart.
|
||||
"""
|
||||
|
||||
# Construct the basic structure:
|
||||
response_dict = {
|
||||
"status": 1 if self.status_code.value[0] else 0,
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -1691,42 +1691,43 @@ if __name__ == "__main__":
|
||||
async def main():
|
||||
|
||||
# Create an instance of the database connector:
|
||||
my_fs = AsyncMongoStorage(
|
||||
connection_string = constants.MONGO_FILE_CONNECTION_STRING,
|
||||
database_name = constants.MONGO_FILE_DATABASE_NAME,
|
||||
my_db = AsyncMongo(
|
||||
connection_string = constants.MONGO_DATA_CONNECTION_STRING,
|
||||
database_name = constants.MONGO_DATA_DATABASE_NAME,
|
||||
max_connections = 10,
|
||||
debug = True
|
||||
)
|
||||
|
||||
# Connect to the database:
|
||||
await my_fs.connect()
|
||||
await my_db.connect()
|
||||
|
||||
# Keep performing the changes in batches till you have corrections to make:
|
||||
while True:
|
||||
# # Get the documents to migrate:
|
||||
# documents = await my_db.find_many(
|
||||
# collection = "scriptData",
|
||||
# filter = {},
|
||||
# limit = 50,
|
||||
# projection = {"_id": False}
|
||||
# )
|
||||
# # print(json.to_string(documents, default = str))
|
||||
#
|
||||
# # Adjust them:
|
||||
# adjusted_documents = []
|
||||
# for document in documents:
|
||||
# script_id = document.pop("scriptId")
|
||||
# adjusted_document = {
|
||||
# "scriptId": script_id,
|
||||
# "desc": "no desc",
|
||||
# "content": document
|
||||
# }
|
||||
# adjusted_documents.append(adjusted_document)
|
||||
# print(json.to_string(adjusted_documents, default = str))
|
||||
#
|
||||
# # Insert the adjusted ones to the new collection:
|
||||
# response = await my_db.insert_many(
|
||||
# collection = "_scriptData",
|
||||
# documents = adjusted_documents
|
||||
# )
|
||||
# print("RESPONSE:", response)
|
||||
|
||||
# Find all the files that have their metadata as a string:
|
||||
files = await my_fs.find_many(
|
||||
filter = {
|
||||
"metadata": {"$type": "string"}
|
||||
},
|
||||
limit = 10
|
||||
)
|
||||
|
||||
print(my_fs.to_json_string(files))
|
||||
break
|
||||
|
||||
# # If no matches were found:
|
||||
# if not files: break
|
||||
#
|
||||
# # Fix the metadata file-by-file:
|
||||
# for file in files:
|
||||
# file_id = file["_id"]
|
||||
# success = await my_fs.replace_metadata_for_one(
|
||||
# filter = {"_id": file_id},
|
||||
# replacement = json.from_string(file["metadata"])
|
||||
# )
|
||||
# print(file_id, ":", success)
|
||||
|
||||
print("Fixes done!")
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user