From 1fd287900de1687b6d7395c5c5c797676611b207 Mon Sep 17 00:00:00 2001 From: khushal Date: Wed, 25 Dec 2024 14:52:07 +0530 Subject: [PATCH] (20241225) Work started on upgrades and server-registration module. --- api_v2/blueprints/cred_and_data/blueprint.py | 46 ++- api_v2/blueprints/logs/blueprint.py | 63 +-- api_v2/main.py | 6 +- controllers/servers/server.py | 394 +++++++++++++++++++ models/logs/api.py | 49 ++- models/servers/core.py | 242 ++++++------ ssh_server.sh | 5 +- to_git.sh | 6 +- 8 files changed, 647 insertions(+), 164 deletions(-) diff --git a/api_v2/blueprints/cred_and_data/blueprint.py b/api_v2/blueprints/cred_and_data/blueprint.py index b1caa11..7496b69 100644 --- a/api_v2/blueprints/cred_and_data/blueprint.py +++ b/api_v2/blueprints/cred_and_data/blueprint.py @@ -7,7 +7,7 @@ DATE: Created: Wednesday, 18th Sept., 2024 - Updated: Tuesday, 8th Oct., 2024 + Updated: Wednesday, 25th Dec., 2024 OBJECTIVE: @@ -61,6 +61,15 @@ from utils_v2.api.async_quart import ( make_ordered_json ) +# Models: +from models.cred_and_data.api import ( + CredAndDataSetRequestHeaders, + CredAndDataGetRequestHeaders, + CredAndDataUpdateRequestHeaders, + CredAndDataUpdateRequestData, + CredAndDataDeleteRequestHeaders +) + # For asynchronous activities: import asyncio @@ -113,8 +122,9 @@ def init(blueprint_setup_state): # --------------------------------------------------------------------------------------------------------------------- -@cred_and_data_bp.route("//set", methods = ["POST", "GET"]) -@set_api_version(api_version = "2.1.0") +@cred_and_data_bp.route("/", methods = ["POST"]) +@cred_and_data_bp.route("//set", methods = ["POST"]) +@set_api_version(api_version = "2.2.0") @read_input(sanitize_headers = True, sanitize_data = True) @log_request_to_mongo( attr_name = "mongo", @@ -126,7 +136,7 @@ def init(blueprint_setup_state): ) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") -@validate_input(mandatory_header_keys = ["X-Script-Id", "X-Script-Desc"]) +@validate_input(header_validator = lambda x: CredAndDataSetRequestHeaders(**x).model_dump()) @handle_cancelled_request() async def set_data( json_type: str = None, @@ -181,8 +191,9 @@ async def set_data( # --------------------------------------------------------------------------------------------------------------------- -@cred_and_data_bp.route("//get", methods = ["POST", "GET"]) -@set_api_version(api_version = "2.1.0") +@cred_and_data_bp.route("/", methods = ["GET"]) +@cred_and_data_bp.route("//get", methods = ["GET"]) +@set_api_version(api_version = "2.2.0") @read_input(sanitize_headers = True, sanitize_data = True) @log_request_to_mongo( attr_name = "mongo", @@ -194,7 +205,7 @@ async def set_data( ) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") -@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@validate_input(header_validator = lambda x: CredAndDataGetRequestHeaders(**x).model_dump()) @handle_cancelled_request() async def get_data( json_type: str = None, @@ -251,7 +262,8 @@ async def get_data( # --------------------------------------------------------------------------------------------------------------------- -@cred_and_data_bp.route("//update", methods = ["POST", "GET"]) +@cred_and_data_bp.route("/", methods = ["PATCH"]) +@cred_and_data_bp.route("//update", methods = ["PATCH"]) @set_api_version(api_version = "2.1.0") @read_input(sanitize_headers = True, sanitize_data = True) @log_request_to_mongo( @@ -264,12 +276,15 @@ async def get_data( ) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") -@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@validate_input( + header_validator = lambda x: CredAndDataUpdateRequestHeaders(**x).model_dump(), + data_validator = lambda x: CredAndDataUpdateRequestData(**x) +) @handle_cancelled_request() async def update_cred( json_type: str = None, inbound_headers: dict = None, - inbound_data: dict = None, + inbound_data: dict | CredAndDataUpdateRequestData = None, inbound_files: dict = None, **kwargs ): @@ -299,10 +314,10 @@ async def update_cred( # Prepare the update JSON. Pre-process the fields to set and unset. # Our actual data/cred are held inside a field called "content", so we must wrap the request in that: update_json = {} - if inbound_data.get("unset"): - update_json["$unset"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["unset"]}) - if inbound_data.get("set"): - update_json["$set"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data["set"]}) + if inbound_data.unsetJson: + update_json["$unset"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data.unsetJson}) + if inbound_data.setJson: + update_json["$set"] = current_app.mongo.dict_to_dot_notation({"content": inbound_data.setJson}) # Make an attempt to set the credentials: success = await current_app.mongo.update_one( @@ -334,7 +349,8 @@ async def update_cred( ) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") -@validate_input(mandatory_header_keys = ["X-Script-Id"]) +@validate_input(header_validator = lambda x: CredAndDataDeleteRequestHeaders(**x).model_dump()) +@handle_cancelled_request() async def delete_data( json_type: str = None, inbound_headers: dict = None, diff --git a/api_v2/blueprints/logs/blueprint.py b/api_v2/blueprints/logs/blueprint.py index d22f267..e8b8b49 100644 --- a/api_v2/blueprints/logs/blueprint.py +++ b/api_v2/blueprints/logs/blueprint.py @@ -7,7 +7,7 @@ DATE: Created: Wednesday, 18th Sept., 2024 - Updated: Tuesday, 8th Oct., 2024 + Updated: Wednesday, 25th Dec., 2024 OBJECTIVE: @@ -58,8 +58,9 @@ from utils_v2.api.async_quart import ( handle_cancelled_request ) -# Data models: +# Models: from utils_v2.api.log import APILogModel +from models.logs.api import LogChainRequestData, LogsByFilterRequestData # For asynchronous activities: import asyncio @@ -128,12 +129,12 @@ async def get_log( if not fetched_log: return ResponseModel( status_code = StatusCodes.FAILED, http_code = HttpCodes.NOT_FOUND, - message = "no such log" + message = "No such log." ) else: return ResponseModel( status_code = StatusCodes.OK, - message = f"log found", + message = f"Log found.", data = {"total": 1, "fetched": 1, "logs": fetched_log} ) @@ -171,12 +172,12 @@ async def get_exception_from_log( if not fetched_log: return ResponseModel( status_code = StatusCodes.FAILED, http_code = HttpCodes.NOT_FOUND, - message = "no such log" + message = "No such log." ) else: return ResponseModel( status_code = StatusCodes.OK, - message = f"log found", + message = f"Log found.", data = {"total": 1, "fetched": 1, "logs": fetched_log} ) @@ -189,10 +190,12 @@ async def get_exception_from_log( @read_input(sanitize_headers = False, sanitize_data = False) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(data_validator = lambda x: LogChainRequestData(**x)) +@handle_cancelled_request() async def get_log_chain( log_chain, inbound_headers: dict = None, - inbound_data: dict = None, + inbound_data: dict | LogChainRequestData = None, inbound_files: dict = None, **kwargs ): @@ -214,10 +217,10 @@ async def get_log_chain( fetched_logs = await current_app.mongo.find_many( collection = "logs", filter = {"logChain": log_chain}, - projection = {"_id": False}, - sort = inbound_data.get("sort", {"_id": -1}), - limit = inbound_data.get("limit", 25), - skip = inbound_data.get("skip", 0) + projection = inbound_data.projection, + sort = inbound_data.sort, + limit = inbound_data.limit, + skip = inbound_data.skip ) fetched_count = len(fetched_logs) @@ -225,12 +228,12 @@ async def get_log_chain( if not fetched_logs: return ResponseModel( status_code = StatusCodes.FAILED, http_code = HttpCodes.NOT_FOUND, - message = "no such log chain" + message = "No such log chain." ) else: return ResponseModel( status_code = StatusCodes.OK, - message = f"{fetched_count} logs fetched", + message = f"{fetched_count} log(s) fetched", data = {"total": total_count, "fetched": fetched_count, "logs": fetched_logs} ) @@ -243,10 +246,12 @@ async def get_log_chain( @read_input(sanitize_headers = False, sanitize_data = False) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(data_validator = lambda x: LogChainRequestData(**x)) +@handle_cancelled_request() async def get_exceptions_from_log_chain( log_chain, inbound_headers: dict = None, - inbound_data: dict = None, + inbound_data: dict | LogChainRequestData = None, inbound_files: dict = None, **kwargs ): @@ -275,9 +280,9 @@ async def get_exceptions_from_log_chain( "ts": True, "exception": True }, - sort = inbound_data.get("sort", {"_id": -1}), - limit = inbound_data.get("limit", 25), - skip = inbound_data.get("skip", 0) + sort = inbound_data.sort, + limit = inbound_data.limit, + skip = inbound_data.skip ) fetched_count = len(fetched_logs) @@ -285,12 +290,12 @@ async def get_exceptions_from_log_chain( if not fetched_logs: return ResponseModel( status_code = StatusCodes.FAILED, http_code = HttpCodes.NOT_FOUND, - message = "no such log chain" + message = "No such log chain." ) else: return ResponseModel( status_code = StatusCodes.OK, - message = f"{fetched_count} logs fetched", + message = f"{fetched_count} log(s) fetched.", data = {"total": total_count, "fetched": fetched_count, "logs": fetched_logs} ) @@ -303,9 +308,11 @@ async def get_exceptions_from_log_chain( @read_input(sanitize_headers = False, sanitize_data = False) @should_not_be_under_maintenance(attr_name = "is_under_maintenance") @only_whitelisted_ips(attr_name = "whitelisted_ips") +@validate_input(data_validator = lambda x: LogsByFilterRequestData(**x)) +@handle_cancelled_request() async def get_logs_by_filter( inbound_headers: dict = None, - inbound_data: dict = None, + inbound_data: dict | LogsByFilterRequestData = None, inbound_files: dict = None, **kwargs ): @@ -320,16 +327,16 @@ async def get_logs_by_filter( total_count = await current_app.mongo.count( collection = "logs", - filter = inbound_data["filter"] + filter = inbound_data.filter ) fetched_logs = await current_app.mongo.find_many( collection = "logs", - filter = inbound_data["filter"], - projection = inbound_data.get("projection", {"_id": False}), - sort = inbound_data.get("sort", {"_id": -1}), - limit = inbound_data.get("limit", 25), - skip = inbound_data.get("skip", 0) + filter = inbound_data.filter, + projection = inbound_data.projection, + sort = inbound_data.sort, + limit = inbound_data.limit, + skip = inbound_data.skip ) fetched_count = len(fetched_logs) @@ -337,12 +344,12 @@ async def get_logs_by_filter( if not fetched_logs: return ResponseModel( status_code = StatusCodes.FAILED, http_code = HttpCodes.NOT_FOUND, - message = "no matching logs" + message = "No matching logs." ) else: return ResponseModel( status_code = StatusCodes.OK, - message = f"{len(fetched_logs)} / {total_count} logs fetched", + message = f"{len(fetched_logs)} / {total_count} log(s) fetched.", data = {"total": total_count, "fetched": fetched_count, "logs": fetched_logs} ) diff --git a/api_v2/main.py b/api_v2/main.py index edc2d75..e8429b7 100644 --- a/api_v2/main.py +++ b/api_v2/main.py @@ -68,8 +68,8 @@ from utils_v2.api.async_quart import ( from icecream import IceCreamDebugger # All the blueprints: -from api.cred_data.blueprint import cred_and_data_bp -from api.logs.blueprint import logs_bp +from api_v2.blueprints.cred_and_data.blueprint import cred_and_data_bp +from api_v2.blueprints.logs.blueprint import logs_bp # ***************************************************************************************************************** @@ -81,7 +81,7 @@ from api.logs.blueprint import logs_bp # Quart related: MODULE_BASE = "internal" -APP_VERSION = "2.0.0" +APP_VERSION = "2.2.0" # ***************************************************************************************************************** diff --git a/controllers/servers/server.py b/controllers/servers/server.py index e69de29..ea47e29 100644 --- a/controllers/servers/server.py +++ b/controllers/servers/server.py @@ -0,0 +1,394 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Thursday, 12th Dec., 2024 + + OBJECTIVE: + + To handle all messages from one place. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# For Quart: +from quart import current_app + +# My async utils: +from utils_v2.string import json +from utils_v2.date_time import date_time +from utils_v2.database.async_mysql_v2 import AsyncMySQL +from utils_v2.database.async_mongo_v2 import AsyncMongo, AsyncMongoStorage + +# Models: +from models.servers.core import CoreServerInfoModel + +# To work with MongoDB: +from bson import ObjectId +from pymongo import InsertOne, UpdateOne, ReplaceOne + +# To work with datatypes: +from typing import Literal, List, Dict, Any + +# To make deep-copies: +import copy + +# To work with base-64 encoding: +import base64 + +# To work with date and time: +import datetime + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** CLASSES *** +# ***** **** +# ***************************************************************************************************************** + + +class CoreServerController: + + # ┏┓┓ ┓┏ + # ┃ ┃┏┓┏┏ ┃┃┏┓┏┓┏ + # ┗┛┗┗┻┛┛ ┗┛┗┻┛ ┛ + + # For MongoDB: + SERVERS_COLLECTION = "_servers" + + # ┏┓┳┓┳┳┳┓ ┏┓ + # ┃ ┣┫┃┃┃┃ ━━ ┃ ┏┓┏┓┏┓╋┏┓ + # ┗┛┛┗┗┛┻┛ ┗┛┛ ┗ ┗┻┗┗ + + async def register( + self, + mongo_conn: AsyncMongo, + server: CoreServerInfoModel + ) -> ObjectId: + + """ + Register one server in the database. + :param mongo_conn: The instance of the database connector to use for the operation. + :param server: The message to save into the database. + :return: The object id of the inserted document. + """ + + # Simply insert the document: + return await mongo_conn.insert_one( + collection = self.MESSAGES_COLLECTION, + document = message, + raise_exception = True + ) + + async def bulk_write( + self, + mongo_conn: AsyncMongo, + mongo_operations: list + ) -> int: + + """ + Needed in cases like forcing re-sync of mails where you need to perform actions like bulk replacements of + existing documents. Not recommended to use. Please use very carefully to ensure document integrity. + :param mongo_conn: The instance of the database connector to use for the operation. + :param mongo_operations: The list operations that are supported by MongoDB's Bulk Write system. + :return: The no. of documents affected. + """ + + return await mongo_conn.bulk_write( + collection = self.MESSAGES_COLLECTION, + requests = mongo_operations, + raise_exception = True + ) + + # ┏┓┳┓┳┳┳┓ ┳┓ • + # ┃ ┣┫┃┃┃┃ ━━ ┣┫┏┓╋┏┓┓┏┓┓┏┏┓ + # ┗┛┛┗┗┛┻┛ ┛┗┗ ┗┛ ┗┗ ┗┛┗ + + async def count_messages( + self, + mongo_conn: AsyncMongo, + token_ids: List[ObjectId | str], + additional_filter: dict = None + ) -> int: + + """ + Just counts the no. of messages that match a given set of conditions. + :param mongo_conn: The instance of the database connector to use for the operation. + :param token_ids: The token ids of the accounts from which these messages must be fetched. + :param additional_filter: Any addition filters to use. + :return: The no. of messages that match the given conditions. + """ + + # Prepare the filter: + if not isinstance(token_ids, list): token_ids = [token_ids] + token_ids = [ObjectId(t) for t in token_ids] + filter_json = {"tokenId": {"$in": token_ids}} + if additional_filter: + for k, v in additional_filter.items(): + filter_json[k] = v + + # Get the count of the documents that match the criteria: + count = await mongo_conn.count( + collection = self.MESSAGES_COLLECTION, + filter = filter_json, + raise_exception = True + ) + + # Done here: + return count + + async def get_previews( + self, + mongo_conn: AsyncMongo, + token_ids: List[ObjectId | str], + limit: int = 100, + skip: int = 0, + additional_filter: dict = None + ) -> List[CoreMessageModel] | None: + + """ + Fetches many messages in one call, but leaves out the full payloads. + :param mongo_conn: The instance of the database connector to use for the operation. + :param token_ids: The token ids of the accounts from which these messages must be fetched. + :param limit: The max. no. of messages to retrieve in this call. + :param skip: The no. of initial messages to skip. Useful for pagination. + :param additional_filter: Any addition filters to use. + :return: The list of messages (as the message model). This list can be empty. + """ + + # Prepare the filter: + if not isinstance(token_ids, list): token_ids = [token_ids] + token_ids = [ObjectId(t) for t in token_ids] + filter_json = {"tokenId": {"$in": token_ids}} + if additional_filter: + for k, v in additional_filter.items(): + filter_json[k] = v + + # We fetch the messages that are identified by a specific token id, + # with the specified fetching limits, while enforcing the sorting condition: + records = await mongo_conn.find_many( + collection = self.MESSAGES_COLLECTION, + filter = filter_json, + limit = limit, + skip = skip, + sort = {"ts": -1}, + projection = { + "_id": True, + "ts": True, + "syncTs": True, + "tokenId": True, + "serviceType": True, + "client": True, + "clientMessageId": True, + "clientThreadId": True, + "isSent": True, + "isBroadcast": True, + "sentSuccessfully": True, + "sender": True, + "chat": True, + "snippet": True, + "aiSnippet": True, + "tags": True + }, + raise_exception = True + ) + + # Convert the fetched records to instances of the data model and return: + for record in records: record["message"] = {} + return [CoreMessageModel(**record) for record in records] + + async def get_messages( + self, + mongo_conn: AsyncMongo, + token_ids: List[ObjectId | str], + limit: int = 100, + skip: int = 0, + additional_filter: dict = None + ) -> List[CoreMessageModel] | None: + + """ + Fetches many full messages in one call. + :param mongo_conn: The instance of the database connector to use for the operation. + :param token_ids: The token ids of the accounts from which these messages must be fetched. + :param limit: The max. no. of messages to retrieve in this call. + :param skip: The no. of initial messages to skip. Useful for pagination. + :param additional_filter: Any addition filters to use. + :return: The list of messages (as the message model). This list can be empty. + """ + + # Prepare the filter: + if not isinstance(token_ids, list): token_ids = [token_ids] + token_ids = [ObjectId(t) for t in token_ids] + filter_json = {"tokenId": {"$in": token_ids}} + if additional_filter: + for k, v in additional_filter.items(): + filter_json[k] = v + + # We fetch the messages that are identified by a specific token id, + # with the specified fetching limits, while enforcing the sorting condition: + records = await mongo_conn.find_many( + collection = self.MESSAGES_COLLECTION, + filter = filter_json, + limit = limit, + skip = skip, + sort = {"ts": -1}, + raise_exception = True + ) + + # Convert the fetched records to instances of the data model and return: + return [CoreMessageModel(**record) for record in records] + + async def get_message( + self, + mongo_conn: AsyncMongo, + message_id: ObjectId | str, + ) -> CoreMessageModel | None: + + """ + Gets one message if you know its message id. + :param mongo_conn: The instance of the database connector to use for the operation. + :param message_id: The id of the message that needs to be read. + :return: The contents of that one message in a structured format. + """ + + # We fetch the whole payload of that one message: + record = await mongo_conn.find_one( + collection = self.MESSAGES_COLLECTION, + filter = {"_id": ObjectId(message_id)}, + raise_exception = True + ) + + # If no such message was found: + if record is None: return None + + # If a record was found, + # we return it as our data model: + return CoreMessageModel(**record) + + # ┏┓┳┓┳┳┳┓ ┳┳ ┓ + # ┃ ┣┫┃┃┃┃ ━━ ┃┃┏┓┏┫┏┓╋┏┓ + # ┗┛┛┗┗┛┻┛ ┗┛┣┛┗┻┗┻┗┗ + # ┛ + + # We don't support updating messages themselves, + # but we will allow updating fields like tags, marking as read or unread, etc. + + async def update_tags( + self, + mongo_conn: AsyncMongo, + message_id: ObjectId | str, + unset_tags: List[str] = None, + set_tags: List[str] = None + ) -> bool: + + """ + Updates the tags on one message. The tags to remove are processed first, the ones to add are processed later. + :param mongo_conn: The instance of the database connector to use for the operation. + :param message_id: The id of the message that needs to be read. + :param unset_tags: The tags to remove from the message. + :param set_tags: The tags to add to the message. + :return: True if the update was successful, else False. + """ + + # Update the tags: + return await mongo_conn.update_one( + collection = self.MESSAGES_COLLECTION, + filter = {"_id": ObjectId(message_id)}, + update = [{ + "$set": { + "tags": { + "$let": { + "vars": { + "removed_tags": { + "$setDifference": [ + "$tags", + unset_tags + ] + } + }, + "in": { + "$setUnion": [ + "$$removed_tags", + set_tags + ] + } + } + } + } + }], + raise_exception = True + ) + + # ┏┓┳┓┳┳┳┓ ┳┓ ┓ + # ┃ ┣┫┃┃┃┃ ━━ ┃┃┏┓┃┏┓╋┏┓ + # ┗┛┛┗┗┛┻┛ ┻┛┗ ┗┗ ┗┗ + + # No support whatsoever for deleting messages. + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + pass diff --git a/models/logs/api.py b/models/logs/api.py index 8b39ab4..cfbab42 100644 --- a/models/logs/api.py +++ b/models/logs/api.py @@ -74,6 +74,50 @@ import datetime # ***************************************************************************************************************** +class LogChainRequestData(BaseModel): + + limit: int = Field( + description = "the max. logs to pick", + default = 25, + frozen = True + ) + + skip: int = Field( + description = "how many initial matches to skip before picking logs; needed for pagination", + default = 0, + frozen = True + ) + + sort: dict = Field( + description = "the sorting conditions for the picked logs", + default = {"ts": 1, "_id": -1}, + frozen = True + ) + + projection: dict = Field( + description = "to decide what fields are retrieved", + default = {"_id": False}, + frozen = True + ) + + # ┏┓ ┏• + # ┃ ┏┓┏┓╋┓┏┓ + # ┗┛┗┛┛┗┛┗┗┫ + # ┛ + + class Config: + extra = "forbid" + + # ┓┏ ┓• ┓ • + # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓ + # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗ + + pass + + +# --------------------------------------------------------------------------------------------------------------------- + + class LogsByFilterRequestData(BaseModel): filter: dict = Field( @@ -117,10 +161,7 @@ class LogsByFilterRequestData(BaseModel): # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓ # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗ - @field_validator("unsetJson", "setJson", mode = "before") - def ensure_non_null(cls, value): - if value is None: value = {} - return value + pass # ***************************************************************************************************************** diff --git a/models/servers/core.py b/models/servers/core.py index 6b74795..a6efd23 100644 --- a/models/servers/core.py +++ b/models/servers/core.py @@ -10,7 +10,7 @@ OBJECTIVE: - To provide a structure to work with requests surrounding Cred and Data handling. + To provide a structure to work with enlisting servers and maintaining their status. REFERENCES: @@ -36,7 +36,7 @@ sys.path.append(".") sys.path.append("..") # For making data behaviour_models: -from pydantic import BaseModel, Field, field_validator, PastDatetime, model_validator +from pydantic import BaseModel, Field, field_validator, PastDatetime, model_validator, AwareDatetime from typing import Optional, Literal, Union # My utils: @@ -74,18 +74,105 @@ import datetime # ***************************************************************************************************************** -class CredAndDataSetRequestHeaders(BaseModel): +class CoreServerInfoModel(BaseModel): - scriptId: str = Field( - description = "the id of the script for whom you are setting cred/data", - frozen = True, - alias = "X-Script-Id" + hostname: str = Field( + description = "the hostname to identify the server", + frozen = True ) - scriptDescription: str = Field( - description = "a short description of the script and what it does", - frozen = True, - alias = "X-Script-Desc" + os: str = Field( + description = "the os the server is running", + frozen = True + ) + + cpu: str = Field( + description = "the cpu that the server has in it", + frozen = True + ) + + pid: str | int = Field( + description = "the process id that registered the details of this server", + frozen = True + ) + + ppid: str | int = Field( + description = "the parent process id that registered the details of this server", + frozen = True + ) + + ipAddr: str | None = Field( + description = "the ip address of the server", + frozen = True + ) + + portNo: str | None = Field( + description = "the port no. that this service is running on", + frozen = True + ) + + project: str = Field( + description = "the name of the project that this server is running", + frozen = True + ) + + service: str = Field( + description = "the service in the said project that this server is running", + frozen = True + ) + + healthCheckUrl: str = Field( + description = "a get request will be sent to this url to see if the server is up", + frozen = True + ) + + healthCheckInterval: int = Field( + description = "the seconds after which to check for the service being up", + ge = 30, + frozen = True + ) + + healthAlertUrl: str = Field( + description = "a get request will be sent to this url when the server goes offline", + frozen = True + ) + + online: bool = Field( + description = "whether this service is online or not", + default = False, + frozen = False + ) + + batchId: str | None = Field( + description = "set some value here when checking the status of this server, set it to null once done", + default = None, + frozen = False + ) + + batchTs: AwareDatetime | None = Field( + description = "set the time (utc) when checking the status of this server, set it to null once done", + default = None, + frozen = False + ) + + firstRegTs: AwareDatetime = Field( + description = "the first time (utc) this server was registered in the database", + frozen = True + ) + + lastRegTs: AwareDatetime = Field( + description = "the last time (utc) this server was registered in the database", + frozen = True + ) + + lastCheckTs: AwareDatetime = Field( + description = "the last time (utc) this server was checked for being online", + frozen = True + ) + + checkAfterTs: AwareDatetime = Field( + description = "the time (utc) after which this server needs to be checked again for being online", + frozen = True ) # ┏┓ ┏• @@ -94,114 +181,47 @@ class CredAndDataSetRequestHeaders(BaseModel): # ┛ class Config: - extra = "allow" - - def model_dump(self, *args, **kwargs): - return super().model_dump(*args, by_alias = True, **kwargs) - - -# --------------------------------------------------------------------------------------------------------------------- - - -class CredAndDataGetRequestHeaders(BaseModel): - - scriptId: str = Field( - description = "the id of the script for whom you are getting cred/data", - frozen = True, - alias = "X-Script-Id" - ) - - # ┏┓ ┏• - # ┃ ┏┓┏┓╋┓┏┓ - # ┗┛┗┛┛┗┛┗┗┫ - # ┛ - - class Config: - extra = "allow" - - def model_dump(self, *args, **kwargs): - return super().model_dump(*args, by_alias = True, **kwargs) - - -# --------------------------------------------------------------------------------------------------------------------- - - -class CredAndDataUpdateRequestHeaders(BaseModel): - - scriptId: str = Field( - description = "the id of the script for whom you are updating cred/data", - frozen = True, - alias = "X-Script-Id" - ) - - # ┏┓ ┏• - # ┃ ┏┓┏┓╋┓┏┓ - # ┗┛┗┛┛┗┛┗┗┫ - # ┛ - - class Config: - extra = "allow" - - def model_dump(self, *args, **kwargs): - return super().model_dump(*args, by_alias = True, **kwargs) - - -# --------------------------------------------------------------------------------------------------------------------- - - -class CredAndDataUpdateRequestData(BaseModel): - - unsetJson: dict = Field( - description = "the items to unset; this is performed first", - frozen = True, - alias = "unset" - ) - - setJson: dict = Field( - description = "the items to set; this is performed after the 'unset' operation", - frozen = True, - alias = "set" - ) - - # ┏┓ ┏• - # ┃ ┏┓┏┓╋┓┏┓ - # ┗┛┗┛┛┗┛┗┗┫ - # ┛ - - class Config: - extra = "forbid" + extra = "ignore" # ┓┏ ┓• ┓ • # ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓ # ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗ - @field_validator("unsetJson", "setJson", mode = "before") - def ensure_non_null(cls, value): - if value is None: value = {} + @staticmethod + def parse_date_time(value): + + # If a null value was given, + # we can't do anything: + if not value: value = None + + # If the input is a string: + if isinstance(value, str): + value = value.strip() + value = date_time.parse_date_time( + input_value = value, + timezone = date_time.TIMEZONE_UTC, + date_formats = ["%Y-%m-%d"] + ) + + # If the input is already a date-time object, + # we just normalize the timestamp: + if isinstance(value, datetime.datetime): + value = date_time.to_timezone( + value, + timezone = date_time.TIMEZONE_UTC + ) + + # Done here: return value - -# --------------------------------------------------------------------------------------------------------------------- - - -class CredAndDataDeleteRequestHeaders(BaseModel): - - scriptId: str = Field( - description = "the id of the script for whom you are deleting cred/data", - frozen = True, - alias = "X-Script-Id" + @field_validator( + "batchTs", + "firstRegTs", "lastRegTs", + "lastCheckTs", "checkAfterTs", + mode = "before" ) - - # ┏┓ ┏• - # ┃ ┏┓┏┓╋┓┏┓ - # ┗┛┗┛┛┗┛┗┗┫ - # ┛ - - class Config: - extra = "allow" - - def model_dump(self, *args, **kwargs): - return super().model_dump(*args, by_alias = True, **kwargs) + def parse_given_date_time(cls, value): + return cls.parse_date_time(value) # ***************************************************************************************************************** diff --git a/ssh_server.sh b/ssh_server.sh index 1991711..16bdc4e 100644 --- a/ssh_server.sh +++ b/ssh_server.sh @@ -19,11 +19,12 @@ if [[ $SELECTION -gt 0 && $SELECTION -le ${#SERVERS[@]} ]]; then # Note down the selection in a variable: SELECTED_SERVER=${SERVERS[$((SELECTION - 1))]} - # Ask the user which directory he wants to upload and his username on the server: + # Ask the username and target port no. on the server:: read -rp "Your username on the server ..... : " USER + read -rp "The target port no. ............. : " PORT # Run the command: - ssh -p 19991 "$USER@$SELECTED_SERVER" + ssh -p "$PORT" "$USER@$SELECTED_SERVER" # Exit with success (assuming that the actual data sending went well): echo "session ended" diff --git a/to_git.sh b/to_git.sh index 03ffea7..a74aea6 100644 --- a/to_git.sh +++ b/to_git.sh @@ -4,6 +4,10 @@ # Run this on the development machine. # NOT to be used in a collaborative environment: +# Accept the name of the branch that you want to work on: +echo "Branch : " +read -r BRANCH + # Accept a comment from the terminal: echo "Comment: " read -r COMMENT @@ -17,5 +21,5 @@ git commit -m "$COMMENT" echo "Commit done." # Push th commit to git: -git push -u https://wtt.ditscentre.in/ditscentre/api_internal.git master +git push -u https://wtt.ditscentre.in/ditscentre/api_internal.git "$BRANCH" echo "Attempt done. Exiting." \ No newline at end of file