khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20250210) Started working on MikroTik config automation.

Browse Source
This commit is contained in:
khushal
2025-02-10 19:35:28 +05:30
parent 95d90e807b
commit 94e5c81087
13 changed files with 789 additions and 836 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers_v2/core/software.py
+7 -328
View File
@@ -6,11 +6,11 @@
DATE:
Thursday, 19th Dec., 2024
Monday, 10th Feb., 2025.
OBJECTIVE:
To handle all messages from one place.
To handle all software from one place.
REFERENCES:
@@ -92,14 +92,13 @@ import httpx
# *****************************************************************************************************************
class CoreMessageController(CoreAuthTokenController):
class CoreSoftwareController(CoreAuthTokenController):
# ┏┓┓ ┓┏
# ┃ ┃┏┓┏┏ ┃┃┏┓┏┓┏
# ┗┛┗┗┻┛┛ ┗┛┗┻┛ ┛
# For MongoDB:
MESSAGES_COLLECTION = "_messages"
pass
# ┏┓
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
@@ -112,14 +111,13 @@ class CoreMessageController(CoreAuthTokenController):
alert_url: str = None,
base_filter: dict = None,
debug: bool = True,
debug_prefix: str = "Message (C) | ",
debug_prefix: str = "Software (C) | ",
debug_only_errors: bool = True
):
"""
This is the foundational controller of all message controllers. You must structure individual message
controllers through this structure. Individual message controllers would be for things like mails, SMS messages,
chat app messages, etc.
This is the foundational controller of all software controllers. You must structure individual software
controllers through this structure.
:param cache: The object to use for caching results from database calls.
:param http_client: The HTTP client
:param base_filter: The basic filter that will be applied to all fetching/updating queries. WARNING: THE BASE
@@ -145,325 +143,6 @@ class CoreMessageController(CoreAuthTokenController):
debug_only_errors = debug_only_errors
)
# ┏┓┳┓┳┳┳┓ ┏┓
# ┃ ┣┫┃┃┃┃ ━━ ┃ ┏┓┏┓┏┓╋┏┓
# ┗┛┛┗┗┛┻┛ ┗┛┛ ┗ ┗┻┗┗
async def save_one_message(
self,
mongo_data_conn: AsyncMongo,
message: CoreMessageModel,
session = None
) -> ObjectId:
"""
Simply insert one message document into the database.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param message: The message to save into the database.
:param session: In case you need to perform this operation as a transaction, pass a session here.
:return: The object id of the inserted document.
"""
# Simply insert the document:
return await mongo_data_conn.insert_one(
collection = self.MESSAGES_COLLECTION,
document = message.model_dump(),
raise_exception = True,
session = session
)
async def bulk_operate_messages(
self,
mongo_data_conn: AsyncMongo,
mongo_operations: list
) -> int:
"""
Needed in cases like forcing re-sync of mails where you need to perform actions like bulk replacements of
existing documents. Not recommended to use. Please use very carefully to ensure document integrity.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param mongo_operations: The list operations that are supported by MongoDB's Bulk Write system.
:return: The no. of documents affected.
"""
return await mongo_data_conn.bulk_write(
collection = self.MESSAGES_COLLECTION,
requests = mongo_operations,
raise_exception = True
)
# ┏┓┳┓┳┳┳┓ ┳┓ •
# ┃ ┣┫┃┃┃┃ ━━ ┣┫┏┓╋┏┓┓┏┓┓┏┏┓
# ┗┛┛┗┗┛┻┛ ┛┗┗ ┗┛ ┗┗ ┗┛┗
async def count_messages(
self,
mongo_data_conn: AsyncMongo,
token_ids: List[ObjectId | str] = None,
additional_filter: dict = None
) -> int:
"""
Just counts the no. of messages that match a given set of conditions.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param token_ids: The token ids of the accounts from which these messages must be fetched.
:param additional_filter: Any addition filters to use.
:return: The no. of messages that match the given conditions.
"""
# We cannot allow counting without any filter whatsoever:
if token_ids is None and not additional_filter:
raise ValueError("Cannot operate without some filter.")
# Prepare the filter:
filter_json = {}
if token_ids is not None:
if not isinstance(token_ids, list): token_ids = [token_ids]
filter_json["tokenId"] = {"$in": token_ids}
if self._base_filter:
for k, v in self._base_filter.items(): filter_json[k] = v
if additional_filter:
for k, v in additional_filter.items(): filter_json[k] = v
# Get the count of the documents that match the criteria:
count = await mongo_data_conn.count(
collection = self.MESSAGES_COLLECTION,
filter = filter_json,
raise_exception = True
)
# Done here:
return count
async def get_message_previews(
self,
mongo_data_conn: AsyncMongo,
token_ids: List[ObjectId | str] = None,
limit: int = 100,
skip: int = 0,
additional_filter: dict = None,
projection: dict = None
) -> List[CoreMessageModel] | None:
"""
Fetches many messages in one call, but leaves out the full payloads.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param token_ids: The token ids of the accounts from which these messages must be fetched.
:param limit: The max. no. of messages to retrieve in this call.
:param skip: The no. of initial messages to skip. Useful for pagination.
:param additional_filter: Any addition filters to use.
:param projection: To decide what is picked from each document. WARNING: THIS MAY BREAK THE BEHAVIOUR OF THE
CORE MESSAGE MODEL. USE CAREFULLY.
:return: The list of messages (as the message model). This list can be empty.
"""
# We cannot allow counting without any filter whatsoever:
if token_ids is None and not additional_filter:
raise ValueError("Cannot operate without some filter.")
# Prepare the filter:
filter_json = {}
if token_ids is not None:
if not isinstance(token_ids, list): token_ids = [token_ids]
filter_json["tokenId"] = {"$in": token_ids}
if self._base_filter:
for k, v in self._base_filter.items(): filter_json[k] = v
if additional_filter:
for k, v in additional_filter.items(): filter_json[k] = v
# We fetch the messages that are identified by a specific token id,
# with the specified fetching limits, while enforcing the sorting condition:
records = await mongo_data_conn.find_many(
collection = self.MESSAGES_COLLECTION,
filter = filter_json,
limit = limit,
skip = skip,
sort = {"ts": -1},
projection = projection or {
"_id": True,
"ts": True,
"syncTs": True,
"tokenId": True,
"serviceType": True,
"client": True,
"clientMessageId": True,
"clientThreadId": True,
"isSent": True,
"isBroadcast": True,
"sentSuccessfully": True,
"sender": True,
"chat": True,
"snippet": True,
"aiSnippet": True,
"tags": True
},
raise_exception = True
)
# Convert the fetched records to instances of the data model and return:
for record in records: record["message"] = {}
return [CoreMessageModel(**record) for record in records]
async def get_messages(
self,
mongo_data_conn: AsyncMongo,
token_ids: List[ObjectId | str] = None,
limit: int = 100,
skip: int = 0,
additional_filter: dict = None,
projection: dict = None
) -> List[CoreMessageModel] | None:
"""
Fetches many full messages in one call.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param token_ids: The token ids of the accounts from which these messages must be fetched.
:param limit: The max. no. of messages to retrieve in this call.
:param skip: The no. of initial messages to skip. Useful for pagination.
:param additional_filter: Any addition filters to use.
:param projection: To decide what is picked from each document. WARNING: THIS MAY BREAK THE BEHAVIOUR OF THE
CORE MESSAGE MODEL. USE CAREFULLY.
:return: The list of messages (as the message model). This list can be empty.
"""
# We cannot allow counting without any filter whatsoever:
if token_ids is None and not additional_filter:
raise ValueError("Cannot operate without some filter.")
# Prepare the filter:
filter_json = {}
if token_ids is not None:
if not isinstance(token_ids, list): token_ids = [token_ids]
filter_json["tokenId"] = {"$in": token_ids}
if self._base_filter:
for k, v in self._base_filter.items(): filter_json[k] = v
if additional_filter:
for k, v in additional_filter.items(): filter_json[k] = v
# We fetch the messages that are identified by a specific token id,
# with the specified fetching limits, while enforcing the sorting condition:
records = await mongo_data_conn.find_many(
collection = self.MESSAGES_COLLECTION,
filter = filter_json,
limit = limit,
skip = skip,
sort = {"ts": -1},
projection = projection,
raise_exception = True
)
# Convert the fetched records to instances of the data model and return:
return [CoreMessageModel(**record) for record in records]
async def get_message(
self,
mongo_data_conn: AsyncMongo,
message_id: ObjectId | str,
additional_filter: dict = None,
projection: dict = None
) -> CoreMessageModel | None:
"""
Gets one message if you know its message id.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param message_id: The id of the message that needs to be read.
:param additional_filter: Any addition filters to use.
:param projection: To decide what is picked from each document. WARNING: THIS MAY BREAK THE BEHAVIOUR OF THE
CORE MESSAGE MODEL. USE CAREFULLY.
:return: The contents of that one message in a structured format.
"""
# Start by preparing the filter:
filter_json = {"_id": ObjectId(message_id)}
if self._base_filter:
for k, v in self._base_filter.items(): filter_json[k] = v
if additional_filter:
for k, v in additional_filter.items(): filter_json[k] = v
# We fetch the whole payload of that one message:
record = await mongo_data_conn.find_one(
collection = self.MESSAGES_COLLECTION,
filter = filter_json,
projection = projection,
raise_exception = True
)
# If no such message was found:
if record is None: return None
# If a record was found,
# we return it as our data model:
return CoreMessageModel(**record)
# ┏┓┳┓┳┳┳┓ ┳┳ ┓
# ┃ ┣┫┃┃┃┃ ━━ ┃┃┏┓┏┫┏┓╋┏┓
# ┗┛┛┗┗┛┻┛ ┗┛┣┛┗┻┗┻┗┗
# ┛
# We don't support updating messages themselves,
# but we will allow updating fields like tags, marking as read or unread, etc.
async def update_message_tags(
self,
mongo_data_conn: AsyncMongo,
message_id: ObjectId | str,
unset_tags: List[str] = None,
set_tags: List[str] = None,
additional_filter: dict = None
) -> bool:
"""
Updates the tags on one message. The tags to remove are processed first, the ones to add are processed later.
:param mongo_data_conn: The instance of the database connector to use for the operation.
:param message_id: The id of the message that needs to be read.
:param unset_tags: The tags to remove from the message.
:param set_tags: The tags to add to the message.
:param additional_filter: Any addition filters to use.
:return: True if the update was successful, else False.
"""
# Start by preparing the filter:
filter_json = {"_id": ObjectId(message_id)}
if self._base_filter:
for k, v in self._base_filter.items(): filter_json[k] = v
if additional_filter:
for k, v in additional_filter.items(): filter_json[k] = v
# Update the tags:
return await mongo_data_conn.update_one(
collection = self.MESSAGES_COLLECTION,
filter = filter_json,
update = [{
"$set": {
"tags": {
"$let": {
"vars": {
"removed_tags": {
"$setDifference": [
"$tags",
unset_tags
]
}
},
"in": {
"$setUnion": [
"$$removed_tags",
set_tags
]
}
}
}
}
}],
raise_exception = True
)
# ┏┓┳┓┳┳┳┓ ┳┓ ┓
# ┃ ┣┫┃┃┃┃ ━━ ┃┃┏┓┃┏┓╋┏┓
# ┗┛┛┗┗┛┻┛ ┻┛┗ ┗┗ ┗┗
# No support whatsoever for deleting messages.
# *****************************************************************************************************************
# ***** ****
controllers_v2/software/mikrotik/all_mikrotik.py
+42 -97
View File
@@ -41,7 +41,7 @@ from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
# Controllers:
from controllers_v2.core.software import CoreSoftwareController
from controllers_v2.software.mikrotik.base import MikroTikController
# To make very controlled API calls:
from utils_v2.rest.controllers.async_base import AsyncREST
@@ -53,6 +53,10 @@ from models.software.mikrotik.auth import (
MikroTikHotspot1000Auth,
MikroTikAuthResponse
)
from models.software.mikrotik.configure import (
MikroTikConfigAttemptResponse,
MikroTikRollBackAttemptResponse
)
# To work with datatypes:
from typing import List, Any
@@ -104,13 +108,13 @@ from abc import ABC, abstractmethod
# *****************************************************************************************************************
class MikroTikController(CoreSoftwareController, ABC):
class AllMikroTikController(MikroTikController):
# ┏┓┓ ┓┏
# ┃ ┃┏┓┏┏ ┃┃┏┓┏┓┏
# ┗┛┗┗┻┛┛ ┗┛┗┻┛ ┛
SERVICE_TYPE = "mikrotik"
pass
# ┏┓
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
@@ -121,9 +125,8 @@ class MikroTikController(CoreSoftwareController, ABC):
cache: AsyncRedisCache = None,
http_client: httpx.AsyncClient = None,
alert_url: str = None,
base_filter: dict = None,
debug: bool = True,
debug_prefix: str = "MikroTik (C) | ",
debug_prefix: str = "All MikroTik (C) | ",
debug_only_errors: bool = True
):
@@ -131,116 +134,28 @@ class MikroTikController(CoreSoftwareController, ABC):
This is the foundational controller for all MikroTik services. This is built on top of the core message
controller, and, in turn, all individual MikroTik client controllers must be built on top of this.
:param cache: The object to use for caching results from database calls.
:param http_client: The HTTP client
:param base_filter: The basic filter that will be applied to all fetching/updating queries. WARNING: THE BASE
FILTER WILL ALWAYS BE APPLIED AUTOMATICALLY. SET THIS UP WISELY.
:param http_client: The HTTP client to use to make REST-ful API calls.
:param debug: Whether, or not, you would like to print debugging messages:
:param debug_prefix: The prefix to print with the debugging messages.
:param debug_only_errors: Whether you would like to print only error messages or all messages.
:return: None.
"""
# Prepare the combined base filter:
mikrotik_filter = {}
for k, v in (base_filter or {}).items(): mikrotik_filter[k] = v
mikrotik_filter["serviceType"] = self.SERVICE_TYPE
# Invoke the parent's constructor:
CoreSoftwareController.__init__(
self,
super().__init__(
cache = cache,
alert_url = alert_url,
http_client = http_client,
base_filter = mikrotik_filter,
base_filter = None,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
# Init a variable in a parent:
self._service_type = self.SERVICE_TYPE
# For controlled REST-ful calls:
self._rest = AsyncREST(
http_client = http_client,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
# ┓┏ ┓
# ┣┫┏┓┃┏┓┏┓┏┓┏
# ┛┗┗ ┗┣┛┗ ┛ ┛
# ┛
@staticmethod
def get_mikrotik_url(
nas_ip: str,
path: str,
port_no: int | str = None,
use_https: bool = True
) -> str:
"""
Simply creates the base URL for hitting the MikroTik server.
:param nas_ip: The IP address of the MikroTik device.
:param path: The path of the REST API to hit.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: The Base URL string.
"""
base_url = r"https://" if use_https else r"http://"
base_url += nas_ip
if port_no is not None: base_url += f":{port_no}"
base_url += "/rest"
if not path.startswith("/"): path = "/" + path
return base_url + path
# ┏┓
# ┗┓┓┏┏╋┏┓┏┳┓
# ┗┛┗┫┛┗┗ ┛┗┗
# ┛
async def get_system_resource(
self,
nas_ip: str,
username: str,
password: str,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To get a summary of the hardware resources available in the MikroTik device. This also becomes a great way to
quickly check if any given device is valid, and up and running.
:param nas_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call and return the response:
return await self._rest.get(
url = self.get_mikrotik_url(
nas_ip = nas_ip,
path = r"/system/resource",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
)
)
# ┏┓ ┓
# ┣┫┓┏╋┣┓
# ┛┗┗┻┗┛┗
@abstractmethod
async def save_auth(
self,
sql_conn: AsyncMySQL,
@@ -257,7 +172,37 @@ class MikroTikController(CoreSoftwareController, ABC):
:return: A structured response to indicate what happened during authorization.
"""
pass
raise NotImplementedError
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
async def roll_back(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikRollBackAttemptResponse:
"""
The rolling-back to the original state (as best as possible) in case the configurations fails midway after
completing some no. of steps.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
raise NotImplementedError
async def configure(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikConfigAttemptResponse:
"""
Run the configuration steps for the system.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
raise NotImplementedError
# *****************************************************************************************************************
controllers_v2/software/mikrotik/base.py
+218 -103
View File
@@ -6,11 +6,11 @@
DATE:
Thursday, 19th Dec., 2024
Monday, 10th Feb., 2025.
OBJECTIVE:
To handle all SMS related behaviour from one place.
To handle all MikroTik configuration from one place.
REFERENCES:
@@ -36,24 +36,30 @@ sys.path.append(".")
sys.path.append("..")
# My async utils:
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
# Controllers:
from controllers_v2.core.message import CoreMessageController
from controllers_v2.core.software import CoreSoftwareController
# To make very controlled API calls:
from utils_v2.rest.controllers.async_base import AsyncREST
from utils_v2.rest.models.api_call import ApiResponse
# Models:
from models.core.auth_token import CoreAuthTokenModel
from models.message.sms.send import (
NimbusSMSIndiaMessage,
SavvyBulkSMSKenyaMessage,
SMSSendOneResult,
SMSSendManyResults
from models.software.mikrotik.auth import (
MikroTikPPPoE1000Auth,
MikroTikHotspot1000Auth,
MikroTikAuthResponse
)
from models.software.mikrotik.configure import (
MikroTikConfigAttemptResponse,
MikroTikRollBackAttemptResponse
)
# SMS clients:
from utils_v2.sms.india.nimbus.controllers.async_nimbus import AsyncNimbusSMS
from utils_v2.sms.kenya.savvy_bulk_sms.controllers.async_savvy_bulk_sms import AsyncSavvyBulkSMS
# To work with IP Addresses:
import ipaddress
# To work with datatypes:
from typing import List, Any
@@ -67,6 +73,9 @@ from bson.objectid import ObjectId
# To make abstract classes:
from abc import ABC, abstractmethod
# Misc:
import math
# *****************************************************************************************************************
# ***** ****
@@ -105,13 +114,13 @@ from abc import ABC, abstractmethod
# *****************************************************************************************************************
class SMSController(CoreMessageController, ABC):
class MikroTikController(CoreSoftwareController, ABC):
# ┏┓┓ ┓┏
# ┃ ┃┏┓┏┏ ┃┃┏┓┏┓┏
# ┗┛┗┗┻┛┛ ┗┛┗┻┛ ┛
SERVICE_TYPE = "sms"
SERVICE_TYPE = "mikrotik"
# ┏┓
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
@@ -124,15 +133,15 @@ class SMSController(CoreMessageController, ABC):
alert_url: str = None,
base_filter: dict = None,
debug: bool = True,
debug_prefix: str = "SMS (C) | ",
debug_prefix: str = "MikroTik (C) | ",
debug_only_errors: bool = True
):
"""
This is the foundational controller for all SMS services. This is built on top of the core message controller,
and, in turn, all individual SMS client controllers must be built on top of this.
This is the foundational controller for all MikroTik services. This is built on top of the core message
controller, and, in turn, all individual MikroTik client controllers must be built on top of this.
:param cache: The object to use for caching results from database calls.
:param http_client: The HTTP client
:param http_client: The HTTP client to use to make REST-ful API calls.
:param base_filter: The basic filter that will be applied to all fetching/updating queries. WARNING: THE BASE
FILTER WILL ALWAYS BE APPLIED AUTOMATICALLY. SET THIS UP WISELY.
:param debug: Whether, or not, you would like to print debugging messages:
@@ -142,17 +151,17 @@ class SMSController(CoreMessageController, ABC):
"""
# Prepare the combined base filter:
sms_filter = {}
for k, v in (base_filter or {}).items(): sms_filter[k] = v
sms_filter["serviceType"] = self.SERVICE_TYPE
mikrotik_filter = {}
for k, v in (base_filter or {}).items(): mikrotik_filter[k] = v
mikrotik_filter["serviceType"] = self.SERVICE_TYPE
# Invoke the parent's constructor:
CoreMessageController.__init__(
CoreSoftwareController.__init__(
self,
cache = cache,
alert_url = alert_url,
http_client = http_client,
base_filter = sms_filter,
base_filter = mikrotik_filter,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
@@ -161,88 +170,194 @@ class SMSController(CoreMessageController, ABC):
# Init a variable in a parent:
self._service_type = self.SERVICE_TYPE
# ┏┓┳┳┓┏┓ ┏┓ ┓•
# ┗┓┃┃┃┗┓ ┗┓┏┓┏┓┏┫┓┏┓┏┓
# ┗┛┛ ┗┗┛ ┗┛┗ ┛┗┗┻┗┛┗┗┫
# ┛
@abstractmethod
async def send_one_sms(
self,
mongo_data_conn: AsyncMongo,
auth_token: CoreAuthTokenModel,
client: AsyncNimbusSMS | AsyncSavvyBulkSMS,
message: NimbusSMSIndiaMessage | SavvyBulkSMSKenyaMessage,
tags: List[Any]
) -> SMSSendOneResult:
"""
To send one SMS message through the third-party client.
:param mongo_data_conn: The database connection to use to perform this task.
:param auth_token: The auth token that will be used to send this message.
:param client: The third-party SMS client to use to send this message.
:param message: The actual message that needs to be sent.
:param tags: Any tags to attach with this SMS for filtering when querying in the listing service.
:return: The structured result of sending one message.
"""
pass
@abstractmethod
async def send_many_sms(
self,
mongo_data_conn: AsyncMongo,
auth_token: CoreAuthTokenModel,
messages: List[NimbusSMSIndiaMessage | SavvyBulkSMSKenyaMessage],
tags: List[Any]
) -> SMSSendManyResults:
"""
To send multiple SMS messages through the third-party client.
individual message, and then aggregates the results.
:param mongo_data_conn: The database connection to use to perform this task.
:param auth_token: The auth token that will be used to send this message.
:param messages: The list of messages to send out.
:param tags: Any tags to attach with these SMS for filtering when querying in the listing service. The same tags
will be applied to all messages. Do not call this method if you need to have different tags for all of them.
:return: The structured result of sending many SMS messages.
"""
pass
# ┏┓┳┳┓┏┓ ┳┳ ┓ •
# ┗┓┃┃┃┗┓ ┃┃┏┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┛ ┗┗┛ ┗┛┣┛┗┻┗┻┗┗┛┗┗┫
# ┛ ┛
# We cannot modify the SMS messages themselves, but we can set/unset tags on them for internal referencing and
# filtering. This will help the users organize their inboxes well.
async def update_sms_tags(
self,
mongo_data_conn: AsyncMongo,
message_id: ObjectId | str,
unset_tags: List[str] = None,
set_tags: List[str] = None
) -> bool:
"""
To set and unset tags on an SMS message.
:param mongo_data_conn: The database connection to use to perform this action.
:param message_id: The ObjectId of the document in MongoDb that holds the message.
:param unset_tags: The list of tags to unset (done before setting new tags).
:param set_tags: The list of tags to set (done after unsetting old tags).
:return: True if successful, else False.
"""
# Simply call the core model:
return await self.update_message_tags(
mongo_data_conn = mongo_data_conn,
message_id = message_id,
unset_tags = unset_tags,
set_tags = set_tags
# For controlled REST-ful calls:
self._rest = AsyncREST(
http_client = http_client,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
# ┓┏ ┓
# ┣┫┏┓┃┏┓┏┓┏┓┏
# ┛┗┗ ┗┣┛┗ ┛ ┛
# ┛
@staticmethod
def get_mikrotik_url(
nas_ip: str,
path: str,
port_no: int | str = None,
use_https: bool = True
) -> str:
"""
Simply creates the base URL for hitting the MikroTik server.
:param nas_ip: The IP address of the MikroTik device.
:param path: The path of the REST API to hit.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: The Base URL string.
"""
base_url = r"https://" if use_https else r"http://"
base_url += nas_ip
if port_no is not None: base_url += f":{port_no}"
base_url += "/rest"
if not path.startswith("/"): path = "/" + path
return base_url + path
@staticmethod
def split_ipv4_range_among_vlans(
start_ip: ipaddress.IPv4Address | str,
end_ip: ipaddress.IPv4Address | str,
vlan_ids: List[int]
) -> List[dict]:
"""
Divides the IPv4 range over the list of VLAN ids.
:param start_ip: The first IP in the full pool (range).
:param end_ip: The last IP in the full pool (range).
:param vlan_ids: The list of VLAN ids (need not be sequential or ordered).
:return: A list of dicts that describes each VLAN.
"""
# Parse the inputs:
start_ip_obj = start_ip if isinstance(start_ip, ipaddress.IPv4Address) else ipaddress.IPv4Address(start_ip)
end_ip_obj = end_ip if isinstance(end_ip, ipaddress.IPv4Address) else ipaddress.IPv4Address(end_ip)
vlan_count = len(vlan_ids)
# Calculate the total number of IPs in the range:
total_ips = int(end_ip_obj) - int(start_ip_obj) + 1
# Calculate the no. of IPs each VLAN gets,
# and add three because we need IPs for network, gateway and broadcast:
ips_per_vlan = math.ceil(total_ips / vlan_count) + 3
# Generate subnets for each VLAN:
subnets = []
current_ip = start_ip_obj
for vlan_id in vlan_ids:
# Calculate the network address for the current VLAN:
subnet_network = ipaddress.IPv4Network(f"{current_ip}/{32 - (ips_per_vlan).bit_length()}", strict = False)
# Check for overlap with the parent network:
subnet_start_ip_obj = subnet_network.network_address
subnet_end_ip_obj = subnet_network.broadcast_address
if subnet_start_ip_obj <= end_ip_obj:
# Add the calculated subnet to the list:
subnets.append({
"vlanId": vlan_id,
"network": str(subnet_network),
"size": int(2 ** (32 - subnet_network.prefixlen)),
"startIp": str(subnet_start_ip_obj),
"endIp": str(subnet_end_ip_obj)
})
# Update the current IP for the next subnet
current_ip = ipaddress.IPv4Address(int(subnet_network.broadcast_address) + 1)
# Done here:
return subnets
# ┏┓
# ┗┓┓┏┏╋┏┓┏┳┓
# ┗┛┗┫┛┗┗ ┛┗┗
# ┛
async def get_system_resource(
self,
nas_ip: str,
username: str,
password: str,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To get a summary of the hardware resources available in the MikroTik device. This also becomes a great way to
quickly check if any given device is valid, and up and running.
:param nas_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call and return the response:
return await self._rest.get(
url = self.get_mikrotik_url(
nas_ip = nas_ip,
path = r"/system/resource",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
)
)
# ┏┓ ┓
# ┣┫┓┏╋┣┓
# ┛┗┗┻┗┛┗
@abstractmethod
async def save_auth(
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikAuthResponse:
"""
Checks if a particular set of incoming credentials give access to a valid server and then stores the
credentials.
:param sql_conn: The database connection to use to perform this task.
:param mongo_data_conn: The database connection to use to perform this task.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during authorization.
"""
pass
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
@abstractmethod
async def roll_back(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikRollBackAttemptResponse:
"""
The rolling-back to the original state (as best as possible) in case the configurations fails midway after
completing some no. of steps.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
pass
@abstractmethod
async def configure(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikConfigAttemptResponse:
"""
Run the configuration steps for the system.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
pass
# *****************************************************************************************************************
# ***** ****
controllers_v2/software/mikrotik/mikrotik_pppoe_1000.py
+80 -26
View File
@@ -10,7 +10,7 @@
OBJECTIVE:
To handle all MikroTik configuration from one place.
To handle configuration for MikroTik servers such that they work in PPPoE mode with support for 1,000 clients.
REFERENCES:
@@ -36,6 +36,7 @@ sys.path.append(".")
sys.path.append("..")
# My async utils:
from utils_v2.string import json
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
@@ -53,6 +54,10 @@ from models.software.mikrotik.auth import (
MikroTikHotspot1000Auth,
MikroTikAuthResponse
)
from models.software.mikrotik.configure import (
MikroTikConfigAttemptResponse,
MikroTikRollBackAttemptResponse
)
# To work with datatypes:
from typing import List, Any
@@ -104,13 +109,13 @@ from abc import ABC, abstractmethod
# *****************************************************************************************************************
class AllMikroTikController(MikroTikController):
class MikroTikPPPoE1000Controller(MikroTikController):
# ┏┓┓ ┓┏
# ┃ ┃┏┓┏┏ ┃┃┏┓┏┓┏
# ┗┛┗┗┻┛┛ ┗┛┗┻┛ ┛
SERVICE_TYPE = "mikrotik"
CLIENT_NAME = "mikrotikPPPoE1000"
# ┏┓
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
@@ -121,52 +126,36 @@ class AllMikroTikController(MikroTikController):
cache: AsyncRedisCache = None,
http_client: httpx.AsyncClient = None,
alert_url: str = None,
base_filter: dict = None,
debug: bool = True,
debug_prefix: str = "All MikroTik (C) | ",
debug_prefix: str = "MTik. PPPoE 1K (C) | ",
debug_only_errors: bool = True
):
"""
This is the foundational controller for all MikroTik services. This is built on top of the core message
controller, and, in turn, all individual MikroTik client controllers must be built on top of this.
This is a special class that specifically handles configuration for MikroTik servers that work in PPPoE mode and
have support for upto 1,000 active clients.
:param cache: The object to use for caching results from database calls.
:param http_client: The HTTP client
:param base_filter: The basic filter that will be applied to all fetching/updating queries. WARNING: THE BASE
FILTER WILL ALWAYS BE APPLIED AUTOMATICALLY. SET THIS UP WISELY.
:param http_client: The HTTP client to use to make REST-ful API calls.
:param debug: Whether, or not, you would like to print debugging messages:
:param debug_prefix: The prefix to print with the debugging messages.
:param debug_only_errors: Whether you would like to print only error messages or all messages.
:return: None.
"""
# Prepare the combined base filter:
mikrotik_filter = {}
for k, v in (base_filter or {}).items(): mikrotik_filter[k] = v
mikrotik_filter["serviceType"] = self.SERVICE_TYPE
# Invoke the parent's constructor:
MikroTikController.__init__(
self,
cache = cache,
alert_url = alert_url,
http_client = http_client,
base_filter = mikrotik_filter,
base_filter = {"client": self.CLIENT_NAME},
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
# Init a variable in a parent:
self._service_type = self.SERVICE_TYPE
# For controlled REST-ful calls:
self._rest = AsyncREST(
http_client = http_client,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
self._client = self.CLIENT_NAME
# ┏┓ ┓
# ┣┫┓┏╋┣┓
@@ -176,7 +165,7 @@ class AllMikroTikController(MikroTikController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
mikrotik_auth: MikroTikPPPoE1000Auth
) -> MikroTikAuthResponse:
"""
@@ -188,6 +177,71 @@ class AllMikroTikController(MikroTikController):
:return: A structured response to indicate what happened during authorization.
"""
# print("IN-AUTH:", json.to_string(mikrotik_auth.model_dump(), default = str))
print("VLAN SUBNETS:", json.to_string(
self.split_ipv4_range_among_vlans(
start_ip = mikrotik_auth.firstPrivateIp,
end_ip = mikrotik_auth.lastPrivateIp,
vlan_ids = mikrotik_auth.vlanRange
),
default = str
))
# Start with a blank response:
auth_response = MikroTikAuthResponse()
# Try connecting to the server to check if the credentials are valid, or not:
api_response = await self.get_system_resource(
nas_ip = mikrotik_auth.nasIp,
port_no = mikrotik_auth.nasPort,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
use_https = False
)
# If the connection attempt failed:
if not api_response.success:
auth_response.exception = api_response.exception
if api_response.httpCode is None:
auth_response.message = "Exception: " + api_response.exception.__class__.__name__
if exception_str := str(auth_response.exception): auth_response.message += f" ({exception_str})"
elif api_response.httpCode in [401]: auth_response.message = "Invalid credentials passed."
elif api_response.httpCode in [502]: auth_response.message = "Could not ."
else: auth_response.message = "Unknown error."
# Done here:
return auth_response
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
async def roll_back(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikRollBackAttemptResponse:
"""
The rolling-back to the original state (as best as possible) in case the configurations fails midway after
completing some no. of steps.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
raise NotImplementedError
async def configure(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikConfigAttemptResponse:
"""
Run the configuration steps for the system.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
raise NotImplementedError