khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20250212) Started a new class (util) that handles MikroTik config. in async mode, and started implementing that in the PPPoE-1000 controller. I can now configure and roll back the first three steps. Long way to go :')

Browse Source
This commit is contained in:
khushal
2025-02-12 19:05:29 +05:30
parent 6e65de9dfc
commit 02dd675f83
8 changed files with 409 additions and 607 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers_v2/software/mikrotik/all_mikrotik.py
+5
View File
@@ -36,6 +36,7 @@ sys.path.append(".")
sys.path.append("..")
# My async utils:
from utils_v2.mikrotik.controllers.async_mikrotik import AsyncMikroTik
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
@@ -181,11 +182,13 @@ class AllMikroTikController(MikroTikController):
async def roll_back(
self,
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikRollBackAttemptResponse:
"""
The rolling-back to the original state (as best as possible) in case the configurations fails midway after
completing some no. of steps.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
@@ -194,10 +197,12 @@ class AllMikroTikController(MikroTikController):
async def configure(
self,
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikConfigAttemptResponse:
"""
Run the configuration steps for the system.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
controllers_v2/software/mikrotik/base.py
+5 -410
View File
@@ -37,6 +37,7 @@ sys.path.append("..")
# My async utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
from utils_v2.mikrotik.controllers.async_mikrotik import AsyncMikroTik
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
@@ -179,416 +180,6 @@ class MikroTikController(CoreSoftwareController, ABC):
debug_only_errors = debug_only_errors
)
# ┓┏ ┓
# ┣┫┏┓┃┏┓┏┓┏┓┏
# ┛┗┗ ┗┣┛┗ ┛ ┛
# ┛
@staticmethod
def get_mikrotik_url(
mikrotik_ip: str,
path: str,
port_no: int | str = None,
use_https: bool = True
) -> str:
"""
Simply creates the base URL for hitting the MikroTik server.
:param mikrotik_ip: The IP address of the MikroTik device.
:param path: The path of the REST API to hit.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: The Base URL string.
"""
base_url = r"https://" if use_https else r"http://"
base_url += mikrotik_ip
if port_no is not None: base_url += f":{port_no}"
base_url += "/rest"
if not path.startswith("/"): path = "/" + path
return base_url + path
@staticmethod
def split_ipv4_range_among_targets(
start_ip: ipaddress.IPv4Address | str,
end_ip: ipaddress.IPv4Address | str,
targets: List
) -> List[dict]:
"""
Divides the IPv4 range over the list of VLAN ids.
:param start_ip: The first IP in the full pool (range).
:param end_ip: The last IP in the full pool (range).
:param targets: The list of targets (need not be sequential or ordered) like VLAN ids.
:return: A list of dicts that describes the subnet for each target.
"""
# Parse the inputs:
start_ip_obj = start_ip if isinstance(start_ip, ipaddress.IPv4Address) else ipaddress.IPv4Address(start_ip)
end_ip_obj = end_ip if isinstance(end_ip, ipaddress.IPv4Address) else ipaddress.IPv4Address(end_ip)
target_count = len(targets)
# Calculate the total number of IPs in the range:
total_ips = int(end_ip_obj) - int(start_ip_obj) + 1
# Calculate the no. of IPs each VLAN gets,
# and add three because we need IPs for network, gateway and broadcast:
ips_per_target = math.ceil(total_ips / target_count) + 3
# Generate subnets for each VLAN:
subnets = []
current_ip = start_ip_obj
for target in targets:
# Calculate the network address for the current VLAN:
subnet_network = ipaddress.IPv4Network(f"{current_ip}/{32 - ips_per_target.bit_length()}", strict = False)
# Check for overlap with the parent network:
subnet_start_ip_obj = subnet_network.network_address
subnet_end_ip_obj = subnet_network.broadcast_address
if subnet_start_ip_obj <= end_ip_obj:
# Add the calculated subnet to the list:
subnets.append({
"target": target,
"network": str(subnet_network),
"size": int(2 ** (32 - subnet_network.prefixlen)),
"startIp": str(subnet_start_ip_obj),
"endIp": str(subnet_end_ip_obj)
})
# Update the current IP for the next subnet
current_ip = ipaddress.IPv4Address(int(subnet_network.broadcast_address) + 1)
# Done here:
return subnets
@staticmethod
def create_comment_json(
created_by: str,
created_ts: datetime.datetime = None,
roll_back_config: dict = None
) -> str:
"""
To create a JSON string that can be put as a comment in any step to later identify the work that was done and
have some hint about how to roll it back if needed.
:param created_by: A hint to put to identify which process/tool created this comment/config/process.
:param created_ts: The datetime (preferably UTC) when this comment was created.
:param roll_back_config: The original config that can be used when rolling back.
:return: The comment string to be used.
"""
# Create the JSON:
comment_json = {
"createdBy": created_by,
"createdTs": (created_ts or date_time.get_current_utc_date_time(as_string = False)).timestamp(),
"rollbackConfig": roll_back_config
}
# Return a string:
return json.to_string(comment_json, no_space = True)
@staticmethod
def parse_comment_json(comment: str) -> dict | list | None:
"""
Tries to parse a comment as if it were a JSON string.
:param comment: The comment string.
:return: The dict/list parsed form the JSON, or a null value of the input was not a valid JSON string.
"""
# Start with a null value:
parsed = None
# Try to parse the comment as if it were a JSON string:
try: parsed = json.from_string(comment)
except: parsed = None
# Done here:
return parsed
# ┏┓
# ┗┓┓┏┏╋┏┓┏┳┓
# ┗┛┗┫┛┗┗ ┛┗┗
# ┛
async def get_system_resource(
self,
mikrotik_ip: str,
username: str,
password: str,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To get a summary of the hardware resources available in the MikroTik device. This also becomes a great way to
quickly check if any given device is valid, and up and running.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.get(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = r"/system/resource",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
)
)
# Done here:
return api_response
# ┳ ┏
# ┃┏┓╋┏┓┏┓╋┏┓┏┏┓┏
# ┻┛┗┗┗ ┛ ┛┗┻┗┗ ┛
async def list_interfaces(
self,
mikrotik_ip: str,
username: str,
password: str,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To enlist all the physical connectivity interfaces available on the MikroTik device.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.get(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = r"/interface",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
)
)
# Done here:
return api_response
async def update_interface(
self,
mikrotik_ip: str,
username: str,
password: str,
dot_id: str,
json_payload: dict,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To update an interface's configuration.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param dot_id: The value of the '.id' field in the list.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:param json_payload: The JSON to send in the body pf the request.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.patch(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = f"/interface/{dot_id}",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
),
json = json_payload
)
# Done here:
return api_response
# ┓┏┓ ┏┓┳┓
# ┃┃┃ ┣┫┃┃┏
# ┗┛┗┛┛┗┛┗┛
async def list_vlans(
self,
mikrotik_ip: str,
username: str,
password: str,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To enlist all the physical connectivity interfaces available on the MikroTik device.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.get(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = r"/interface/vlan",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
)
)
# Done here:
return api_response
async def add_vlan(
self,
mikrotik_ip: str,
username: str,
password: str,
json_payload: dict,
port_no: int | str = None,
use_https: bool = True
) -> ApiResponse:
"""
To add a new VLAN to the MikroTik's configuration.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param json_payload: The JSON to send in the body pf the request.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.put(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = r"/interface/vlan",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
),
json = json_payload
)
# Done here:
return api_response
async def update_vlan(
self,
mikrotik_ip: str,
username: str,
password: str,
dot_id: str,
json_payload: dict,
port_no: int | str = None,
use_https: bool = True,
) -> ApiResponse:
"""
To update an existing VLAN in the MikroTik's configuration.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param dot_id: The value of the '.id' field in the list.
:param json_payload: The JSON to send in the body pf the request.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.patch(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = f"/interface/vlan/{dot_id}",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
),
json = json_payload
)
# Done here:
return api_response
async def remove_vlan(
self,
mikrotik_ip: str,
username: str,
password: str,
dot_id: str,
port_no: int | str = None,
use_https: bool = True,
):
"""
To delete an existing VLAN from the MikroTik's configuration.
:param mikrotik_ip: The IP address of the MikroTik device.
:param username: The username to get access to the MikroTik device.
:param password: The password to get access to the MikroTik device.
:param dot_id: The value of the '.id' field in the list.
:param port_no: The port no. to hit the MikroTik device on.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured API response.
"""
# Make the API call:
api_response = await self._rest.delete(
url = self.get_mikrotik_url(
mikrotik_ip = mikrotik_ip,
path = f"/interface/vlan/{dot_id}",
port_no = port_no,
use_https = use_https
),
auth = httpx.BasicAuth(
username = username,
password = password
)
)
# Done here:
return api_response
# ┏┓ ┓
# ┣┫┓┏╋┣┓
# ┛┗┗┻┗┛┗
@@ -620,12 +211,14 @@ class MikroTikController(CoreSoftwareController, ABC):
@abstractmethod
async def roll_back(
self,
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikRollBackAttemptResponse:
"""
The rolling-back to the original state (as best as possible) in case the configurations fails midway after
completing some no. of steps.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
@@ -635,11 +228,13 @@ class MikroTikController(CoreSoftwareController, ABC):
@abstractmethod
async def configure(
self,
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikConfigAttemptResponse:
"""
Run the configuration steps for the system.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
controllers_v2/software/mikrotik/mikrotik_pppoe_1000.py
+369 -189
View File
@@ -21,7 +21,8 @@
N/A
"""
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
@@ -37,6 +38,7 @@ sys.path.append("..")
# My async utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
from utils_v2.mikrotik.controllers.async_mikrotik import AsyncMikroTik
from utils_v2.database.async_mysql_v2 import AsyncMySQL
from utils_v2.database.async_mongo_v2 import AsyncMongo
from utils_v2.cache.async_redis_cache_v2 import AsyncRedisCache
@@ -54,10 +56,7 @@ from models.software.mikrotik.auth import (
MikroTikHotspot1000Auth,
MikroTikAuthResponse
)
from models.software.mikrotik.configure import (
MikroTikConfigAttemptResponse,
MikroTikRollBackAttemptResponse
)
from models.software.mikrotik.configure import MikroTikConfigAttemptResponse
# To work with datatypes:
from typing import List, Any
@@ -71,6 +70,9 @@ from bson.objectid import ObjectId
# To make abstract classes:
from abc import ABC, abstractmethod
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
@@ -118,9 +120,16 @@ class MikroTikPPPoE1000Controller(MikroTikController):
CLIENT_NAME = "mikrotikPPPoE1000"
# For automated configuration, and identification of automated configuration:
CREATED_BY_NAME = "easyfi"
HW_INTERFACE_NAME = "easyfi-pppoe"
VLAN_NAME = "easyfi-vlan-{}" # ... Substitute the VLAN's id here.
NAME_PREFIX = "easyfi"
CREATED_BY_NAME = NAME_PREFIX + "-pppoe-1000"
HW_INTERFACE_NAME = NAME_PREFIX + "-pppoe"
VLAN_NAME = NAME_PREFIX + "-vlan-{}" # .................... Substitute the VLAN's id here.
PRIVATE_IP_POOL_NAME = NAME_PREFIX + "-pppoe-pool-{}" # ... Substitute the VLAN's id here.
PPP_PROFILE_NAME = NAME_PREFIX + "-pppoe-prf-{}" # ........ Substitute the VLAN's id here.
PPP_SERVER_NAME = NAME_PREFIX + "-pppoe-srv-{}" # ......... Substitute the VLAN's id here.
RADIUS_SERVER_NAME = NAME_PREFIX + "-radius" # ............ Substitute the VLAN's id here.
NAT_RULE_NAME = NAME_PREFIX + "-pppoe-nat-{}" # ........... Substitute rule no. here.
SNMP_COMMUNITY_NAME = NAME_PREFIX + "-snmp"
# ┏┓
# ┃ ┏┓┏┓┏╋┏┓┓┏┏╋┏┓┏┓
@@ -162,46 +171,21 @@ class MikroTikPPPoE1000Controller(MikroTikController):
# Init a variable in a parent:
self._client = self.CLIENT_NAME
# ┓┏ ┓
# ┣┫┏┓┃┏┓┏┓┏┓┏
# ┛┗┗ ┗┣┛┗ ┛ ┛
# ┛
def created_by_easyfi(
self,
resource_json: dict = None
) -> bool:
"""
Checks if the resource was created/modified by this class.
:param resource_json; The JSON of the resource that needs to be checked.
:return: True if created by this class, else False.
"""
# Extract values:
resource_name = resource_json.get("name") or ""
comment_json = self.parse_comment_json(resource_json.get("comment")) or {}
# Perform checks:
if (
comment_json.get("createdBy", "???") == self.CREATED_BY_NAME or
resource_name.lower().find("easyfi") >= 0
): return True
else: return False
# ┳ ┏
# ┃┏┓╋┏┓┏┓╋┏┓┏┏┓┏
# ┻┛┗┗┗ ┛ ┛┗┻┗┗ ┛
async def set_up_interface(
self,
mikrotik_auth: MikroTikPPPoE1000Auth,
mikrotik_client: AsyncMikroTik,
dot_id: str = None,
use_https: bool = True
) -> MikroTikConfigAttemptResponse:
"""
To find the first available interface and set it up for use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:param mikrotik_client: The client to use.
:param dot_id: If you already have an interface in mind, send its id here.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
"""
@@ -210,13 +194,7 @@ class MikroTikPPPoE1000Controller(MikroTikController):
step_response = MikroTikConfigAttemptResponse()
# Enlist all the interfaces:
api_response = await self.list_interfaces(
mikrotik_ip = mikrotik_auth.nasIp,
port_no = mikrotik_auth.nasPort,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
use_https = use_https
)
api_response = await mikrotik_client.list_interfaces(use_https = use_https)
# If the listing failed:
if not api_response.success:
@@ -225,47 +203,51 @@ class MikroTikPPPoE1000Controller(MikroTikController):
step_response.exception = step_response.exception
return step_response
# We find the first available interface:
unused_if_json = None
for if_json in await api_response.get_json():
if str(if_json.get("running")).lower() == "false":
unused_if_json = if_json
# We find/select an interface:
selected_if_json = None
for if_json in api_response.data:
if (
str(if_json.get("type")).lower() == "ether" and
(
(
dot_id is not None and
str(if_json.get(".id")).lower() == str(dot_id)
) or
(
dot_id is None and
str(if_json.get("running")).lower() == "false"
)
)
):
selected_if_json = if_json
break
# If we found no available interface:
if unused_if_json is None:
if selected_if_json is None:
step_response.success = False
step_response.message = "Failed to find an idle h/w interface."
step_response.message = "Failed to select a h/w interface."
step_response.exception = None
return step_response
# Reconfigure the idle interface:
api_response = await self.update_interface(
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
dot_id = unused_if_json.get(".id"),
port_no = mikrotik_auth.nasPort,
use_https = use_https,
api_response = await mikrotik_client.update_interface(
dot_id = selected_if_json.get(".id"),
json_payload = {
"name": self.HW_INTERFACE_NAME,
"disabled": "false",
"comment": self.create_comment_json(
created_by = self.CREATED_BY_NAME,
created_ts = date_time.get_current_utc_date_time(as_string = False),
roll_back_config = unused_if_json
)
}
"comment": mikrotik_client.create_comment_json(roll_back_config = selected_if_json)
},
use_https = use_https,
)
# Check if the attempt was successful:
if api_response.success:
step_response.success = True
step_response.message = f"H/w interface ('{unused_if_json.get('.id')}') configured."
step_response.message = f"H/w interface ('{selected_if_json.get('.id')}') configured."
step_response.exception = None
else:
step_response.success = False
step_response.message = f"H/w interface ('{unused_if_json.get('.id')}') found, but could NOT be configured."
step_response.message = f"H/w interface ('{selected_if_json.get('.id')}') found, but could NOT be configured."
step_response.exception = api_response.exception
# Done here:
@@ -273,13 +255,13 @@ class MikroTikPPPoE1000Controller(MikroTikController):
async def roll_back_interface(
self,
mikrotik_auth: MikroTikPPPoE1000Auth,
mikrotik_client: AsyncMikroTik,
use_https: bool = True
) -> MikroTikRollBackAttemptResponse:
) -> MikroTikConfigAttemptResponse:
"""
To find the first available interface.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:param mikrotik_client: The client to use.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
"""
@@ -290,13 +272,7 @@ class MikroTikPPPoE1000Controller(MikroTikController):
step_response = MikroTikConfigAttemptResponse()
# Enlist all the interfaces:
api_response = await self.list_interfaces(
mikrotik_ip = mikrotik_auth.nasIp,
port_no = mikrotik_auth.nasPort,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
use_https = use_https
)
api_response = await mikrotik_client.list_interfaces(use_https = use_https)
# If the listing failed:
if not api_response.success:
@@ -308,35 +284,29 @@ class MikroTikPPPoE1000Controller(MikroTikController):
# Loop through all the interfaces,
# identify which ones were configured by this class,
# and perform rollback on those:
for if_json in await api_response.get_json():
for if_json in api_response.data:
# Check if the configuration was made by this class:
created_by = None
original_config = None
comment_json = self.parse_comment_json(if_json.get("comment"))
if comment_json is not None:
created_by = comment_json.get("createdBy")
original_config = comment_json.get("rollbackConfig")
# Extract the needed values:
comment_json = mikrotik_client.parse_comment_json(if_json.get("comment"))
original_config = mikrotik_client.get_original_config(comment_json)
# If this was indeed set up by this class,
# and if rollback data is available:
if created_by == self.CREATED_BY_NAME and original_config is not None:
api_response = await self.update_interface(
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
if mikrotik_client.is_my_config(
config_json = if_json,
name_substring = self.NAME_PREFIX
):
api_response = await mikrotik_client.update_interface(
dot_id = if_json.get(".id"),
port_no = mikrotik_auth.nasPort,
use_https = use_https,
json_payload={
"name": original_config.get("name", self.HW_INTERFACE_NAME),
json_payload = {
"name": original_config.get("name"),
"disabled": "false",
"comment": original_config.get("comment", "")
}
"comment": original_config.get("comment")
},
use_https = use_https,
)
total_count += 1
if api_response.success: rolled_back_count += 1
print("RB API RESP:", api_response)
# Construct the final response:
step_response.message = f"Rolled back {rolled_back_count}/{total_count} h/w interface configuration(s)."
@@ -351,16 +321,16 @@ class MikroTikPPPoE1000Controller(MikroTikController):
async def set_up_one_vlan(
self,
mikrotik_auth: MikroTikPPPoE1000Auth,
mikrotik_client: AsyncMikroTik,
vlan_id: int,
vlan_list: List[dict],
use_https: bool = True
):
) -> MikroTikConfigAttemptResponse:
"""
To set up just one of the needed VLANs. If the VLAN id is not taken, a new VLAN will be created, otherwise the
existing one will be updated.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:param mikrotik_client: The client to use.
:param vlan_id: The id of the VLAN you want to create.
:param vlan_list: The list of existing VLANs already configured in the MikroTik device. Helps us decide between
the use of PUT and PATCH methods.
@@ -386,22 +356,14 @@ class MikroTikPPPoE1000Controller(MikroTikController):
if existing_dot_id is None:
# We add the VLAN:
api_response = await self.add_vlan(
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
api_response = await mikrotik_client.add_vlan(
json_payload = {
"name": self.VLAN_NAME.format(vlan_id),
"vlan-id": str(vlan_id),
"interface": self.HW_INTERFACE_NAME,
"disabled": "false",
"comment": self.create_comment_json(
created_by = self.CREATED_BY_NAME,
created_ts = date_time.get_current_utc_date_time(as_string = False),
roll_back_config = existing_vlan_json
)
"comment": mikrotik_client.create_comment_json(roll_back_config = existing_vlan_json)
},
port_no = mikrotik_auth.nasPort,
use_https = use_https
)
@@ -419,22 +381,14 @@ class MikroTikPPPoE1000Controller(MikroTikController):
else:
# We update it:
api_response = await self.update_vlan(
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
api_response = await mikrotik_client.update_vlan(
dot_id = existing_dot_id,
json_payload = {
"name": self.VLAN_NAME.format(vlan_id),
"interface": self.HW_INTERFACE_NAME,
"disabled": "false",
"comment": self.create_comment_json(
created_by = self.CREATED_BY_NAME,
created_ts = date_time.get_current_utc_date_time(as_string = False),
roll_back_config = existing_vlan_json
)
"comment": mikrotik_client.create_comment_json(roll_back_config = existing_vlan_json)
},
port_no = mikrotik_auth.nasPort,
use_https = use_https
)
@@ -453,12 +407,14 @@ class MikroTikPPPoE1000Controller(MikroTikController):
async def set_up_vlans(
self,
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth,
use_https: bool = True
) -> ApiResponse:
) -> MikroTikConfigAttemptResponse:
"""
To set up all the needed VLANs:
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
@@ -468,13 +424,7 @@ class MikroTikPPPoE1000Controller(MikroTikController):
step_response = MikroTikConfigAttemptResponse()
# Enlist all the existing VLANs:
api_response = await self.list_vlans(
mikrotik_ip = mikrotik_auth.nasIp,
port_no = mikrotik_auth.nasPort,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
use_https = use_https
)
api_response = await mikrotik_client.list_vlans(use_https = use_https)
# If the listing failed:
if not api_response.success:
@@ -484,28 +434,26 @@ class MikroTikPPPoE1000Controller(MikroTikController):
return step_response
# Extract the list here:
vlan_list = await api_response.get_json()
vlan_list = api_response.data
# Create a task for each VLAN that you need to set up,
# then fire them all asynchronously:
tasks = [
self.set_up_one_vlan(
mikrotik_auth = mikrotik_auth,
mikrotik_client = mikrotik_client,
vlan_id = vlan_id,
vlan_list = vlan_list,
use_https = use_https
) for vlan_id in mikrotik_auth.vlanRange
) for vlan_id in mikrotik_auth.vlanIds
]
results = await asyncio.gather(*tasks)
# Assess the results:
total_count = len(results)
success_count = 0
all_messages = []
for result in results:
if result.success: success_count += 1
all_messages.append(result.message)
step_response.message = " -> ".join(all_messages)
step_response.message = f"{success_count}/{total_count} VLAN(s) configured."
step_response.success = True if success_count == total_count else False
# Done here:
@@ -513,13 +461,13 @@ class MikroTikPPPoE1000Controller(MikroTikController):
async def roll_back_vlans(
self,
mikrotik_auth: MikroTikPPPoE1000Auth,
mikrotik_client: AsyncMikroTik,
use_https: bool = True
) -> MikroTikRollBackAttemptResponse:
) -> MikroTikConfigAttemptResponse:
"""
To reset VLANs to their original state (if they already existed) or remove them if this class added them.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:param mikrotik_client: The client to use.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
"""
@@ -531,13 +479,7 @@ class MikroTikPPPoE1000Controller(MikroTikController):
step_response = MikroTikConfigAttemptResponse()
# Enlist all the existing VLANs:
api_response = await self.list_vlans(
mikrotik_ip = mikrotik_auth.nasIp,
port_no = mikrotik_auth.nasPort,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
use_https = use_https
)
api_response = await mikrotik_client.list_vlans(use_https = use_https)
# If the listing failed:
if not api_response.success:
@@ -549,14 +491,17 @@ class MikroTikPPPoE1000Controller(MikroTikController):
# Loop through all the VLANs,
# and either reset them to their original state (if they already existed),
# or remove them entirely if they didn't exist before we came:
for vlan_json in await api_response.get_json():
for vlan_json in api_response.data:
# Check if the VLAN was created by this class:
if not self.created_by_easyfi(vlan_json): continue
if not mikrotik_client.is_my_config(
config_json = vlan_json,
name_substring = self.NAME_PREFIX
): continue
# Extract the rollback information:
comment = self.parse_comment_json(vlan_json.get("comment")) or {}
roll_back_json = comment.get("rollbackConfig")
comment_json = mikrotik_client.parse_comment_json(vlan_json.get("comment"))
roll_back_json = mikrotik_client.get_original_config(comment_json)
vlan_id = vlan_json.get("vlan-id")
dot_id = vlan_json.get(".id")
@@ -564,14 +509,7 @@ class MikroTikPPPoE1000Controller(MikroTikController):
if roll_back_json is None:
# We delete the resource:
await self.remove_vlan(
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
dot_id = dot_id,
port_no = mikrotik_auth.nasPort,
use_https = use_https
)
await mikrotik_client.remove_vlan(dot_id = dot_id, use_https = use_https)
# Assess the result:
total_count += 1
@@ -584,10 +522,7 @@ class MikroTikPPPoE1000Controller(MikroTikController):
else:
# We roll back to the previous configuration:
await self.update_vlan(
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
await mikrotik_client.update_vlan(
dot_id = dot_id,
json_payload = {
"name": roll_back_json.get("name"),
@@ -595,7 +530,6 @@ class MikroTikPPPoE1000Controller(MikroTikController):
"disabled": roll_back_json.get("disabled"),
"comment": ""
},
port_no = mikrotik_auth.nasPort,
use_https = use_https
)
@@ -607,9 +541,235 @@ class MikroTikPPPoE1000Controller(MikroTikController):
else: all_messages.append(f"Failed to roll back VLAN with id {vlan_id}")
# Assess the overall results:
all_messages.append(f"Removed/rolled-back {success_count}/{total_count} VLANs.")
if success_count == total_count: step_response.success = True
step_response.message = " -> ".join(all_messages)
step_response.message = f"Removed/rolled-back {success_count}/{total_count} VLAN(s)."
# Done here:
return step_response
# ┳┏┓ ┏┓ ┓
# ┃┃┃ ┃┃┏┓┏┓┃┏
# ┻┣┛ ┣┛┗┛┗┛┗┛
async def set_up_one_ip_pool(
self,
mikrotik_client: AsyncMikroTik,
ip_pool: dict,
existing_ip_pools: List[dict],
use_https: bool = True
) -> MikroTikConfigAttemptResponse:
"""
To set up one of the needed IP pools.
:param mikrotik_client: The client to use.
:param ip_pool: The IP pool you want configured by the end of the process.
:param existing_ip_pools: A list of the existing IP pools. Helps to decide between 'PUT' and 'PATCH' methods.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
"""
# Start by assuming failure:
step_response = MikroTikConfigAttemptResponse()
# Check if the configuration already exists:
existing_dot_id = None
existing_ip_pool_json = None
for existing_ip_pool in existing_ip_pools:
if existing_ip_pool.get("name", "???") == ip_pool["name"]:
existing_dot_id = existing_ip_pool[".id"]
existing_ip_pool_json = existing_ip_pool
break
# If the IP pool doesn't already exist:
if existing_dot_id is None:
# Add the configuration:
api_response = await mikrotik_client.add_ip_pool(
json_payload = {
"name": ip_pool["name"],
"ranges": ip_pool["network"],
"comment": mikrotik_client.create_comment_json()
},
use_https = use_https
)
# And assess the result:
if api_response.success:
step_response.success = True
step_response.message = f"Added new IP pool '{ip_pool['name']}'."
step_response.exception = None
else:
step_response.success = False
step_response.message = f"Failed to add new IP pool '{ip_pool['name']}'."
step_response.exception = api_response.exception
# If the IP pool already exists:
else:
# Update the configuration:
api_response = await mikrotik_client.update_ip_pool(
dot_id = existing_dot_id,
json_payload = {
"name": ip_pool["name"],
"ranges": ip_pool["network"],
"comment": mikrotik_client.create_comment_json(roll_back_config = existing_ip_pool_json)
},
use_https = use_https
)
# And assess the result:
if api_response.success:
step_response.success = True
step_response.message = f"Updated existing IP pool '{ip_pool['name']}'."
step_response.exception = None
else:
step_response.success = False
step_response.message = f"Failed to update existing IP pool '{ip_pool['name']}'."
step_response.exception = api_response.exception
# Done here:
return step_response
async def set_up_ip_pools(
self,
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth,
use_https: bool = True
) -> MikroTikConfigAttemptResponse:
"""
To set up ALL the needed IP pools.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
"""
# Start by assuming failure:
step_response = MikroTikConfigAttemptResponse()
# Enlist all the existing VLANs:
api_response = await mikrotik_client.list_ip_pools(use_https = use_https)
# If the listing failed:
if not api_response.success:
step_response.success = False
step_response.message = "Failed to enlist existing IP pools during configuration."
step_response.exception = step_response.exception
return step_response
# Extract the list here:
existing_ip_pools = api_response.data
# First we split the whole private IP range
# into the no. of VLANs we had been asked to make:
ip_pools = mikrotik_client.split_ipv4_range_equally(
start_ip = mikrotik_auth.firstPrivateIp,
end_ip = mikrotik_auth.lastPrivateIp,
targets = mikrotik_auth.vlanIds
)
# Create tasks to create IP Pools,
# and fire them asynchronously:
tasks = [
self.set_up_one_ip_pool(
mikrotik_client = mikrotik_client,
ip_pool = {
"name": self.PRIVATE_IP_POOL_NAME.format(ip_pool["target"]),
"network": ip_pool["network"],
},
existing_ip_pools = existing_ip_pools,
use_https = use_https
) for ip_pool in ip_pools
]
results = await asyncio.gather(*tasks)
# Assess the results:
total_count = len(results)
success_count = 0
for result in results:
if result.success: success_count += 1
step_response.message = f"{success_count}/{total_count} IP pool(s) configured."
step_response.success = True if success_count == total_count else False
# Done here:
return step_response
async def roll_back_ip_pools(
self,
mikrotik_client: AsyncMikroTik,
use_https: bool = True
) -> MikroTikConfigAttemptResponse:
"""
To reset IP pools to their original state (if they already existed) or remove them if this class added them.
:param mikrotik_client: The client to use.
:param use_https: Whether to use HTTPS, or HTTP.
:return: A structured response to indicate what happened during the process.
"""
# Start by assuming failure:
total_count = 0
success_count = 0
step_response = MikroTikConfigAttemptResponse()
# Enlist all the existing VLANs:
api_response = await mikrotik_client.list_ip_pools(use_https = use_https)
# If the listing failed:
if not api_response.success:
step_response.success = False
step_response.message = "Failed to enlist existing IP pools during rollback."
step_response.exception = step_response.exception
return step_response
# Loop through all the VLANs,
# and either reset them to their original state (if they already existed),
# or remove them entirely if they didn't exist before we came:
for resource_json in api_response.data:
# Check if the VLAN was created by this class:
if not mikrotik_client.is_my_config(
config_json = resource_json,
name_substring = self.NAME_PREFIX
): continue
# Extract the rollback information:
comment_json = mikrotik_client.parse_comment_json(resource_json.get("comment"))
roll_back_json = mikrotik_client.get_original_config(comment_json)
dot_id = resource_json.get(".id")
# If the resource has no rollback information:
if roll_back_json is None:
# We delete the resource:
await mikrotik_client.remove_ip_pool(dot_id = dot_id, use_https = use_https)
# Assess the result:
total_count += 1
if api_response.success: success_count += 1
# If the resource has rollback information:
else:
# We roll back to the previous configuration:
await mikrotik_client.update_ip_pool(
dot_id = dot_id,
json_payload = {
"name": roll_back_json.get("name"),
"ranges": roll_back_json.get("ranges"),
"comment": ""
},
use_https = use_https
)
# Assess the result:
total_count += 1
if api_response.success: success_count += 1
# Assess the overall results:
if success_count == total_count: step_response.success = True
step_response.message = f"Removed/rolled-back {success_count}/{total_count} IP pool(s)."
# Done here:
return step_response
@@ -634,31 +794,34 @@ class MikroTikPPPoE1000Controller(MikroTikController):
:return: A structured response to indicate what happened during authorization.
"""
# # print("IN-AUTH:", json.to_string(mikrotik_auth.model_dump(), default = str))
# print("VLAN SUBNETS:", json.to_string(
# self.split_ipv4_range_among_targets(
# start_ip = mikrotik_auth.firstPrivateIp,
# end_ip = mikrotik_auth.lastPrivateIp,
# targets = mikrotik_auth.vlanRange
# ),
# default = str
# ))
# Start with a blank response:
auth_response = MikroTikAuthResponse()
# Try to configure the system:
config_response = await self.configure(mikrotik_auth)
print("CONFIG RESPONSE:", config_response)
auth_response = config_response
# Create a MikroTik client:
mikrotik_client = AsyncMikroTik(
config_by = self.CREATED_BY_NAME,
mikrotik_ip = mikrotik_auth.nasIp,
username = mikrotik_auth.username,
password = mikrotik_auth.password,
port = mikrotik_auth.nasPort,
use_https = True,
http_client = None,
action_log_conn = mongo_data_conn
)
await asyncio.sleep(10.0)
# Try to configure the system:
config_response = await self.configure(mikrotik_client, mikrotik_auth)
print("CONFIG RESPONSE:", config_response)
await asyncio.sleep(2.0)
# Try to roll all configuration back:
roll_back_response = await self.roll_back(mikrotik_auth)
roll_back_response = await self.roll_back(mikrotik_client, mikrotik_auth)
print("ROLL-BACK RESPONSE:", roll_back_response)
config_response += roll_back_response
# Done here:
auth_response = config_response
return auth_response
# ┏┓ ┏•
@@ -668,30 +831,38 @@ class MikroTikPPPoE1000Controller(MikroTikController):
async def roll_back(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
) -> MikroTikRollBackAttemptResponse:
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth
) -> MikroTikConfigAttemptResponse:
"""
The rolling-back to the original state (as best as possible) in case the configurations fails midway after
completing some no. of steps.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
# Start with some variables:
keep_going = True
all_messages = []
all_messages = ["STARTING ROLLBACK."]
roll_back_response = MikroTikConfigAttemptResponse()
# Next we remove all the VLANs:
# Next, we roll back all the IP pools:
if keep_going:
step_response = await self.roll_back_vlans(mikrotik_auth, use_https = False)
step_response = await self.roll_back_ip_pools(mikrotik_client, use_https = False)
all_messages.append(step_response.message)
keep_going = step_response.success
# Next we roll back all the VLANs:
if keep_going:
step_response = await self.roll_back_vlans(mikrotik_client, use_https = False)
all_messages.append(step_response.message)
keep_going = step_response.success
# Next, we free-up the interface:
if keep_going:
step_response = await self.roll_back_interface(mikrotik_auth, use_https = False)
step_response = await self.roll_back_interface(mikrotik_client, use_https = False)
all_messages.append(step_response.message)
keep_going = step_response.success
@@ -702,30 +873,39 @@ class MikroTikPPPoE1000Controller(MikroTikController):
async def configure(
self,
mikrotik_auth: MikroTikPPPoE1000Auth | MikroTikHotspot1000Auth
mikrotik_client: AsyncMikroTik,
mikrotik_auth: MikroTikPPPoE1000Auth
) -> MikroTikConfigAttemptResponse:
"""
Run the configuration steps for the system.
:param mikrotik_client: The client to use.
:param mikrotik_auth: The set of credentials as received from the UI/API.
:return: A structured response to indicate what happened during the configuration attempt.
"""
# Start with some variables:
keep_going = True
all_messages = []
config_response = MikroTikConfigAttemptResponse()
all_messages = ["STARTING CONFIG."]
config_response = MikroTikConfigAttemptResponse(actionChain = mikrotik_client.action_chain)
# First, we arrange an interface:
if keep_going:
step_response = await self.set_up_interface(mikrotik_auth, use_https = False)
step_response = await self.set_up_interface(mikrotik_client, dot_id = "*3", use_https = False)
all_messages.append(step_response.message)
keep_going = step_response.success
# Next, we create the needed VLANs:
if keep_going:
await asyncio.sleep(0.25)
step_response = await self.set_up_vlans(mikrotik_auth, use_https = False)
step_response = await self.set_up_vlans(mikrotik_client, mikrotik_auth, use_https = False)
all_messages.append(step_response.message)
keep_going = step_response.success
# Next, we create the Private IP Pools:
if keep_going:
await asyncio.sleep(0.25)
step_response = await self.set_up_ip_pools(mikrotik_client, mikrotik_auth, use_https = False)
all_messages.append(step_response.message)
keep_going = step_response.success