(20260109) Almost done with Cosec-TCAOFF sync.
This commit is contained in:
@@ -0,0 +1,55 @@
|
||||
import time
|
||||
from utils_v2.system import files
|
||||
from utils_v2.cache_v2.async_mem_cache import AsyncMemCache
|
||||
from utils_v2.cache_v2.async_disk_cache import AsyncDiskCache
|
||||
import asyncio
|
||||
|
||||
async def main():
|
||||
|
||||
proj_dir = files.get_file_directory(include_filename = False)
|
||||
proj_dir = files.get_parent_directory(proj_dir, depth = 1)
|
||||
print("PROJ. DIR.:", proj_dir)
|
||||
|
||||
memcache = AsyncMemCache()
|
||||
# diskcache = AsyncDiskCache(
|
||||
# caching_dir = os.path.join(proj_dir, "local"),
|
||||
# lock_on_read = False,
|
||||
# lock_wait_timeout = 5.0,
|
||||
# expiry_check_interval = 60.0
|
||||
# )
|
||||
|
||||
cache = memcache
|
||||
|
||||
# print("\n\n---\n\n")
|
||||
# print("EXPIRY CHECK:")
|
||||
# await cache.set("k1", "v1", expiry = 3.0)
|
||||
# print(await cache.get("k1"))
|
||||
# time.sleep(3.1)
|
||||
# print(await cache.get("k1"))
|
||||
|
||||
print("\n\n---\n\n")
|
||||
print("DELETE CHECK:")
|
||||
await cache.set("k1", "v1", expiry = 30.0)
|
||||
await cache.set("k2", "v2", expiry = 30.0)
|
||||
await cache.set("k3", "v3", expiry = 30.0)
|
||||
print("LIST:", await cache.list_keys())
|
||||
print("LIST:", await cache.list_keys(match = "k[1-2]"))
|
||||
print(await cache.get("k1"))
|
||||
await cache.delete("k1")
|
||||
print(await cache.get("k1"))
|
||||
print("LIST:", await cache.list_keys())
|
||||
|
||||
# print("\n\n---\n\n")
|
||||
# print("COUNTING:")
|
||||
# count = await cache.count("c1", expiry = 3.1)
|
||||
# for i in range(10):
|
||||
# time.sleep(1.0)
|
||||
# count = await cache.count("c1", expiry = 3.0, raise_exception = True)
|
||||
# ttl = await cache.ttl("c1")
|
||||
# print(count, await cache.get("c1"), ttl)
|
||||
# print("OUTSIDE LOOP!")
|
||||
# print(await cache.get("c1"))
|
||||
# time.sleep(3.1)
|
||||
# print(await cache.get("c1"))
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,9 @@
|
||||
from utils_v2.string import json
|
||||
|
||||
data = json.from_file(r"D:\kps\PycharmProjects\cosec\local\cache\muster_roll_cache.json")
|
||||
|
||||
for d in data["report"]:
|
||||
if d["Branch Name"] == "Client": continue
|
||||
if str(d["Category Name"]).lower().strip() != str(d["Grade Name"]).lower().strip():
|
||||
print(json.to_string(d))
|
||||
print("\n\n---\n\n")
|
||||
@@ -0,0 +1,141 @@
|
||||
# IPDR at VKNTPL
|
||||
|
||||
Implementation by **Bhushan C Thakkar** and **Khushal P Soonderji** in November of 2025.
|
||||
|
||||
_Documentation made on 20251128_
|
||||
|
||||
---
|
||||
|
||||
## 1. Login:
|
||||
|
||||
The username is `vknipdr` and the server address is `ipdr.prysmnet.com`. You may SSH into the device to perform your activities.
|
||||
|
||||
>ssh vknipdr@ipdr.prysmnet.com
|
||||
|
||||
---
|
||||
|
||||
## 2. Capturing Logs With Syslog-ng:
|
||||
|
||||
The system uses **syslog-ng** to capture IPDR-related logs from **Mikrotik** hardware and saves them to a file on the server in fast and efficient **CSV format**.
|
||||
|
||||
The file that hold the config can be accessed by the following command (`sudo` required):
|
||||
|
||||
> nano /etc/syslog-ng/conf.d/mikrotik.conf
|
||||
|
||||
Here is the code snippet that makes the magic work:
|
||||
|
||||
```text
|
||||
# ------------------------------------------------------------
|
||||
# SOURCE — receive logs from MikroTik
|
||||
# ------------------------------------------------------------
|
||||
source s_mikrotik_nat {
|
||||
udp(ip(0.0.0.0) port(514) keep-hostname(yes));
|
||||
|
||||
};
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Filter - accept on Forward Chains
|
||||
# ------------------------------------------------------------
|
||||
|
||||
filter f_forward {
|
||||
match("forward:") and not match("src-mac");
|
||||
};
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# PARSER — extract NAT fields using regex (flat declaration)
|
||||
# ------------------------------------------------------------
|
||||
|
||||
parser p_nat_forward_regex {
|
||||
regexp-parser(
|
||||
template("${MESSAGE}")
|
||||
patterns(
|
||||
"forward: in:(?<in>.*) out:(?<out>.*), proto (?<proto>.*), (?<src_ip>[\\d\\.]*):(?<src_port>[\\d]+)->(?<dst_ip>[\\d\\.]*):(?<dst_port>[\\d]+)(?:, NAT \\(.*?(?<nat_trans_ip>[\\d\\.]+):(?<nat_trans_port>[\\d]+)\\)[^,]*)?, len (?<len>\\d+)"
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# TEMPLATE —
|
||||
# ------------------------------------------------------------
|
||||
template t_nat_csv {
|
||||
template("${DATE},$src_ip,$src_port,$nat_trans_ip,$nat_trans_port,$dst_ip,$dst_port\n");
|
||||
template-escape(no);
|
||||
};
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# DESTINATION — MongoDB
|
||||
# ------------------------------------------------------------
|
||||
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# DESTINATION — Raw log file (for testing)
|
||||
# ------------------------------------------------------------
|
||||
destination d_rawfile {
|
||||
file(
|
||||
"/mnt/storage/syslog-ng/prysmnet/mikrotik-raw.log"
|
||||
template(t_nat_csv)
|
||||
flush_lines(1)
|
||||
);
|
||||
};
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# LOG PATH — Write all incoming MikroTik messages directly to file
|
||||
# ------------------------------------------------------------
|
||||
log {
|
||||
source(s_mikrotik_nat);
|
||||
filter(f_forward);
|
||||
parser(p_nat_forward_regex);
|
||||
destination(d_rawfile);
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Restarting Syslog-ng:
|
||||
|
||||
Once you have saved your config file, you will need to restart the process with the following command (`sudo` required):
|
||||
|
||||
> systemctl restart syslog-ng
|
||||
|
||||
---
|
||||
|
||||
## 4. Checking The Logs:
|
||||
|
||||
You can see a *live trail* of the logs by running the following command:
|
||||
|
||||
> tail -f -n 10 /mnt/storage/syslog-ng/prysmnet/mikrotik-raw.log
|
||||
|
||||
You will see the logs in the following format:
|
||||
|
||||
```text
|
||||
Nov 11 14:39:03,10.252.247.94,54084,103.171.2.187,54084,142.250.194.234,443
|
||||
Nov 11 14:39:03,10.252.255.135,56424,103.171.2.219,56424,123.63.54.23,443
|
||||
Nov 11 14:39:03,10.252.247.94,54084,,,142.250.194.234,443
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Log Rotation:
|
||||
|
||||
Logs can add up very fast and the file can become unreasonable large and tough to manage. For this, we need to perform
|
||||
log rotation. A service named `logrotate` has been used for the same. At the time of first setup, the file was set up such
|
||||
that the configuration could be seen by running the following command:
|
||||
|
||||
> cat /etc/logrotate.d/syslog-forward
|
||||
|
||||
And the contents were:
|
||||
|
||||
```text
|
||||
/var/log/syslog-ng/mikrotik-raw.log{
|
||||
size 20G
|
||||
rotate 365
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
copytruncate
|
||||
dateext
|
||||
dateformat -%Y-%m-%d_%H-%M-%S
|
||||
}
|
||||
```
|
||||
Reference in New Issue
Block a user