khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resetting utils subtree.

Browse Source
This commit is contained in:
khushal
2025-02-04 17:56:45 +05:30
parent c5c8e34d5a
commit 08e1d153bb
205 changed files with 42 additions and 144963 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils_v2/.gitignore
-8
View File
@@ -1,8 +0,0 @@
/.venv/
/.idea/
**/__pycache__/
__pycache__/
*.pem
*.pyc
*.pyd
utils_v2/__init__.py
View File
utils_v2/ai/__init__.py
View File
utils_v2/ai/image_classification/__init__.py
View File
utils_v2/ai/image_classification/async_blur.py
-202
View File
@@ -1,202 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Monday, 2nd Sept., 2024
OBJECTIVE:
To provide an easy way to assess images for blurriness.
Tried and implemented using HuggingFace behaviour_models.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# For using the AI model:
import torch
from transformers import pipeline
# To download images from URLs:
import requests
# To read images:
from PIL import Image
# Common:
from shared import variables
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AssessImageBlur:
def __init__(
self,
model,
blur_label = "Blur",
clarity_label = "Normal",
blur_threshold = 0.25,
clarity_threshold = 0.65
):
"""
This class assesses the input image and tells if it is blurry or clear.
:param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is
stored.
:param blur_threshold: The max allowed blurriness (0 to 1 range).
:param clarity_threshold: The minimum needed clarity (0 to 1 range).
"""
# Note down the config:
self.__model = model
self.__device = "gpu" if torch.cuda.is_available() else "cpu"
self.__blur_label = blur_label
self.__clarity_label = clarity_label
self.__blur_threshold = blur_threshold
self.__clarity_threshold = clarity_threshold
# Initialize the classifier:
self.__classifier = pipeline(
task = "image-classification",
model = self.__model,
device = self.__device
)
@staticmethod
async def read_image(source):
"""
Reads an image in whichever format it is provided and returns it as a PIL object.
:param source: The image as either a path or a URL or a io.BytesIO object.
:return: The image opened as a PIL object.
"""
# If the input image is already a PIL image:
if isinstance(source, Image.Image): return source
# If a buffer is provided:
elif isinstance(source, io.BytesIO):
source.seek(0)
return Image.open(source)
# If a string is provided (local path or URL):
elif isinstance(source, str):
if source.startswith("http://") or source.startswith("https://"):
response = await variables.http_client.get(source)
return Image.open(io.BytesIO(response.content))
else: return Image.open(source)
async def classify(self, image):
"""
To get the prediction of the model from the given input image.
:param image: The image (as a PIL object or file path or a URL).
:return: The dictionary of classes with their respective predictions.
"""
image = await self.read_image(image)
predictions = self.__classifier(image)
classes = {p["label"]: p["score"] for p in predictions}
return classes
async def is_ok(
self,
image,
blur_threshold = None,
clarity_threshold = None
):
"""
Checks if the image given to it can be used, or should be rejected.
:param image: The image (as a PIL object or file path or a URL).
:param blur_threshold: A custom threshold to test against. If not provided, the default will be taken that was
provided when the instance was created.
:param clarity_threshold: A custom threshold to test against. If not provided, the default will be taken that
was provided when the instance was created.
:return: True if the image is okay, else False.
"""
image = await self.read_image(image)
blur_threshold = blur_threshold or self.__blur_threshold
clarity_threshold = clarity_threshold or self.__clarity_threshold
classes = await self.classify(image)
if (
classes[self.__blur_label] <= blur_threshold and
classes[self.__clarity_label] >= clarity_threshold
): return True
return False
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
async def main():
image = r"/home/developer/Downloads/low-res-check.png"
assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh")
usable = await assessor.is_ok(image)
classes = await assessor.classify(image)
print("IS OKAY:", usable)
print("CLASSES:", classes)
asyncio.run(main())
utils_v2/ai/image_classification/async_nsfw.py
-188
View File
@@ -1,188 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Monday, 2nd Sept., 2024
OBJECTIVE:
To provide an easy way to assess images for adult content.
Tried and implemented using HuggingFace behaviour_models.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# For using the AI model:
import torch
from transformers import pipeline
# To download images from URLs:
import requests
# To read images:
from PIL import Image
# Common:
from shared import variables
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AssessImageNSFW:
def __init__(
self,
model,
nsfw_label = "nsfw",
nsfw_threshold = 0.25,
):
"""
This class assesses the input image and tells if it is blurry or clear.
:param model: The model to use. Either the name of the HuggingFace repo, or the directory where the model is
stored.
:param nsfw_threshold: The max allowed blurriness (0 to 1 range).
"""
# Note down the config:
self.__model = model
self.__device = "gpu" if torch.cuda.is_available() else "cpu"
self.__nsfw_label = nsfw_label
self.__nsfw_threshold = nsfw_threshold
# Initialize the classifier:
self.__classifier = pipeline(
task = "image-classification",
model = self.__model,
device = self.__device
)
@staticmethod
async def read_image(source):
"""
Reads an image in whichever format it is provided and returns it as a PIL object.
:param source: The image as either a path or a URL or a io.BytesIO object.
:return: The image opened as a PIL object.
"""
# If the input image is already a PIL image:
if isinstance(source, Image.Image): return source
# If a buffer is provided:
elif isinstance(source, io.BytesIO):
source.seek(0)
return Image.open(source)
# If a string is provided (local path or URL):
elif isinstance(source, str):
if source.startswith("http://") or source.startswith("https://"):
response = await variables.http_client.get(source)
return Image.open(io.BytesIO(response.content))
else: return Image.open(source)
async def classify(self, image):
"""
To get the prediction of the model from the given input image.
:param image: The image (as a PIL object or file path or a URL).
:return: The dictionary of classes with their respective predictions.
"""
image = await self.read_image(image)
predictions = self.__classifier(image)
classes = {p["label"]: p["score"] for p in predictions}
return classes
async def is_ok(
self,
image,
nsfw_threshold = None
):
"""
Checks if the image given to it can be used, or should be rejected.
:param image: The image (as a PIL object or file path or a URL).
:param nsfw_threshold: A custom threshold to test against. If not provided, the default will be taken that was
provided when the instance was created.
:return: True if the image is okay, else False.
"""
image = await self.read_image(image)
nsfw_threshold = nsfw_threshold or self.__nsfw_threshold
classes = await self.classify(image)
return True if classes[self.__nsfw_label] <= nsfw_threshold else False
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
async def main():
assessor = AssessImageNSFW(model = r"/path/to/model")
usable = await assessor.is_ok(r"/path/to/image/img.jpg")
classes = await assessor.classify(r"https://...")
asyncio.run(main())
utils_v2/ai/image_segmentation/__init__.py
View File
utils_v2/ai/image_segmentation/async_dichotomous_segmentation.py
-223
View File
@@ -1,223 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Saturday, 14th Sept., 2024
OBJECTIVE:
To provide an easy way to get masks from dichotomous image segmentation.
This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait".
You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This
means that the model should have only two classes like "foreground", and "background". The specified model was
trained for implementing portrait mode style blurring of backgrounds.
The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not
support drop-in replacement for other behaviour_models, please be aware about this.
REFERENCES:
01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait
02. https://github.com/ZhengPeng7/BiRefNet
DOWNLOADS:
01. https://huggingface.co/ZhengPeng7/BiRefNet-portrait/tree/main
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For system-level activities:
import io
# To work with PIL images:
from PIL import Image
# To make asynchronous HTTP calls:
from shared.variables import http_client
# For using the AI model:
import torch
from torchvision import transforms
from transformers import AutoModelForImageSegmentation
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class DichotomousSegmenter:
def __init__(
self,
model = r"zhengpeng7/BiRefNet-portrait"
):
"""
This class uses Dichotomous Image Segmentation to produce a mask of what is in the foreground (or what is of
interest in a given image).
:param model: The path of the model.
"""
# Check if GPU is available for faster predictions:
self.__cuda_is_available = True if torch.cuda.is_available() else False
# Initialize the model:
self.__model = AutoModelForImageSegmentation.from_pretrained(
pretrained_model_name_or_path = model,
trust_remote_code = True
)
torch.set_float32_matmul_precision(["high", "highest"][0])
self.__model.to("cuda" if self.__cuda_is_available else "cpu")
@staticmethod
async def read_image(source):
"""
Reads an image in whichever format it is provided and returns it as a PIL object.
:param source: The image as either a path or a URL or a io.BytesIO object.
:return: The image opened as a PIL object.
"""
# If the input image is already a PIL image:
if isinstance(source, Image.Image): return source
# If a buffer is provided:
elif isinstance(source, io.BytesIO):
source.seek(0)
return Image.open(source)
# If a string is provided (local path or URL):
elif isinstance(source, str):
if source.startswith("http://") or source.startswith("https://"):
response = await http_client.get(source)
return Image.open(io.BytesIO(response.content))
else: return Image.open(source)
async def get_mask(self, image):
"""
Process the image to figure out the mask. In some cases (like the image of a sunset) you will have no white
pixels in the mask. White pixels represent the areas that the AI considered to be the foreground (area of
interest) and the black parts are the background.
:param image: The image data as either a PIL object, or a path to a file on the local disk, or a URL.
:return: The mask as a PIL object.
"""
# Prepare the image transformer:
image_size = (1024, 1024)
transform_image = transforms.Compose([
transforms.Resize(image_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
# Read and transform the image:
image = await self.read_image(image)
input_image = transform_image(image).unsqueeze(0).to("cuda" if self.__cuda_is_available else "cpu")
# Make the prediction:
with torch.no_grad():
if self.__cuda_is_available: mask = self.__model(input_image)[-1].sigmoid().cuda()
else: mask = self.__model(input_image)[-1].sigmoid().cpu()
# Create a PIL object of the mask data:
mask = mask[0].squeeze()
mask = transforms.ToPILImage()(mask)
mask = mask.resize(image.size)
# Done here:
return mask
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
import numpy as np
async def main():
model = DichotomousSegmenter()
images = [
# r"/home/developer/Downloads/kate.jpg",
# r"/home/developer/Downloads/IMG-20240730-WA0001.jpg",
# r"/home/developer/Downloads/pexels-pixabay-57416.jpg",
# r"/home/developer/Downloads/summer_clothes.jpg",
# r"/home/developer/Downloads/low_res_cat - upscaled.png",
# r"/home/developer/Downloads/blurry_traffic.jpg",
r"/home/developer/Downloads/card_0.jpg",
r"/home/developer/Downloads/card_1.jpg",
r"/home/developer/Downloads/card_2.jpg",
r"/home/developer/Downloads/card_3.jpg",
r"/home/developer/Downloads/sushmita_card.jpg",
r"/home/developer/Downloads/niranjan_card.jpg",
r"/home/developer/Downloads/niranjan_card_2.jpg",
]
for image in images:
print(image.split("/")[-1])
image = await DichotomousSegmenter.read_image(image)
mask = await model.get_mask(image)
image.show()
mask.show()
print("MASK!")
array = np.array(mask)
print("Dimensions of the array:", array.shape)
masked_count = np.sum(array > 125)
print(masked_count, type(masked_count), np.size, type(np.size))
percentage = masked_count / np.prod(array.shape)
print("Area:", round(percentage * 100, 4))
# image.putalpha(mask)
# image.show()
# mask.show()
asyncio.run(main())
utils_v2/ai/object_detection/__init__.py
View File
utils_v2/ai/object_detection/yolo.py
-176
View File
@@ -1,176 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 4th Jul, 2024
OBJECTIVE:
To provide a class to detect objects in images using YOLO behaviour_models.
REFERENCES:
1) Code Examples: https://docs.ultralytics.com/usage/python/
2) Models: https://docs.ultralytics.com/models/yolov8/#supported-tasks-and-modes
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import os
# To use YOLO architecture:
from ultralytics import YOLO
# To work with images:
from PIL import Image
# utils:
from utils_v2.string import json
# Common:
from shared import constants
# For debugging:
from icecream import IceCreamDebugger
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class YoloDetect:
def __init__(
self,
model_file,
debug = True,
debug_prefix = "YOLO | "
):
# Load the model:
self._model = YOLO(model = model_file)
# Initialize the dbugging tool:
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self._printer.disable()
def disable_debug(self):
self._printer.disable()
def enable_debug(self):
self._printer.enable()
@staticmethod
def _process_result(raw_result):
# Extract the various types of results available from the inference:
speed = {k: (v / 100.0) for k, v in raw_result.speed.items()}
class_mapping = raw_result.names
probabilities = raw_result.probs
boxes = raw_result.boxes
masks = raw_result.masks
# Construct a default response:
response_json = {
"speed": speed,
"classes": class_mapping,
"boxes": None
}
# Object-Detection results:
formatted_boxes = []
for box in boxes:
detected_class = int(box.cls.numpy()[0])
x1, y1, x2, y2 = box.xyxy[0]
formatted_boxes.append(
{
"class": detected_class,
"className": class_mapping[detected_class],
"confidence": float(box.conf.numpy()[0]),
"x1": int(x1.numpy()),
"y1": int(y1.numpy()),
"x2": int(x2.numpy()),
"y2": int(y2.numpy())
}
)
response_json["boxes"] = formatted_boxes
# Done here:
return response_json
def predict(self, image, show = False, verbose = False):
# Process the input image with the given task:
results = self._model(
source = image,
show = show,
verbose = verbose
)
# Based on the task, interpret the results:
results_json = self._process_result(results[0])
# Done here:
return results_json
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
# model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt")
model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt"
# sample_image_path = r"/home/developer/Downloads/2_cats.jpg"
sample_image_path = r"/home/developer/Downloads/flattened_image.jpg"
my_yolo = YoloDetect(model_file = model_file_path)
results = my_yolo.predict(image = Image.open(sample_image_path))
print("FINAL RESULTS:", json.to_string(results))
utils_v2/ai/tts/__init__.py
View File
utils_v2/ai/tts/easy_tts.py
-175
View File
@@ -1,175 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 26th Sept., 2024
OBJECTIVE:
To provide a fast way to make audio from TTS engines.
REFERENCES:
01. Usage: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#python-api
02. Installation: https://github.com/myshell-ai/MeloTTS/blob/main/docs/install.md#linux-and-macos-install
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# To use the AI:
from melo.api import TTS
import numpy as np
import soundfile
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class EasyTTS:
__sampling_rate = 44_100
def __init__(
self,
language = "EN",
speed = 1.0
):
self.__audio = np.zeros(1)
self.__language = language
self.__speed = speed
self.__model = TTS(language = language, device = "auto")
self.__speakers = self.__model.hps.data.spk2id
def list_speakers(self):
return list(self.__speakers.keys())
def speak(
self,
text,
speaker
):
this_audio = self.__model.tts_to_file(
text,
self.__speakers[speaker],
speed = self.__speed,
quiet = True
)
self.__audio = np.concatenate((self.__audio, this_audio))
def pause(self, seconds):
self.__audio = np.concatenate((
self.__audio,
np.zeros(int(self.__sampling_rate * seconds))
))
def to_wav(self, path = None):
if path is None:
audio_buffer = io.BytesIO()
soundfile.write(audio_buffer, self.__audio, self.__sampling_rate)
return audio_buffer
else: soundfile.write(path, self.__audio, self.__sampling_rate)
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
speaker = "EN-BR"
tts_maker = EasyTTS(language = "EN", speed = 0.9)
tts_maker.speak(
text = """
Imagine delighting your doctors with a personalized calendar,
crafted from their own cherished memories.
""",
speaker = speaker
)
tts_maker.pause(seconds = 0.3)
tts_maker.speak(
text = """
Every day, as they turn the page,
they’ll not only relive those special moments but also remember you,
the one who made it happen.
""",
speaker = speaker
)
tts_maker.pause(seconds = 0.75)
tts_maker.speak(
text = "STEP 1:",
speaker = speaker
)
tts_maker.pause(seconds = 0.3)
tts_maker.speak(
text = "Start by identifying the doctors you’d like to engage with, and add them to our app.",
speaker = speaker
)
tts_maker.pause(seconds = 0.3)
tts_maker.speak(
text = "No rush, you can add their photographs later as well.",
speaker = speaker
)
tts_maker.pause(seconds = 0.3)
tts_maker.speak(
text = "With this, your engagement funnel is created.",
speaker = speaker
)
tts_maker.to_wav(r"/home/developer/Downloads/voiceover.wav")
utils_v2/ai/tts/test.wav
BIN
View File
Binary file not shown.
utils_v2/api/__init__.py
View File
utils_v2/api/__pycache__/__init__.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/api/__pycache__/async_quart.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/api/__pycache__/codes.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/api/__pycache__/log.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/api/__pycache__/metrics_prometheus.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/api/__pycache__/response.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/api/async_quart.py
-1276
View File
File diff suppressed because it is too large Load Diff
utils_v2/api/codes.py
-164
View File
@@ -1,164 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 12th Sept., 2024
OBJECTIVE:
To maintain all status codes in one place.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
from enum import Enum, unique
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
@unique
class HttpCodes(Enum):
"""
Commonly used standard HTTP status codes.
Can be sent after an API Call is processed.
Refer to: https://http.dev/status
NOTE: THIS LIST IS NOT EXHAUSTIVE!
"""
# 1XX - Informational:
CONTINUE = 100 # .............. https://http.dev/102
SWITCHING_PROTOCOLS = 101 # ... https://http.dev/101
PROCESSING = 102 # ............ https://http.dev/102
EARLY_HINTS = 103 # ........... https://http.dev/103
# 2XX - Success:
SUCCESS = 200 # .................. https://http.dev/200
CREATED = 201 # .................. https://http.dev/201
ACCEPTED = 202 # ................. https://http.dev/202
NON_AUTHORITATIVE_INFO = 203 # ... https://http.dev/203
NO_CONTENT = 204 # ............... https://http.dev/204
RESET_CONTENT = 205 # ............ https://http.dev/205
PARTIAL_CONTENT = 206 # .......... https://http.dev/206
MULTI_STATUS = 207 # ............. https://http.dev/207
ALREADY_REPORTED = 208 # ......... https://http.dev/208
THIS_IS_FINE = 218 # ............. https://http.dev/218
IM_USED = 226 # .................. https://http.dev/226
# 3XX - Redirection:
MULTIPLE_CHOICES = 300 # ..... https://http.dev/300
MOVED_PERMANENTLY = 301 # .... https://http.dev/301
MOVED_TEMPORARILY = 302 # .... https://http.dev/302
SEE_OTHER = 303 # ............ https://http.dev/303
NOT_MODIFIED = 304 # ......... https://http.dev/304
USE_PROXY = 305 # ............ https://http.dev/305
SWITCH_PROXY = 306 # ......... https://http.dev/306
TEMPORARY_REDIRECT = 307 # ... https://http.dev/307
PERMANENT_REDIRECT = 308 # ... https://http.dev/308
# 4XX - Client Errors:
BAD_REQUEST = 400 # ..................... https://http.dev/401
UNAUTHORIZED = 401 # .................... https://http.dev/401
PAYMENT_REQUIRED = 402 # ................ https://http.dev/402
FORBIDDEN = 403 # ....................... https://http.dev/403
NOT_FOUND = 404 # ....................... https://http.dev/404
METHOD_NOT_ALLOWED = 405 # .............. https://http.dev/405
NOT_ACCEPTABLE = 406 # .................. https://http.dev/406
PROXY_AUTH_REQUIRED = 407 # ............. https://http.dev/407
REQUEST_TIMEOUT = 408 # ................. https://http.dev/408
CONFLICT = 409 # ........................ https://http.dev/409
GONE = 410 # ............................ https://http.dev/410
LENGTH_REQUIRED = 411 # ................. https://http.dev/411
PRECONDITION_FAILED = 412 # ............. https://http.dev/412
PAYLOAD_TOO_LARGE = 413 # ............... https://http.dev/413
URI_TOO_LONG = 414 # .................... https://http.dev/414
UNSUPPORTED_MEDIA_TYPE = 415 # .......... https://http.dev/415
PAGE_EXPIRED = 419 # .................... https://http.dev/419
TOO_MANY_REQUESTS = 429 # ............... https://http.dev/429
UNAVAILABLE_FOR_LEGAL_REASONS = 451 # ... https://http.dev/451
INVALID_TOKEN = 498 # ................... https://http.dev/498
CLIENT_CLOSED_REQUEST = 499 # ........... https://http.dev/499
# 5XX - Server Errors:
INTERNAL_SERVER_ERROR = 500 # ........... https://http.dev/500
NOT_IMPLEMENTED = 501 # ................. https://http.dev/501
BAD_GATEWAY = 502 # ..................... https://http.dev/502
SERVICE_UNAVAILABLE = 503 # ............. https://http.dev/503
GATEWAY_TIMEOUT = 504 # ................. https://http.dev/504
HTTP_VERSION_NOT_SUPPORTED = 505 # ...... https://http.dev/505
VARIANT_ALSO_NEGOTIATES = 506 # ......... https://http.dev/506
INSUFFICIENT_STORAGE = 507 # ............ https://http.dev/507
LOOP_DETECTED = 508 # ................... https://http.dev/508
BANDWIDTH_LIMIT_EXCEEDED = 509 # ........ https://http.dev/509
WEB_SERVER_DOWN = 521 # ................. https://http.dev/521
ORIGIN_IS_UNREACHABLE = 523 # ........... https://http.dev/523
SERVICE_IS_OVERLOADED = 529 # ........... https://http.dev/529
NETWORK_READ_TIMEOUT_ERROR = 598 # ...... https://http.dev/598
NETWORK_CONNECT_TIMEOUT_ERROR = 599 # ... https://http.dev/599
# ---------------------------------------------------------------------------------------------------------------------
@unique
class StatusCodes(Enum):
"""
To be used internally within the context of your service. Customize these to match your service.
The format is: (SUCCESS_INDICATOR, INTERNAL_NUMERIC_CODE, HTTP_CODE)
Example: (True, 1, 200)
"""
# Legacy Codes:
OK = (True, 1, HttpCodes.SUCCESS.value)
FAILED = (False, 0, HttpCodes.INTERNAL_SERVER_ERROR.value)
PARTIAL_SUCCESS = (True, 2, HttpCodes.PARTIAL_CONTENT.value)
PARTIAL_FAILURE = (False, 3, HttpCodes.PARTIAL_CONTENT.value)
# Authentication Codes:
LOGGED_IN_SUCCESSFULLY = (True, 200, HttpCodes.SUCCESS.value)
LOGIN_FAILED = (False, 201, HttpCodes.UNAUTHORIZED.value)
INVALID_SESSION_TOKEN = (False, 202, HttpCodes.UNAUTHORIZED.value)
AUTHENTICATION_DETAILS_INCOMPLETE = (False, 203, HttpCodes.BAD_REQUEST.value)
# Authorization Codes:
AUTHORIZED_SUCCESSFULLY = (True, 300, HttpCodes.SUCCESS.value)
NOT_ALLOWED = (False, 301, HttpCodes.FORBIDDEN.value)
AUTHORIZATION_DETAILS_INCOMPLETE = (False, 302, HttpCodes.BAD_REQUEST.value)
AUTHORIZATION_FAILED = (False, 303, HttpCodes.UNAUTHORIZED.value)
# General failures:
DOWN_FOR_MAINTENANCE = (False, 800, HttpCodes.SERVICE_UNAVAILABLE.value)
UNKNOWN_ERROR = (False, 801, HttpCodes.INTERNAL_SERVER_ERROR.value)
DATA_INCOMPLETE = (False, 802, HttpCodes.BAD_REQUEST.value)
HEADERS_INCOMPLETE = (False, 803, HttpCodes.BAD_REQUEST.value)
FILES_MISSING = (False, 804, HttpCodes.BAD_REQUEST.value)
CLIENT_CLOSED_REQUEST = (False, 805, HttpCodes.CLIENT_CLOSED_REQUEST.value)
# Validation failure:
DATA_VALIDATION_FAILURE = (False, 900, HttpCodes.BAD_REQUEST.value)
RATE_LIMIT_EXCEEDED = (False, 901, HttpCodes.TOO_MANY_REQUESTS.value)
utils_v2/api/log.py
-139
View File
@@ -1,139 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 12th Sept., 2024
OBJECTIVE:
To have a structure to the response sent from the API calls.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# System-level activities:
import distro
import socket
import platform
# For data-modelling:
from pydantic import BaseModel, Field
from typing import Any, Optional, List, Literal
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Info for logging that will stay constant during runtime:
SERVER_HOSTNAME = str(socket.gethostname())
PLATFORM_INFO = platform.uname()
HOST_OS = str(distro.name(True))
HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})"
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class APILogModel(BaseModel):
# To identify the machine the code is running on.
# DO NOT MODIFY THESE:
hostname: str = SERVER_HOSTNAME
os: str = HOST_OS
cpu: str = HOST_CPU
# Can modify these:
pid: Optional[Any] = None
ppid: Optional[Any] = None
# To identify the project and actions:
project: Optional[str] = None
log: str
operation: str
apiVer: Optional[str] = None
logId: Optional[str] = None
logChain: Optional[str] = None
# Timing metrics:
ts: datetime.datetime
tat: float
cpuTime: float
# To understand the request that came in:
sessionInfo: Optional[Any] = None
method: Optional[str] = None
url: Optional[str] = None
route: Optional[str] = None
headers: Optional[Any] = None
data: Optional[Any] = None
files: Optional[Any] = None
# To understand the output that went out:
exception: Optional[Any] = None
response: Optional[Any] = None
httpCode: Optional[int] = None
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
my_log = APILogModel(
log = "internal"
)
print(my_log)
utils_v2/api/response.py
-133
View File
@@ -1,133 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 12th Sept., 2024
OBJECTIVE:
To have a structure to the response sent from the API calls.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# For data-modelling:
from pydantic import BaseModel
from typing import Any, Optional, List
# My utils:
from utils_v2.api.codes import StatusCodes, HttpCodes
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class ResponseModel(BaseModel):
"""
A model for how the response should be when developing API endpoints.
"""
# The fields that you want in your response:
status_code: StatusCodes
message: Optional[str | List] = None
data: Optional[Any] = None
seconds: Optional[float | int] = None
log_id: Optional[str] = None
http_code: Optional[HttpCodes] = None
api_version: Optional[str] = None
def for_quart(self):
"""
Call this when you are using either Flask or Quart as your framework.
:return: The output as expected by Flask and Quart.
"""
# Figure out the HTTP code:
response_http_code = self.http_code.value if self.http_code is not None else self.status_code.value[2]
# Construct the basic structure:
response_dict = {
"status": 1 if self.status_code.value[0] else 0,
"code": response_http_code,
"message": self.message or self.status_code.name.replace("_", " ").lower(),
"data": self.data,
"apiVer": self.api_version
}
# Now add the additional fields:
if self.seconds is not None: response_dict["seconds"] = self.seconds
if self.log_id is not None: response_dict["logId"] = self.log_id
# Done here:
return response_dict, response_http_code
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
my_response = ResponseModel(
status_code = StatusCodes.RATE_LIMIT_EXCEEDED
)
my_response.log_id = "abc123"
print(my_response.for_quart())
utils_v2/cache/__init__.py
Vendored
View File
utils_v2/cache/__pycache__/__init__.cpython-310.pyc
Vendored
BIN
View File
Binary file not shown.
utils_v2/cache/__pycache__/async_redis_cache.cpython-310.pyc
Vendored
BIN
View File
Binary file not shown.
utils_v2/cache/async_redis_cache.py
Vendored
-497
View File
@@ -1,497 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 11th April, 2024
OBJECTIVE:
To provide an easy way to work with '.json' files in an async manner.
REFERENCES:
1) https://www.w3schools.com/python/python_json.asp
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To use redis:
import redis.asyncio as redis
# Other utils:
from utils_v2.string import json
# For debugging:
from icecream import IceCreamDebugger
# To make a decorator:
from functools import wraps
# For hashing and shortening the hash:
import hashlib
import base64
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** WRAPPERS ***
# ***** ****
# *****************************************************************************************************************
def cache_it(cache = None, expiry = 120):
"""
This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
cache instead of going through the whole function again.
:param cache: The instance of 'AsyncRedisCache'.
:param expiry: The time in seconds after which the cached data must be cleared.
:return: The decorator that automatically caches your data.
"""
def decorator(func):
@wraps(func)
async def wrapper(*args, **kwargs):
# We first use the name of the function and the inputs given to it to generate a key for Redis with the
# simple hashing and shortening by way of base64 strings:
inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
sha256_hash = hashlib.sha256()
sha256_hash.update(inputs_given.encode("utf-8"))
hashed_key = sha256_hash.digest()
base64_key = base64.b64encode(hashed_key).decode("utf-8")
# Now we check if we have the value in cache:
try: response = await cache.get(base64_key, raise_exception = True)
# If the key doesn't exist, we pass through the function and store the results.
except:
response = await func(*args, **kwargs)
await cache.set(key = base64_key, value = response, expiry = expiry)
# Return the response from the wrapped function.
return response
return wrapper
return decorator
# ---------------------------------------------------------------------------------------------------------------------
def cache_class_methods(attr_name, expiry = 120):
"""
This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
cache instead of going through the whole function again.
:param attr_name: The name of the variable that has an instance of "AsyncRedisCache".
:param expiry: The time in seconds after which the cached data must be cleared.
:return: The decorator that automatically caches your data.
"""
def decorator(func):
@wraps(func)
async def wrapper(self, *args, **kwargs):
# Get the cache object first:
cache_obj = getattr(self, attr_name)
# We first use the name of the function and the inputs given to it to generate a key for Redis with the
# simple hashing and shortening by way of base64 strings:
inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
sha256_hash = hashlib.sha256()
sha256_hash.update(inputs_given.encode("utf-8"))
hashed_key = sha256_hash.digest()
base64_key = base64.b64encode(hashed_key).decode("utf-8")
# Now we check if we have the value in cache:
try: response = await cache_obj.get(base64_key, raise_exception = True)
# If the key doesn't exist, we pass through the function and store the results.
except:
response = await func(self, *args, **kwargs)
await cache_obj.set(key = base64_key, value = response, expiry = expiry)
# Return the response from the wrapped function.
return response
return wrapper
return decorator
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncRedisCache:
# Handle datatypes when setting values to cache:
__set_converters = {
"set": lambda x: list(x),
"tuple": lambda x: list(x),
"complex": lambda x: {"r": x.real, "i": x.imag}
}
# Handle datatypes when getting from cache:
__get_converters = {
"set": lambda x: set(x),
"tuple": lambda x: tuple(x),
"complex": lambda x: complex(x["r"], x["i"])
}
def __init__(
self,
connection_string,
ping_counter = 1_000,
debug = False,
debug_prefix = "R-Cache | "
):
"""
Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools,
strings, dicts, lists, sets, and tuples :)
:param connection_string: The connection URL for connecting to Redis.
:param ping_counter: The number of requests to Redis after which you want to ping to ensure connection.
:param debug: Whether, or not, you want to show debugging messages from the start.
:param debug_prefix: The prefix text to show with the debugging messages.
"""
# Note down the configuration:
self.__client = None
self.__ping_counter = ping_counter
self.__requests_since_last_ping = 0
self.__connection_string = connection_string
# For debugging:
self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self.__printer.disable()
def enable_debug(self):
self.__printer.enable()
def disable_debug(self):
self.__printer.disable()
@staticmethod
def make_key(*args, **kwargs):
"""
Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the
inputs stay the same, the output stays the same.
:param args: Any number of args that you would like to use to generate the key.
:param kwargs: Any number of kwargs that you would like to use to generate the key.
:return: A string that can be used as a key to store values on Redis.
"""
# Take everything into one plain text string:
plain_text = "".join(str(a) for a in args)
plain_text += json.to_string(kwargs, no_space = True)
# Hash the plain text value, and create a key from it:
sha256_hash = hashlib.sha256()
sha256_hash.update(plain_text.encode("utf-8"))
hashed_key = sha256_hash.digest()
base64_key = base64.b64encode(hashed_key).decode("utf-8")
# Done here:
return base64_key
def add_converter(
self,
type_name,
set_converter_func,
get_converter_func,
):
"""
To add custom data converters to handle new values.
RULE: Each of the converter functions must take in exactly on argument.
:param type_name: The name of the type of the object to handle. HINT: type(obj).__name__.
:param set_converter_func: A converter function that converts from the custom datatype to a datatype that Redis
can work with. Try converting to an bool, int, float, str, list or dict.
:param get_converter_func: The inverse of the set converter. This function will be used to convert from a
datatype that Redis can work with to the custom datatype.
:return: None.
"""
self.__set_converters[type_name] = lambda x: set_converter_func(x)
self.__get_converters[type_name] = lambda x: get_converter_func(x)
async def connect(self):
"""
To make an asynchronous connection request to the Redis server to establish a connection.
:return: True if connected, else False.
"""
try:
self.__client = redis.from_url(
self.__connection_string
)
return True
except Exception as exception:
self.__printer(exception)
return False
async def disconnect(self):
"""
Close the connection to the Redis server.
:return: True if successful, else False.
"""
if self.__client is not None:
try: await self.__client.close()
except Exception as exception:
self.__printer(exception)
return False
return True
return True
async def ensure_connection(self):
"""
To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected.
:return: True if connected, else False.
"""
# If we are not connected, we try to establish a connection:
if self.__client is None: return await self.connect()
# Else we check if we are connected. If not, we try to connect.
# But we check only once in a while. In the meantime, we assume that we are connected.
elif self.__requests_since_last_ping > self.__ping_counter:
try:
await self.__client.ping()
self.__requests_since_last_ping = 0
return True
except Exception as exception:
self.__printer(exception)
return await self.connect()
else:
self.__requests_since_last_ping += 1
return True
async def set(self, key, value, expiry: float = 120, raise_exception = False):
"""
Saves some value to the cache.
:param key: The key with which the data will be stored and retrieved.
:param value: The value to store.
:param expiry: The time in seconds after which the data will expire. Must be a positive number.
:param raise_exception: If you want to raise an exception if the process fails.
:return: True if cached, else False.
"""
await self.ensure_connection()
try:
# We handle conversion for unsupported Pythonic datatypes:
original_type = type(value).__name__
converter = self.__set_converters.get(original_type)
if converter is not None: value = converter(value)
# Here we actually try to store the data:
response = await self.__client.setex(
key,
int(expiry),
json.to_string({
"value": value,
"type": original_type
}, indent = 0, no_space = False)
)
return response
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return False
async def get(self, key, raise_exception = False, on_fail = None):
"""
Retrieve the cached value.
:param key: The key with which the data was saved.
:param raise_exception: If you want to raise an exception if the process fails.
:param on_fail: What to return if the process fails due to an exception.
:return: The retrieved data or null if not found.
"""
await self.ensure_connection()
try:
data = await self.__client.get(key)
data = data.decode("utf-8")
data = json.from_string(data)
converter = self.__get_converters.get(data["type"])
if converter is not None: return converter(data["value"])
return data["value"]
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return on_fail
async def delete(self, key, raise_exception = False):
"""
Prematurely delete the value from the cache before it expires.
:param key: The key with which the data was saved.
:param raise_exception: If you want to raise an exception if the process fails.
:return: True if deleted, else False.
"""
await self.ensure_connection()
try:
response = await self.__client.delete(key)
return True if response else False
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return False
async def count(self, key, value: int = 1, expiry: float = None, raise_exception = False):
"""
To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the
counter will be created and the value will be incremented from 0.
:param key: The name of the counter.
:param value: The amount to increment the value by. Send negative values to count backwards.
:param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created.
This value has to be an integer. If a float is passed, the value will be rounded off.
:param raise_exception: If you want to raise an exception if the process fails.
:return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed.
"""
await self.ensure_connection()
try:
already_existed = await self.__client.exists(key)
new_value = await self.__client.incrby(key, value)
if expiry and not already_existed: await self.__client.expire(key, int(expiry))
return new_value
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return None
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
def set_complex(x):
return {"r": x.real, "i": x.imag}
def get_complex(x):
return complex(x["r"], x["i"])
async def main():
my_cache = AsyncRedisCache(
connection_string = r"redis://:dc4da94197c843ab6a730113c2b801d9@redis.ditscentre.in/0",
ping_counter = 100,
debug = False
)
my_cache.add_converter(
type_name = type(2j).__name__,
set_converter_func = set_complex,
get_converter_func = get_complex
)
value = await my_cache.get(key = "6b61afd0-b066-4611-9791-411a30d34624")
print("GET:", value)
print("TYP:", type(value), end = "\n\n")
# success = await my_cache.set(
# key = "name",
# value = {"first": "John", "last": "Doe"},
# expiry = 10
# )
# print("SET:", success, end = "\n\n")
#
# value = await my_cache.get(key = "name")
# print("GET:", value)
# print("TYP:", type(value), end = "\n\n")
#
# success = await my_cache.delete(key = "name")
# print("DEL:", success, end = "\n\n")
#
# value = await my_cache.get(key = "cnt")
# print("GET:", value)
# print("TYP:", type(value), end = "\n\n")
#
# await my_cache.delete(key = "cnt")
# for _ in range(50):
# await asyncio.sleep(1.0)
# counter = await my_cache.count(key = "cnt", value = 1, expiry = 10)
# print("COUNTER:", counter)
asyncio.run(main())
utils_v2/cache/async_redis_cache_v2.py
Vendored
-568
View File
@@ -1,568 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Saturday, 26th Oct., 2024
OBJECTIVE:
To provide an easy way to cache data for fast access. This version has the change that it can handle custom
serializers by way of dependency injection.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To use redis:
import redis.asyncio as redis
# Other utils:
from utils_v2.string import json
from utils_v2.serialization.pickle_serializer import PickleSerializer
# For debugging:
from icecream import IceCreamDebugger
# To make a decorator:
from functools import wraps
# For hashing and shortening the hash:
import hashlib
import base64
# To work with datatypes:
from typing import List, Any
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** WRAPPERS ***
# ***** ****
# *****************************************************************************************************************
def cache_it(cache = None, expiry = 120):
"""
This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
cache instead of going through the whole function again.
:param cache: The instance of 'AsyncRedisCache'.
:param expiry: The time in seconds after which the cached data must be cleared.
:return: The decorator that automatically caches your data.
"""
def decorator(func):
@wraps(func)
async def wrapper(*args, **kwargs):
# We first use the name of the function and the inputs given to it to generate a key for Redis with the
# simple hashing and shortening by way of base64 strings:
inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
sha256_hash = hashlib.sha256()
sha256_hash.update(inputs_given.encode("utf-8"))
hashed_key = sha256_hash.digest()
base64_key = base64.b64encode(hashed_key).decode("utf-8")
# Now we check if we have the value in cache:
try: response = await cache.get(base64_key, raise_exception = True)
# If the key doesn't exist, we pass through the function and store the results.
except:
response = await func(*args, **kwargs)
await cache.set(key = base64_key, value = response, expiry = expiry)
# Return the response from the wrapped function.
return response
return wrapper
return decorator
# ---------------------------------------------------------------------------------------------------------------------
def cache_class_methods(attr_name, expiry = 120):
"""
This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there.
If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the
cache instead of going through the whole function again.
:param attr_name: The name of the variable that has an instance of "AsyncRedisCache".
:param expiry: The time in seconds after which the cached data must be cleared.
:return: The decorator that automatically caches your data.
"""
def decorator(func):
@wraps(func)
async def wrapper(self, *args, **kwargs):
# Get the cache object first:
cache_obj = getattr(self, attr_name)
# We first use the name of the function and the inputs given to it to generate a key for Redis with the
# simple hashing and shortening by way of base64 strings:
inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs)
sha256_hash = hashlib.sha256()
sha256_hash.update(inputs_given.encode("utf-8"))
hashed_key = sha256_hash.digest()
base64_key = base64.b64encode(hashed_key).decode("utf-8")
# Now we check if we have the value in cache:
try: response = await cache_obj.get(base64_key, raise_exception = True)
# If the key doesn't exist, we pass through the function and store the results.
except:
response = await func(self, *args, **kwargs)
await cache_obj.set(key = base64_key, value = response, expiry = expiry)
# Return the response from the wrapped function.
return response
return wrapper
return decorator
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncRedisCache:
def __init__(
self,
connection_string,
serializer = None,
ping_counter = 1_000,
debug = False,
debug_prefix = "R-Cache | "
):
"""
Implements a simple cache in Redis which holds and returns all native datatypes like ints, floats, bools,
strings, dicts, lists, sets, and tuples :)
:param connection_string: The connection URL for connecting to Redis.
:param serializer: The serializer to use.
:param ping_counter: The number of requests to Redis after which you want to ping to ensure connection.
:param debug: Whether, or not, you want to show debugging messages from the start.
:param debug_prefix: The prefix text to show with the debugging messages.
"""
# Note down the configuration:
self.__client = None
self.__serializer = serializer or PickleSerializer()
self.__ping_counter = ping_counter
self.__requests_since_last_ping = 0
self.__connection_string = connection_string
# For debugging:
self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self.__printer.disable()
def enable_debug(self):
self.__printer.enable()
def disable_debug(self):
self.__printer.disable()
@staticmethod
def make_key(*args, **kwargs) -> str:
"""
Generates a key by hashing the args and kwargs sent to it. Can be useful to generate a predictable key. If the
inputs stay the same, the output stays the same.
:param args: Any number of args that you would like to use to generate the key.
:param kwargs: Any number of kwargs that you would like to use to generate the key.
:return: A string that can be used as a key to store values on Redis.
"""
# Take everything into one plain text string:
plain_text = "".join(str(a) for a in args)
plain_text += json.to_string(kwargs, no_space = True)
# Hash the plain text value, and create a key from it:
sha256_hash = hashlib.sha256()
sha256_hash.update(plain_text.encode("utf-8"))
hashed_key = sha256_hash.digest()
base64_key = base64.b64encode(hashed_key).decode("utf-8")
# Done here:
return base64_key
async def connect(self) -> bool:
"""
To make an asynchronous connection request to the Redis server to establish a connection.
:return: True if connected, else False.
"""
try:
self.__client = redis.from_url(
self.__connection_string
)
return True
except Exception as exception:
self.__printer(exception)
return False
async def disconnect(self) -> bool:
"""
Close the connection to the Redis server.
:return: True if successful, else False.
"""
if self.__client is not None:
try: await self.__client.close()
except Exception as exception:
self.__printer(exception)
return False
return True
return True
async def ensure_connection(self) -> bool:
"""
To ensure that we are connected. We keep pinging the Redis server every once in a while even when connected.
:return: True if connected, else False.
"""
# If we are not connected, we try to establish a connection:
if self.__client is None: return await self.connect()
# Else we check if we are connected. If not, we try to connect.
# But we check only once in a while. In the meantime, we assume that we are connected.
elif self.__requests_since_last_ping > self.__ping_counter:
try:
await self.__client.ping()
self.__requests_since_last_ping = 0
return True
except Exception as exception:
self.__printer(exception)
return await self.connect()
else:
self.__requests_since_last_ping += 1
return True
async def list_keys(
self,
match: str = "*",
raise_exception: bool = False
) -> List[str] | None:
"""
To get a list of all the keys that match the pattern.
:param match: To match a glob-style pattern. THIS IS NOT FULL-FLEDGED REGEX.
:param raise_exception: If you want to raise an exception if the process fails.
:return: The list of keys if successful, else None.
"""
# Standard connectivity check:
await self.ensure_connection()
# Start with an empty list:
keys_list = []
try:
# Start at the beginning,
# and break out of the loop if the pointer returns to zero:
pointer = 0
while True:
pointer, keys = await self.__client.scan(pointer, match = match)
keys_list = keys_list + [k.decode("utf-8") for k in keys]
if pointer == 0: break
# Done here:
return list(set(keys_list)) if keys_list else None
# If an exception occurs in th eprocess:
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return None
async def ttl(
self,
key: str | bytes,
raise_exception: bool = False
) -> int | float | None:
"""
To get the no. of seconds till the expiry of some key.
:param key: The key to check the expiry of.
:param raise_exception:
:return: -1 if the key is persistent (i.e., no expiry time set), -2 if the key does not exist, or the time left
in seconds if the key exists and is not persistent. If something goes wrong, you will get a null value.
"""
# Standard connectivity check:
await self.ensure_connection()
# Start by assuming failure:
ttl = None
try:
# Get the TTL of the key in ms,
# convert it to seconds and return the value:
ttl_ms = await self.__client.pttl(key)
if ttl_ms >= 0: ttl = ttl_ms / 1_000
return ttl
# If an exception occurs in th eprocess:
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return None
async def set(
self,
key: str | bytes,
value: Any,
expiry: float = None,
raise_exception: bool = False
) -> bool:
"""
Saves some value to the cache. If an expiry is specified, the data will be deleted after that many seconds.
:param key: The key with which the data will be stored and retrieved.
:param value: The value to store.
:param expiry: The time in seconds after which the data will expire. Must be a positive number.
:param raise_exception: If you want to raise an exception if the process fails.
:return: True if cached, else False.
"""
# Standard connectivity check:
await self.ensure_connection()
try:
# Here we actually try to set the data:
data = self.__serializer.serialize(value)
if expiry: response = await self.__client.setex(key, int(expiry), data)
else: response = await self.__client.set(key, data)
return response
# If an exception occurs in th eprocess:
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return False
async def get(
self,
key: str | bytes,
raise_exception: bool = False,
on_fail: Any = None
) -> Any:
"""
Retrieve the cached value.
:param key: The key with which the data was saved.
:param raise_exception: If you want to raise an exception if the process fails.
:param on_fail: What to return if the process fails due to an exception.
:return: The retrieved data or null if not found.
"""
# Standard connectivity check:
await self.ensure_connection()
try:
# Here we try to fetch the data:
data = await self.__client.get(key)
data = self.__serializer.deserialize(data)
return data
# If an exception occurs in th eprocess:
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return on_fail
async def delete(
self,
key: str | bytes,
raise_exception: bool = False
) -> bool:
"""
Prematurely delete the value from the cache before it expires.
:param key: The key with which the data was saved.
:param raise_exception: If you want to raise an exception if the process fails.
:return: True if deleted, else False.
"""
# Standard connectivity check:
await self.ensure_connection()
try:
# Try to manually delete the key before expiry:
response = await self.__client.delete(key)
return True if response else False
# If an exception occurs in th eprocess:
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return False
async def count(
self,
key: str | bytes,
value: int = 1,
expiry: float = None,
raise_exception: bool = False
) -> int | None:
"""
To use simple counters. If the counter (identified by the 'key') exists, it will be incremented, else the
counter will be created and the value will be incremented from 0.
:param key: The name of the counter.
:param value: The amount to increment the value by. Send negative values to count backwards.
:param expiry: The time (in seconds) in which the counter expires. Starts from the time the counter is created.
This value has to be an integer. If a float is passed, the value will be rounded off.
:param raise_exception: If you want to raise an exception if the process fails.
:return: The latest value of the counter. Will be null if something went wrong and the exception was suppressed.
"""
# Standard connectivity check:
await self.ensure_connection()
try:
# check if the key already exists,
# regardless of that, increment the counter:
already_existed = await self.__client.exists(key)
new_value = await self.__client.incrby(key, value)
# If the key didn't already exist, specify the expiry:
if expiry and not already_existed: await self.__client.expire(key, int(expiry))
# Done here:
return new_value
# If an exception occurs in th eprocess:
except Exception as exception:
self.__printer(exception)
if raise_exception: raise
else: return None
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
from utils_v2.string import json
async def main():
my_cache = AsyncRedisCache(
connection_string = str(input("Paste your conn. str.: ")),
ping_counter = 100,
debug = True
)
print("Connecting.")
connected = await my_cache.connect()
print("Success:", connected)
# Proceed only if we connected to the database successfully:
if connected:
existing_keys = await my_cache.list_keys()
print(f"KEYS ({len(existing_keys)}):", json.to_string(existing_keys, default = str))
key = str(input("Paste a key to check its TTL: "))
key_ttl = await my_cache.ttl(key = key)
print(f"KEY '{key}' has {key_ttl} seconds of TTL.")
value = await my_cache.get(key = str(input("Paste a key to get: ")))
print("GET:", json.to_string(value, default = str) if isinstance(value, (dict, list)) else value)
print("TYP:", type(value), end = "\n\n")
success = await my_cache.set(
key = "name",
value = {"first": "John", "last": "Doe"},
expiry = 10
)
print("SET:", success, end = "\n\n")
value = await my_cache.get(key = "name")
print("GET:", value)
print("TYP:", type(value), end = "\n\n")
success = await my_cache.delete(key = "name")
print("DEL:", success, end = "\n\n")
print("Starting count test.")
await my_cache.delete(key = "cnt")
for _ in range(5):
print("Step:", _)
await asyncio.sleep(1.0)
counter = await my_cache.count(key = "cnt", value = 1, expiry = 10)
print("COUNTER:", counter)
asyncio.run(main())
utils_v2/database/__init__.py
View File
utils_v2/database/__pycache__/__init__.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/database/__pycache__/async_mongo_v2.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/database/__pycache__/async_mysql_v2.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/database/async_firebase.py
-504
View File
@@ -1,504 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
Bhushan Thakkar
DATE:
Friday, 19th April, 2024
OBJECTIVE:
To provide an easy interface to work with Firebase.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
# ---
import sys
sys.path.append(".")
sys.path.append("..")
# For async operations:
# ---
import asyncio
# For system-level activities:
# ---
import os
# My async utils:
# ---
import async_json_utils
# Firebase:
# ---
import firebase_admin
import firebase_admin.firestore_async
import firebase_admin.auth
import firebase_admin.db
# For debugging and logging:
# ---
from icecream import IceCreamDebugger
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class AsyncFirebase:
def __init__(
self,
credentials_json_path,
app_name,
max_connections = 5,
debug = True
):
# Debugging print:
# ---
self.__printer = IceCreamDebugger(prefix = f"FBase ({app_name[:8]}) | ", includeContext = True)
if not debug: self.__printer.disable()
alert = f"Firebase session starting with max. {max_connections} connections."
self.__printer(alert)
# Initialize the instance:
# ---
self.__semaphore = asyncio.Semaphore(max_connections)
credentials = firebase_admin.credentials.Certificate(credentials_json_path)
self.__firebase_app = firebase_admin.initialize_app(credentials, name = app_name)
self.__firestore = firebase_admin.firestore_async.client(self.__firebase_app)
def __del__(self):
firebase_admin.delete_app(self.__firebase_app)
self.__firestore.close()
alert = "Firebase session ended."
self.__printer(alert)
def __user_to_json(self, firebase_user):
user_json = {
"uid": firebase_user.uid,
"email": firebase_user.email,
"emailVerified": firebase_user.email_verified,
"displayName": firebase_user.display_name,
"phoneNo": firebase_user.phone_number,
"photoUrl": firebase_user.photo_url,
"customClaims": firebase_user.custom_claims,
"disabled": firebase_user.disabled,
"providerId": firebase_user.provider_id,
"providerData": firebase_user.provider_data,
"tenantId": firebase_user.tenant_id
}
return user_json
async def create_custom_token(self, uid):
async with self.__semaphore:
try:
custom_token = firebase_admin.auth.create_custom_token(
uid = uid,
app = self.__firebase_app
).decode("utf-8")
return custom_token
except Exception as exception:
self.__printer(exception)
return None
async def authenticate_token(self, token):
"""
To authenticate the given session token.
:param token: The session token generated by Firebase on a successful sign-in.
:return: Either the retrieved user information or a blank dictionary.
"""
try:
user_info = firebase_admin.auth.verify_id_token(token, app = self.__firebase_app)
return user_info
except Exception as exception:
self.__printer(exception)
return {}
async def create_user(
self,
uid,
display_name = None,
password = None,
email = None,
phone_number = None
):
"""
Create a new user.
:param uid: The id to identify the user by.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
firebase_admin.auth.create_user(
uid = uid,
app = self.__firebase_app
)
return True
except firebase_admin.auth.UidAlreadyExistsError as excp:
return True
except Exception as exception:
self.__printer(exception)
return False
async def delete_user(self, uid):
async with self.__semaphore:
try:
firebase_admin.auth.delete_user(
uid = uid,
app = self.__firebase_app
)
return True
except firebase_admin.auth.UserNotFoundError as excp:
return True
except Exception as exception:
self.__printer(exception)
return False
async def get_user(self, uid):
async with self.__semaphore:
try:
user = firebase_admin.auth.get_user(
uid = uid,
app = self.__firebase_app
)
return self.__user_to_json(user)
except Exception as exception:
self.__printer(exception)
return None
async def list_users(self, users_per_page, page_token = None):
"""
To get a list of users. Firebase has a limit of 1000 per call of this API. So we use page-tokens to fetch next
pages of users.
:param users_per_page: How many users you want to list in this API call (Max. 1,000).
:param page_token: To be used in case of pagination.
:return: The list of users and the page token to be used for the next call. Will be None in case of failure.
"""
async with self.__semaphore:
try:
users = firebase_admin.auth.list_users(
page_token = page_token,
max_results = min(users_per_page, 1000),
app = self.__firebase_app
)
next_page_token = users.next_page_token if users.has_next_page else None
users = users.users
users_json = [self.__user_to_json(user) for user in users]
return users_json, next_page_token
except Exception as exception:
self.__printer(exception)
return None, None
async def create_document(self, collection_path, document_name, document_data = None):
"""
To create a new document with the specified data in an existing collection.
:param collection_path: The path of the collection (can be a sub-collection).
:param document_name: The name of the document you want to create.
:param document_data: The data that you want to populate in the document.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.collection(
collection_path
).add(
document_id = document_name,
document_data = document_data or {}
)
return True
except Exception as exception:
self.__printer(exception)
return False
async def delete_document(self, path):
"""
To delete a document.
:param path: The path of the document.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.document(path).delete()
return True
except Exception as excp:
print("FIRESTORE DOCUMENT DELETION EXCEPTION:", excp)
return False
async def set_document(self, path, data):
"""
To overwrite the data in a document.
:param path: The path of the document.
:param data: The data you want to update as a dictionary.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.document(path).set(data)
return True
except Exception as exception:
self.__printer(exception)
return False
async def update_document(self, path, data):
"""
To update the data in a document. Provide only the fields that you want to update.
:param path: The path of the document.
:param data: The data you want to update as a dictionary.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.document(path).update(data)
return True
except Exception as exception:
self.__printer(exception)
return False
async def get_document(self, path):
"""
To get the data in a document.
:param path: The path of the document.
:return: The dictionary of data as found in the path specified or None if the operation failed.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.document(path).get()
return snapshot.to_dict()
except Exception as exception:
self.__printer(exception)
return None
async def get_document_fields(self, path, fields):
"""
To get only specific fields (keys) in a document. Like how projections are used in Mongo.
:param path: The path of the document.
:param fields: The list of fields (keys) of the document that you want.
:return: The dictionary of data as found in the path specified.
"""
async with self.__semaphore:
try:
if type(fields) is not list: fields = [fields]
snapshot = await self.__firestore.document(path).get(fields)
return snapshot.to_dict()
except Exception as exception:
self.__printer(exception)
return None
async def create_collection(self, collection_name, document_name, document_data):
"""
Creates a new collection in the root of the database. Note that Firebase doesn't allow creating new empty
collections, so we must add one first document in it.
:param collection_name: The name of the collection you want to create.
:param document_name: The name of the first document you want to put in the collection.
:param document_data: The data that you want to put in the first document of the new collection.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.collection(
collection_name
).add(
document_id = document_name,
document_data = document_data
)
return True
except Exception as exception:
self.__printer(exception)
return False
async def create_sub_collection(
self,
document_path,
sub_collection_name,
sub_document_name = None,
sub_document_data = None
):
"""
Firebase allows you to create collections inside documents. This method is built for that. Note that Firebase
doesn't allow creating new empty collections, so we must add one first document in it.
:param document_path: The path of the document in which you want to create a new collection.
:param sub_collection_name: The name of the collection you want to create.
:param sub_document_name: The name of the first document you want to put in the collection.
:param sub_document_data: The data that you want to put in the first document of the new collection.
:return: True or False based on the success of the operation.
"""
async with self.__semaphore:
try:
snapshot = await self.__firestore.document(
document_path
).collection(
sub_collection_name
).add(
document_id = sub_document_name,
document_data = sub_document_data or {}
)
return True
except Exception as exception:
self.__printer(exception)
return False
async def get_collection(self, path):
"""
Get a whole collection's data.
:param path: The path of the collection.
:return: The collection.
"""
async with self.__semaphore:
try:
snapshots = await self.__firestore.collection(path).get()
return {snapshot.id: snapshot.to_dict() for snapshot in snapshots}
except Exception as exception:
self.__printer(exception)
return None
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import time
from shared import constants
user_data = {
"firebaseUid": None,
"roles": "ClientAdmin",
"sessionToken": "siddhesh_20240419"
}
async def test():
my_firebase = AsyncFirebase(
f"{constants.PROJECT_DIRECTORY}/utils/cred/firebase_certs.json",
"myFire",
max_connections = 5
)
print("\n\n---\n\n")
results = await my_firebase.get_document("testCollection/testDoc")
print(async_json_utils.to_json_string(results))
print("\n\n---\n\n")
results = await my_firebase.get_document_fields("testCollection/testDoc", ["sampleMap"])
print(async_json_utils.to_json_string(results))
# await my_firebase.create_sub_collection(
# "myDeepCollection/deepDocId",
# "subCollection3",
# "subDoc",
# {"sub_key": "sub_val"}
# )
# await my_firebase.create_collection(
# "rootCollection",
# "subDoc",
# {"sub_key": "sub_val"}
# )
# await my_firebase.delete_document("activeSessions/9d402f1502dfd55a4326fa7fc8e6cb7d")
# print(async_json_utils.to_json_string(await my_firebase.get_collection("myDeepCollection")))
start_time = time.time()
asyncio.run(test())
print(f"FINISHED IN {time.time() - start_time} SECONDS.")
utils_v2/database/async_mongo.py
-843
View File
@@ -1,843 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 28th May, 2024
OBJECTIVE:
To have one central place from where all async database connectivity happens.
REFERENCES:
01. https://motor.readthedocs.io/en/stable/
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For async behaviour:
import asyncio
# For datetime handling:
import pytz
import datetime
# MongoDB:
from motor.motor_asyncio import AsyncIOMotorClient
from bson.objectid import ObjectId
from bson.json_util import dumps, loads
# For debugging:
from icecream import IceCreamDebugger
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** EXCEPTIONS ***
# ***** ****
# *****************************************************************************************************************
class MongoFindException(Exception):
def __init__(self, hint = None, origin = None):
self.__hint = hint
self.__origin = origin
def __str__(self):
message = "mongo find operation failed"
if self.__hint is not None: message = f"{message} ({self.__hint})"
if self.__origin is not None: message = f"{self.__origin} --> {message}"
return message
# ---------------------------------------------------------------------------------------------------------------------
class MongoInsertException(Exception):
def __init__(self, hint = None, origin = None):
self.__hint = hint
self.__origin = origin
def __str__(self):
message = "mongo insert operation failed"
if self.__hint is not None: message = f"{message} ({self.__hint})"
if self.__origin is not None: message = f"{self.__origin} --> {message}"
return message
# ---------------------------------------------------------------------------------------------------------------------
class MongoUpdateException(Exception):
def __init__(self, hint = None, origin = None):
self.__hint = hint
self.__origin = origin
def __str__(self):
message = "mongo update operation failed"
if self.__hint is not None: message = f"{message} ({self.__hint})"
if self.__origin is not None: message = f"{self.__origin} --> {message}"
return message
# ---------------------------------------------------------------------------------------------------------------------
class MongoReplaceException(Exception):
def __init__(self, hint = None, origin = None):
self.__hint = hint
self.__origin = origin
def __str__(self):
message = "mongo replace operation failed"
if self.__hint is not None: message = f"{message} ({self.__hint})"
if self.__origin is not None: message = f"{self.__origin} --> {message}"
return message
# ---------------------------------------------------------------------------------------------------------------------
class MongoDeleteException(Exception):
def __init__(self, hint = None, origin = None):
self.__hint = hint
self.__origin = origin
def __str__(self):
message = "mongo delete operation failed"
if self.__hint is not None: message = f"{message} ({self.__hint})"
if self.__origin is not None: message = f"{self.__origin} --> {message}"
return message
# ---------------------------------------------------------------------------------------------------------------------
class MongoException(Exception):
def __init__(self, hint = None, origin = None):
self.__hint = hint
self.__origin = origin
def __str__(self):
message = "mongo operation failed"
if self.__hint is not None: message = f"{message} ({self.__hint})"
if self.__origin is not None: message = f"{self.__origin} --> {message}"
return message
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncMongo:
__db = None
__db_name = None
__client = None
def __init__(
self,
database_name = "myDb",
max_connections = 5,
debug = True,
debug_only_errors = True,
host = "localhost",
port = 27017,
connection_string = None
):
# Basic config:
self.__max_connections = max_connections
self.__db_name = database_name
self.__host = host,
self.__port = port
self.__connection_string = connection_string
# For debugging:
self.__debug_only_errors = debug_only_errors
self.__printer = IceCreamDebugger(prefix = f"Mongo ({self.__db_name}) | ", includeContext = True)
if not debug: self.__printer.disable()
# Rate/access control:
self.__exclusive_semaphore = asyncio.Semaphore(1)
def enable_debug(self):
self.__printer.enable()
def disable_debug(self):
self.__printer.disable()
@staticmethod
def generate_id():
"""
Just generates an '_id' in MongoDB style.
:return: The '_id' in MongoDB style.
"""
return str(ObjectId())
async def connect(self):
try:
if self.__connection_string is None:
self.__client = AsyncIOMotorClient(
self.__host,
self.__port,
maxPoolSize = self.__max_connections,
minPoolSize = self.__max_connections
)
else:
self.__client = AsyncIOMotorClient(
self.__connection_string,
maxPoolSize = self.__max_connections,
minPoolSize = self.__max_connections
)
self.__db = self.__client.get_database(self.__db_name)
if not self.__debug_only_errors:
server_info = await self.__client.server_info()
self.__printer(server_info)
except Exception as exception: self.__printer(exception)
async def ensure_connection(self):
if self.__db is None:
async with self.__exclusive_semaphore:
await self.connect()
@property
async def client(self):
await self.ensure_connection()
return self.__client
@staticmethod
def dict_to_dot_notation(input_dict, parent_key = "", separator = "."):
items = []
for k, v in input_dict.items():
new_key = f"{parent_key}{separator}{k}" if parent_key else k
if isinstance(v, dict) and v:
items.extend(AsyncMongo.dict_to_dot_notation(v, new_key, separator = separator).items())
else:
items.append((new_key, v))
return dict(items)
@staticmethod
def normalize_date_time(document):
"""
MongoDB doesn't support timezones. A good strategy would be to convert everything to UTC format and store it.
This method does exactly that. Any datetime object is converted to UTC timezone. If the datetime object was
timezone naive, UTC timezone will be applied to it without changing the time value.
:param document: The document that you want to normalize the date-time in.
:return: The document with normalized datetime.
"""
if isinstance(document, datetime.datetime):
utc_tz = pytz.timezone("UTC")
if document.tzinfo is None: document = utc_tz.localize(document)
else: document = document.astimezone(utc_tz)
if type(document) is list:
document = [AsyncMongo.normalize_date_time(item) for item in document]
if type(document) is dict:
document = {
AsyncMongo.normalize_date_time(k): AsyncMongo.normalize_date_time(v)
for k, v in document.items()
}
return document
@staticmethod
def __from_json_string(json_data):
return loads(json_data)
@staticmethod
def __to_json_string(python_data, indent = 4, default = None):
return dumps(python_data, indent = indent, default = default)
async def list_indexes(
self,
collection,
session = None,
raise_exception = False
):
"""
Lists out the indexes of a collection.
:param collection: The collection whose indexes you want to list out,
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: The list of indexes or None if the action fails.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
indexes = None
# Try to list the indexes:
try:
responses = await self.__db[collection].list_indexes(session = session).to_list(None)
indexes = [{key: value for key, value in response.items()} for response in responses]
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not indexes: raise MongoException(hint = "list indexes")
return indexes
async def create_index(
self,
collection,
keys,
options = None,
session = None,
raise_exception = False
):
"""
Creates an index on a collection.
:param collection: The collection to create the index on.
:param keys: The keys (and sorting) to implement the index on.
:param options: Additional config.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: True or False based on the success of the execution.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
success = False
# Try to make the insertion:
try:
options = options or {}
keys = [(k, v) for k, v in keys.items()]
response = await self.__db[collection].create_index(keys, session = session, **options)
if response: success = True
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not success: raise MongoException(hint = "create index")
return success
async def insert_one(
self,
collection,
document,
session = None,
raise_exception = False
):
"""
Insert data into a collection.
:param collection: The collection you want to feed the data into.
:param document: The data to be stored.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: The id of the inserted data, or null if the action fails.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
inserted_id = None
# Try to make the insertion:
try:
response = await self.__db[collection].insert_one(document.copy(), session = session)
inserted_id = response.inserted_id
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not inserted_id: raise MongoInsertException(hint = f"{collection}")
return inserted_id
async def insert_many(
self,
collection,
documents,
session = None,
raise_exception = False
):
"""
Insert a lot of data into a collection.
:param collection: The collection you want to feed the data into.
:param documents: The data to be stored.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: The id of the inserted data, or null if the action fails.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
inserted_ids = []
# Try to make the insertion:
try:
response = await self.__db[collection].insert_many(documents, session = session)
inserted_ids = response.inserted_ids
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not inserted_ids: raise MongoInsertException(hint = f"{collection}")
return inserted_ids
async def update_one(
self,
collection,
filter_json = None,
update_json = None,
upsert = False,
session = None,
raise_exception = False
):
"""
Update one document.
:param collection: The collection you want to update.
:param filter_json: The selection criteria to locate the document to update.
:param update_json: The values you want to update.
:param upsert: If you want to insert if the document doesn't already exist.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: True or False based on the success of the operation.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
success = False
# Try to make the insertion:
try:
response = await self.__db[collection].update_one(
filter_json,
update_json,
upsert = upsert,
session = session
)
success = False if response.modified_count == 0 else True
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not success: raise MongoUpdateException(hint = f"{collection}")
return success
async def update_many(
self,
collection,
filter_json = None,
update_json = None,
upsert = False,
session = None,
raise_exception = False
):
"""
Update many documents.
:param collection: The collection you want to update.
:param filter_json: The selection criteria to locate the document to update.
:param update_json: The values you want to update.
:param upsert: If you want to insert if the document doesn't already exist.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: True or False based on the success of the operation.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
update_count = 0
# Try to make the insertion:
try:
response = await self.__db[collection].update_many(
filter_json,
update_json,
upsert = upsert,
session = session
)
update_count = response.modified_count
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not update_count: raise MongoUpdateException(hint = f"{collection}")
return update_count
async def count(
self,
collection,
filter_json = None
):
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
count = 0
# try to query the data:
try:
if filter_json is None: filter_json = {}
count = await self.__db[collection].count_documents(filter_json)
except Exception as exception: self.__printer(exception)
# Done here:
return count
async def bulk_write(
self,
collection,
requests,
session = None,
raise_exception = False
):
"""
To perform various individual operations in one go. You will have to import individual actions like "UpdateOne"
and "InsertMany" from PyMongo and pass them as an array of requests (operations) to this method.
:param collection: The collection you want to run the requests on.
:param requests: The array of requests (operations) to be performed.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return:
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
count = 0
# try to query the data:
try:
response = await self.__db[collection].bulk_write(requests, session = session)
count = response.modified_count + response.inserted_count + response.upserted_count + response.deleted_count
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not count: raise MongoException(hint = "bulk write")
return count
async def find_many(
self,
collection,
filter_json,
projections = None,
skip = 0,
limit = None,
sort = None,
session = None,
as_json_string = False,
indent = 4,
raise_exception = False
):
"""
Finds one or more records that match the given conditions.
:param collection: The name of the collection to perform the search in.
:param filter_json: The filter criteria.
:param projections: What parts of the matching data you want to fetch.
:param skip: The no. of records to skip before picking next ones. Needed for pagination.
:param limit: The max. no. of records you want to fetch.
:param sort: The sorting rules to apply.
:param session: The session if you need to do this in a transaction.
:param as_json_string: Whether you want it as a JSON string or a Python dict/list.
:param indent: The indentation to use if you want it as a JSON string.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: The array of matching records or null if there was an exception.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
results = None
# Assume defaults:
if sort is None: sort = {"_id": -1}
if limit is None: limit = 10
# try to query the data:
try:
results = await self.__db[collection].find(
filter_json,
projections,
session = session
).sort(sort).skip(skip).limit(limit).to_list(None)
if as_json_string: results = self.__to_json_string(results, indent = indent, default = str)
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not results: raise MongoFindException(hint = f"{collection}")
return results
async def find_one(
self,
collection,
filter_json,
projections = None,
session = None,
as_json_string = False,
indent = 4,
raise_exception = False
):
"""
Finds one record that matches the given conditions.
:param collection: The name of the collection to perform the search in.
:param filter_json: The filter criteria.
:param projections: What parts of the matching data you want to fetch.
:param session: The session if you need to do this in a transaction.
:param as_json_string: Whether you want it as a JSON string or a Python dict/list.
:param indent: The indentation to use if you want it as a JSON string.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: The array of matching records or null if there was an exception.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
results = None
# try to query the data:
try:
results = await self.__db[collection].find_one(filter_json, projections, session = session)
if as_json_string: results = self.__to_json_string(results, indent = indent, default = str)
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not results: raise MongoFindException(hint = f"{collection}")
return results
async def replace_one(
self,
collection,
filter_json,
replacement_json,
upsert = False,
session = None,
raise_exception = False
):
"""
To delete one document from a collection.
:param collection: The collection from which you want to delete many records.
:param filter_json: The filter criteria.
:param replacement_json: The data to put in place of the existing document.
:param upsert: If you want to insert if the document doesn't already exist.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: True or False based on the success of the operation.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
success = False
# try to query the data:
try:
result = await self.__db[collection].replace_one(
filter_json,
replacement_json,
upsert = upsert,
session = session
)
if result.modified_count or result.upserted_id: success = True
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not success: raise MongoReplaceException(hint = f"{collection}")
return success
async def delete_one(
self,
collection,
filter_json,
session = None,
raise_exception = False
):
"""
To delete one document from a collection.
:param collection: The collection from which you want to delete many records.
:param filter_json: The filter criteria.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: True or False based on the success of the operation.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
count = 0
# try to query the data:
try:
result = await self.__db[collection].delete_one(filter_json, session = session)
count = result.deleted_count
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}")
return count
async def delete_many(
self,
collection,
filter_json,
session = None,
raise_exception = False
):
"""
To delete many documents from a collection.
WARNING: sending {} in the filter would mean deleting ALL the documents.
:param collection: The collection from which you want to delete many records.
:param filter_json: The filter criteria.
:param session: The session if you need to do this in a transaction.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: True or False based on the success of the operation.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
count = 0
# try to query the data:
try:
result = await self.__db[collection].delete_many(filter_json, session = session)
count = result.deleted_count
except Exception as exception:
self.__printer(exception)
# Check results and return:
if raise_exception and not count: raise MongoDeleteException(hint = f"{collection}")
return count
async def aggregate(
self,
collection,
pipeline,
limit = None,
raise_exception = False
):
"""
Perform an advance query on the data.
:param collection: The collection to perform the query on.
:param pipeline: The pipeline of actions to take. Must be a list.
:param limit: The max. no. of records to retrieve.
:param raise_exception: Whether, or not, you want to raise an exception when something fails.
:return: The array of matching records or null if there was an exception.
"""
# Ensure you are connected:
await self.ensure_connection()
# Assume failure:
results = None
# try to perform the aggregation action:
try: results = await self.__db[collection].aggregate(pipeline).to_list(limit)
except Exception as exception: self.__printer(exception)
# Check results and return:
if raise_exception and not results: raise MongoException(hint = "aggregation")
return results
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/database/async_mongo_storage.py
-358
View File
@@ -1,358 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 28th May, 2024
OBJECTIVE:
To have one central place from where all async database connectivity happens.
REFERENCES:
01. https://motor.readthedocs.io/en/stable/
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
# ---
import sys
sys.path.append(".")
sys.path.append("..")
# For system-level activity:
import io
# For async behaviour:
import asyncio
# MongoDB for File Storage:
from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
from bson.objectid import ObjectId
# For debugging:
from icecream import IceCreamDebugger
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncMongoStorage:
__db = None
__db_name = None
__client = None
__store = None
def __init__(
self,
connection_string = None,
max_connections = 5,
host_name = "localhost",
port = 27017,
database_name = "fileStore",
debug = True,
debug_prefix = "GridFS (M) | ",
debug_only_errors = True
):
# Database Initialization:
self.__host_name = host_name
self.__port = port
self.__db_name = database_name
self.__connection_string = connection_string
self.__max_connections = max_connections
# Debugging:
self.__debug_only_errors = debug_only_errors
self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self.__printer.disable()
@staticmethod
def generate_id():
"""
Just generates an '_id' in MongoDB style.
:return: The '_id' in MongoDB style.
"""
return str(ObjectId())
async def connect(self):
"""
Initialize the database connection.
:return: Nothing.
"""
if self.__connection_string is None:
self.__client = AsyncIOMotorClient(
self.__host_name,
self.__port,
maxPoolSize = self.__max_connections,
minPoolSize = self.__max_connections
)
else:
self.__client = AsyncIOMotorClient(
self.__connection_string,
maxPoolSize = self.__max_connections,
minPoolSize = self.__max_connections
)
self.__db = self.__client.get_database(self.__db_name)
self.__store = AsyncIOMotorGridFSBucket(self.__db)
@property
def fs(self):
"""
To access the features that have not been wrapped in this reportlab directly.
This could include things like streaming files chunk-by-chunk.
:return: The file-store instance.
"""
return self.__store
async def write_from_memory(self, file_name, file_data, metadata_json = None):
"""
Save a file (from RAM) to Mongo. Suitable for smaller files (a few MBs max.).
:param file_name: The name of the file.
:param file_data: The data of the file (held in RAM).
:param metadata_json: A JSON of metadata information that can later be used to search files (RECOMMENDED).
:return: The file's id as a string (if it gets saved) or None.
"""
if self.__store is None: await self.connect()
file_data.seek(0)
file_size = file_data.__sizeof__()
file_id = None
try: file_id = await self.__store.upload_from_stream(file_name, file_data, metadata = metadata_json)
except Exception as exception: self.__printer(exception, file_name, file_size, file_id)
if not self.__debug_only_errors: self.__printer(file_name, file_size, file_id)
return str(file_id)
async def read_to_memory(self, file_identifier, by_id = True):
"""
To retrieve a file (in RAM) based on the provided identifier.
Suitable for smaller files (a few MBs max.).
:param file_identifier: Either the name or the "_id" of the file.
:param by_id: Set to True if you are fetching by the "_id" of the file.
:return: Either the file (in RAM) or None.
"""
if self.__store is None: await self.connect()
file_data = None
try:
if by_id: grid_out = await self.__store.open_download_stream(ObjectId(file_identifier))
else: grid_out = await self.__store.open_download_stream_by_name(file_identifier)
file_data = io.BytesIO(await grid_out.read())
file_data.seek(0)
except Exception as exception:
file_data = None
self.__printer(exception, file_identifier, by_id)
if not self.__debug_only_errors: self.__printer(file_identifier, by_id)
return file_data
async def delete_file_by_id(self, file_id):
"""
Tries to delete one file by the id.
:param file_id: The id of the file in the database.
:return: True or False based on the success of the operation.
"""
if self.__store is None: await self.connect()
deleted = False
try:
response = await self.__store.delete(file_id = ObjectId(file_id))
deleted = True
except Exception as exception:
self.__printer(exception, file_id, deleted)
return deleted
def __format_metadata_json(self, metadata_json):
"""
NOTE: ONLY USE WHEN SEARCHING FILES BY METADATA.
MongoDB expects dot-notation while searching for files by the metadata. We are making a function to search
files assuming that the conditions are to be applied to the metadata itself. So this function add the
dot-notation to the right places to conduct a successful search.
:param metadata_json: The JSON to format.
:return: The formatted JSON that has the right dot-notation.
"""
formatted_metadata_json = {}
for key, value in metadata_json.items():
if not key.startswith("$"): key = f"metadata.{key}"
else:
if type(value) is dict: value = self.__format_metadata_json(value)
if type(value) is list: value = [self.__format_metadata_json(item) for item in value]
formatted_metadata_json[key] = value
return formatted_metadata_json
async def find_file_by_metadata(self, metadata_json, limit = None, skip = None, sort = None):
"""
This method only lists the files that match the criteria mentioned in the metadata JSON.
:param metadata_json: The JSON that describes what you want to find.
:param limit: Max. no. of records to retrieve.
:param skip: No. of starting results to skip. Useful for pagination.
:param sort: The sorting conditions to follow.
:return: A list of (JSONs of) files that match the conditions. The list can be empty.
"""
if self.__store is None: await self.connect()
files_list = []
try:
limit = limit or 10
skip = skip or 0
sort = {"_id": -1} if not isinstance(sort, dict) else sort
formatted_metadata_json = self.__format_metadata_json(metadata_json)
return await self.__store.find(
formatted_metadata_json
).sort(sort).skip(skip).limit(limit).to_list(None)
except Exception as exception: self.__printer(exception, metadata_json, len(files_list))
if not self.__debug_only_errors: self.__printer(metadata_json, len(files_list))
return files_list
async def find_file_by_id(self, file_id):
"""
This method allows you to get the file's info from the id of the file.
:param file_id: The id that was assigned by Mongo during upload.
:return: The file's info or None if the file doesn't exist.
"""
if self.__store is None: await self.connect()
file_info = None
try:
formatted_metadata_json = {"_id": ObjectId(file_id)}
file_info = (await self.__store.find(formatted_metadata_json).to_list(1))[0]
except Exception as exception: self.__printer(exception, file_id, file_info)
return file_info
async def get_file_name(self, file_id):
"""
Returns the file name if the id of the file is known.
:param file_id: The id of the file as assigned by MongoDB when the file was stored.
:return: The file's name (if it exists), or None.
"""
# Ensure that we are connected:
if self.__store is None: await self.connect()
# Ensure that the input given is of 'ObjectId' type:
if type(file_id) is not ObjectId: file_id = ObjectId(str(file_id))
# Fetch and return the file name:
files_list = await self.__store.find(
{"_id": file_id},
{"filename": True}
).sort({"_id": -1}).limit(1).to_list(None)
try: file_name = files_list[0]["filename"]
except: file_name = None
return file_name
async def aggregate(
self,
collection,
pipeline,
limit = None
):
"""
Perform an advance query on the data.
:param collection: The collection to perform the query on.
:param pipeline: The pipeline of actions to take. Must be a list.
:param limit: The max. no. of records to retrieve.
:return: The array of matching records or null if there was an exception.
"""
if self.__store is None: await self.connect()
results = None
try: results = await self.__db[collection].aggregate(pipeline).to_list(limit)
except Exception as exception: self.__printer(exception)
return results
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/database/async_mongo_v2.py
-1749
View File
File diff suppressed because it is too large Load Diff
utils_v2/database/async_mysql_v2.py
-451
View File
@@ -1,451 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Friday, 30th Aug., 2024
OBJECTIVE:
To be able to access SQL-based databases from python in a simple way.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# MySQL Database:
import aiomysql
import decimal
# For data-crunching:
import pandas as pd
# For time-keeping:
import time
# OS-level operations:
import os
# My utils:
from utils_v2.string import json
# For async activities:
import asyncio
# For debugging:
from icecream import IceCreamDebugger
import traceback
# To work with datatypes:
from typing import List
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncMySQL:
def __init__(
self,
pool_size,
*args,
**kwargs
):
"""
A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as
JSON-like structures (list or dict). The format for the results was very specific to our use case for serving
Bicree's requirement. This may not serve your requirement at all.
:param pool_size: The number of connections to maintain n a pool.
:param args: Any arguments to pass. Not used.
:param kwargs: Pass the connection configuration from here.
"""
# Set up the variables:
self.__args = args
self.__kwargs = kwargs
self.__min_pool_size = 10
self.__max_pool_size = max(pool_size, self.__min_pool_size)
self.__pool: aiomysql = None
# For establishing connection:
self.__exclusive_semaphore = asyncio.Semaphore(1)
# Minor adjustments for backward compatibility:
self.__kwargs["db"] = self.__kwargs.pop("database")
# Set up the debugging tools:
self.__printer = IceCreamDebugger(prefix = "MySQL | ", includeContext = True)
def __del__(self):
pass
async def connect(self) -> bool:
"""
Establish a connection and create a pool of connections to call from.
:return: True if connected, else False
"""
try:
# Ensure that only one connection attempt is being made at one time,
# and try to connect to the database:
async with self.__exclusive_semaphore:
if self.__pool is None:
self.__pool = await aiomysql.create_pool(
minsize = self.__min_pool_size,
maxsize = self.__max_pool_size,
loop = asyncio.get_event_loop(),
**self.__kwargs
)
# In case of any exception:
except Exception as exception:
self.__printer(exception)
self.__pool = None
# Done here:
return False if self.__pool is None else True
async def disconnect(self) -> bool:
try:
# Try to close the pool and wait for the process to complete:
self.__pool.close()
await self.__pool.wait_closed()
# In case of any exception:
except Exception as exception:
self.__printer(exception)
# Done here:
return False if self.__pool is None else True
async def ensure_connection(self):
"""
Tries to ensure that a connection is present.
Can be called before every function to make sure that our pool is established.
:return: None.
"""
if self.__pool is None: await self.connect()
@staticmethod
def __parse_row(row):
"""
Converts from the custom objects of 'aiomysql' to types that are supported by Python.
:param row: The row from the result.
:return: The parsed row which will have types that are closer to being native to Python..
"""
parsed_row = []
for item in row:
if isinstance(item, decimal.Decimal): parsed_row.append(float(item))
else: parsed_row.append(item)
return parsed_row
async def fetch_all(self, cursor):
# Make a variable to hold all the result sets.
# Needed for when the procedure responds with many "tables":
all_result_sets = []
# Iterate over all result sets,
# and process them one-by-one:
while True:
# Process the current result set:
this_result_set = []
result = await cursor.fetchall()
if not cursor.description: break
columns = [desc[0] for desc in cursor.description]
for row in result: this_result_set.append(dict(zip(columns, self.__parse_row(row))))
all_result_sets.append(this_result_set)
# Move to the next set,
# or break out of the loop if all done:
if not await cursor.nextset(): break
# Done here:
return all_result_sets
async def call_procedure(
self,
procedure_name: str,
procedure_args: tuple,
commit: bool = True
):
"""
To call stored procedures and retrieve all the responses.
:param procedure_name: The name of the stored procedure that must be called.
:param procedure_args: The args to be sent to the stored procedure.
:param commit: Whether, or not, you would like to commit the execution.
:return: The raw result set as received from the database.
"""
# Make sure we have a connection:
await self.ensure_connection()
# Make a variable to hold all the result sets.
# Needed for when the procedure responds with many "tables":
all_result_sets = []
# Call the procedure and get the results:
async with self.__pool.acquire() as connection:
async with connection.cursor() as cursor:
await cursor.callproc(procedure_name, procedure_args)
all_result_sets = await self.fetch_all(cursor)
if commit: await connection.commit()
# Done here:
return all_result_sets
async def call_procedure_and_get_json(
self,
procedure_name,
procedure_args,
commit: bool = True,
retry_count = 1,
backoff_seconds = 0.5,
backoff_multiplier = 1.1,
return_exception = False
):
"""
The method to call when you need to call a stored procedure and retrieve the response as a JSON-like object.
This is custom formatting based on the structure created by Mr. bhushan Thakkar in late April (2024).
:param procedure_name: The name of the stored procedure that must be called.
:param procedure_args: The args to be sent to the stored procedure.
:param commit: Whether, or not, you would like to commit the execution.
:param retry_count: The max. number of times to try in case one or more attempts fail.
:param backoff_seconds: The time to wait before making the next attempt if the retry count is more than 1.
:param backoff_multiplier: The factor that dictates how much to modify the time delay by when waiting to retry.
:param return_exception: Whether, or not, you would like to return the exception object if something goes wrong.
:return: The formatted response and the exception (if asked for).
"""
# Note down the start time:
start_ts = time.time()
# Try to get the data from the database:
results = []
exception = None
for _ in range(retry_count):
try: results = await self.call_procedure(
procedure_name = procedure_name,
procedure_args = procedure_args,
commit = commit
)
except Exception as exc: exception = exc
if exception is None: break
await asyncio.sleep(backoff_seconds)
backoff_seconds = backoff_seconds * backoff_multiplier
# If the results are blank:
if len(results) == 0:
formatted_results = {
"status": 0,
"message": "Please contact admin (NE)" if exception is None else "Please contact admin (E)",
"seconds": time.time() - start_ts,
"data": {}
}
if return_exception: return formatted_results, exception
else: return formatted_results
# Extract the very basic success or failure indicators:
formatted_results = {
"status": results[0][0]["status"],
"message": results[0][0].get("message", "ok"),
"seconds": 0.0,
"data": {}
}
# Handle the remaining keys of the zeroth result set:
for key, value in results[0][0].items():
if key not in formatted_results.keys():
formatted_results["data"][key] = value
# Format
for index in range(len(results)):
if index > 0: formatted_results["data"][f"rs{index-1}"] = results[index]
# Note down the time taken:
formatted_results["seconds"] = time.time() - start_ts
# Done here:
if return_exception: return formatted_results, exception
else: return formatted_results
async def execute_one(
self,
query: str,
commit: bool = True,
return_exception: bool = False
):
"""
Runs one command / query in SQL.
:param query: The query / command to run.
:param commit: Whether, or not, you would like to commit the execution.
:param return_exception: Whether, or not, you would like to return the exception from this function.
:return: Either just the result or the result and the exception.
"""
# Make sure we have a connection:
await self.ensure_connection()
# Start by assuming failure:
rows_affected = None
results = None
excp = None
try:
# Get a connection and execute the command:
async with self.__pool.acquire() as connection:
async with connection.cursor() as cursor:
rows_affected = await cursor.execute(query)
results = await self.fetch_all(cursor)
if commit: await connection.commit()
# SQL-specific errors:
except aiomysql.MySQLError as exception:
self.__printer("SQL Exception", exception)
excp = exception
# Other errors:
except Exception as exception:
self.__printer("Other Exception", exception)
excp = exception
# Done here:
if return_exception: return rows_affected, results, excp
else: return rows_affected, results
async def execute_many(
self,
query: str,
data: List[tuple | List],
commit: bool = True,
return_exception: bool = False
):
"""
Runs many commands / queries in SQL.
Consider the following example:
QUERY: "INSERT INTO pincodeMaster (pincode, city, state) VALUES (%s, %s, %s);"
DATA: [
('110001', 'New Delhi', 'Delhi'),
('500001', 'Hyderabad', 'Telangana'),
('600001', 'Chennai', 'Tamil Nadu')
]
:param query: The query / command to run.
:param data: The data to substitute into the query string.
:param commit: Whether, or not, you would like to commit the execution.
:param return_exception: Whether, or not, you would like to return the exception from this function.
:return: Either just the result or the result and the exception.
"""
# Make sure we have a connection:
await self.ensure_connection()
# Start by assuming failure:
rows_affected = None
results = None
excp = None
try:
# Get a connection and execute the command:
async with self.__pool.acquire() as connection:
async with connection.cursor() as cursor:
rows_affected = await cursor.executemany(query, data)
results = await self.fetch_all(cursor)
if commit: await connection.commit()
# SQL-specific errors:
except aiomysql.MySQLError as exception:
self.__printer("SQL Exception", exception)
excp = exception
# Other errors:
except Exception as exception:
self.__printer("Other Exception", exception)
excp = exception
# Done here:
if return_exception: return rows_affected, results, excp
else: return rows_affected, results
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/dataframe/__init__.py
View File
utils_v2/dataframe/splitter.py
-118
View File
@@ -1,118 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 22nd Oct., 2024
OBJECTIVE:
To provide an easy way to split dataframes into chunks and divide the workload nto more manageable batches.
REFERENCES:
N/A
DOWNLOADS:
N/A
USAGE EXAMPLE:
for sub_df in DataFrameSplitter(df, chunk_size = 50):
# Do your task here:
pass
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class DataFrameSplitter:
def __init__(self, df, chunk_size):
"""
Use this to process your dataframe in batches. Useful for when you need to send out alerts at intervals or need
to maintain checkpoints.
:param df: The dataframe to iterate over.
:param chunk_size: The max. no. of rows to pick at once.
"""
self.df = df
self.row_count = df.shape[0]
self.chunk_size = chunk_size
self.offset = 0
def __iter__(self):
self.offset = 0
return self
def __next__(self):
if self.offset >= self.row_count:
raise StopIteration
end = self.offset + self.chunk_size
chunk = self.df.iloc[self.offset:end]
self.offset = end
return chunk
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/date_time/__init__.py
View File
utils_v2/date_time/date_time.py
-349
View File
@@ -1,349 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Friday, 21st jun, 2024
OBJECTIVE:
To provide an easy way to work with time.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For date and time handling:
import pytz
from datetime import datetime, time, timedelta, tzinfo
import dateparser
# For mathematical operations:
import math
# To handle date-time objects from a Numpy array and Pandas Dataframe:
import numpy as np
import pandas as pd
# To work with datatypes:
from typing import List, Literal
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Date-Time Formats:
DATE_TIME_FORMATS = (
"%d/%m/%y",
"%d-%b-%y",
"%d-%m-%y",
"%d.%m.%y",
"%d/%m/%Y",
"%d-%b-%Y",
"%d-%m-%Y",
"%d.%m.%Y",
"%d/%b",
"%d%m%Y",
"%Y%m%d",
"%Y/%m/%d",
"%Y-%m-%d",
"%Y-%m-%d %H:%M:%S"
)
# Useful Timezones:
TIMEZONE_UTC = pytz.timezone("UTC")
TIMEZONE_IST = pytz.timezone("Asia/Kolkata")
TIMEZONE_ET = pytz.timezone("America/New_York")
TIMEZONE_CT = pytz.timezone("America/Chicago")
TIMEZONE_MT = pytz.timezone("America/Denver")
TIMEZONE_PT = pytz.timezone("America/Los_Angeles")
TIMEZONE_JST = pytz.timezone("Asia/Tokyo")
TIMEZONE_CET = pytz.timezone("Europe/Paris")
TIMEZONE_GMT = pytz.timezone("GMT")
TIMEZONE_AEST = pytz.timezone("Australia/Sydney")
TIMEZONE_NZST = pytz.timezone("Pacific/Auckland")
TIMEZONE_CST = pytz.timezone("Asia/Shanghai")
TIMEZONE_KST = pytz.timezone("Asia/Seoul")
TIMEZONE_MSK = pytz.timezone("Europe/Moscow")
TIMEZONE_BRT = pytz.timezone("America/Sao_Paulo")
TIMEZONE_GST = pytz.timezone("Asia/Dubai")
TIMEZONE_SAST = pytz.timezone("Africa/Johannesburg")
TIMEZONE_AST = pytz.timezone("Asia/Riyadh")
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def translate_date_time_string(
datetime_string: str,
source_format: str = None,
destination_format: str = "%Y-%m-%dT%H:%M:%S"
) -> str | None:
"""
To convert an input datetime string to a different format.
:param datetime_string: The datetime string to translate.
:param source_format: The current format of the string. If not provided, dateparser will be used.
:param destination_format: The format to convert to.
:return: The converted datetime string.
"""
try:
if source_format is None: datetime_obj = dateparser.parse(datetime_string)
else: datetime_obj = datetime.strptime(datetime_string, source_format)
return datetime_obj.strftime(destination_format)
except Exception as exception:
return None
# ---------------------------------------------------------------------------------------------------------------------
def parse_date_time(
input_value: datetime | str | int | float,
timezone: str | tzinfo = None,
date_formats: List[str] = None
) -> datetime:
"""
To take any kind of input and interpret the datetime from it.
:param input_value: Either a string or an integer or some form of datetime representation.
:param timezone: The timezone to apply to the interpreted datetime. IF THE INPUT IS NAIVE, THIS TIMEZONE WILL BE
APPLIED AS IS, ELSE THE TIMEZONE WILL BE TRANSLATED.
:param date_formats: The string formats to consider when parsing a string input.
:return: The parsed datetime or null.
"""
datetime_object = None
date_formats = date_formats or DATE_TIME_FORMATS
# It could either be in seconds or milliseconds from epoch time's base date (January 1, 1970),
# or it could be days since Microsoft Excel's base date (December 31, 1899).
if isinstance(input_value, (int, float, np.number)) and not np.isnan(input_value):
if input_value > 9999999999.0: datetime_object = datetime.fromtimestamp(input_value / 1000.0)
if input_value > 999999.0: datetime_object = datetime.fromtimestamp(input_value)
else: datetime_object = datetime.fromtimestamp(input_value * 24 * 60 * 60.0) - timedelta(days = 25569)
# The input can even be a pre-formatted date:
if isinstance(input_value, str):
datetime_object = dateparser.parse(
input_value,
date_formats = date_formats,
settings = {
"DATE_ORDER": "DMY",
"PREFER_DAY_OF_MONTH": "first",
}
)
# If the type is a datetime object, then return it as it is:
if isinstance(input_value, datetime):
datetime_object = input_value
# If the type is the native datetime format of pandas:
if isinstance(input_value, pd._libs.tslibs.timestamps.Timestamp):
datetime_object = input_value.to_pydatetime()
# Process the timezone:
if datetime_object is not None and timezone is not None:
datetime_object = to_timezone(datetime_object, timezone)
# Done here:
return datetime_object
# ---------------------------------------------------------------------------------------------------------------------
def get_current_date_time(
timezone: str | tzinfo = None,
as_string: bool = False
) -> datetime:
"""
Returns the current time as a datetime object.
:param timezone: The timezone to apply to the returned datetime.
:param as_string: Whether, or not, you want the output as a string.
:return: The datetime object/string representing the current time.
"""
if timezone is not None and isinstance(timezone, str): timezone = pytz.timezone(timezone)
now = datetime.now(timezone)
return now.isoformat() if as_string else now
# ---------------------------------------------------------------------------------------------------------------------
def get_current_ist_date_time(
as_string = False
) -> datetime:
"""
Gives out the current time in IST timezone.
:param as_string: Whether, or not, you want the output as a string.
:return: The datetime object or string representing the current time.
"""
return get_current_date_time(
timezone = TIMEZONE_IST,
as_string = as_string
)
# ---------------------------------------------------------------------------------------------------------------------
def get_current_utc_date_time(
as_string: bool = False
) -> datetime:
"""
Gives out the current time in UTC timezone.
:param as_string: Whether, or not, you want the output as a string.
:return: The datetime object or string representing the current time.
"""
return get_current_date_time(
timezone = TIMEZONE_UTC,
as_string = as_string
)
# ---------------------------------------------------------------------------------------------------------------------
def as_if_timezone(
datetime_object: datetime,
timezone: str | tzinfo
) -> datetime:
"""
Ignores existing timezone info and applies the intended timezone.
The time stays the same, only the timezone marker changes.
e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-09 00:00:00+00:00
HINT: IT PRETENDS "AS IF" THE TIMEZONE WAS THE INPUT TIMEZONE.
:param datetime_object: The datetime object on which the timezone needs to be applied.
:param timezone: The timezone that needs to be applied.
:return: A timezone-aware datetime object.
"""
tz_object = pytz.timezone(timezone) if isinstance(timezone, str) else timezone
return tz_object.localize(datetime_object.replace(tzinfo = None))
# ---------------------------------------------------------------------------------------------------------------------
def to_timezone(
datetime_object: datetime,
timezone: str | tzinfo
) -> datetime:
"""
Converts from one timezone to another. The time is adjusted by computing the difference between the two timezones.
NOTE: THIS FUNCTION APPLIES TH EINPUT TIMEZONE IF THE INPUT DATETIME WAS TIMEZONE-NAIVE.
e.g. for IST to UTC: 2024-08-09 00:00:00+05:30 --> 2024-08-08 18:30:00+00:00
:param datetime_object: The datetime object on which the timezone needs to be applied.
:param timezone: The timezone that needs to be applied.
:return: A timezone-aware datetime object.
"""
if isinstance(timezone, str): timezone = pytz.timezone(timezone)
if datetime_object.tzinfo is None: dto = datetime_object.astimezone(timezone)
else: dto = datetime_object.astimezone(timezone)
return dto
# ---------------------------------------------------------------------------------------------------------------------
def round_date_time(
datetime_object: datetime,
rounding_seconds: int,
mode: Literal["floor", "ceil", "nearest"] = "nearest"
) -> datetime:
"""
'Snaps' the time to the closest 'n-second' window. For example, if you want to round the time off to the nearest
5-minute period (maybe for use cases like trading), you set the value of seconds to 300. This way any input value
of, say, 12:01:50 AM gets converted to 12:00:00 AM; and a value of 12:02:31 AM gets converted to 12:05:00 AM.
:param datetime_object: The input datetime that you want to round off.
:param rounding_seconds: The period of rounding in seconds. 60 for 1 minute, 300 for 5 minutes and so on.
:param mode: "floor" means the previous time bucket, "ceil" means the next time bucket, and "nearest" means
whichever is closer will be picked.
:return: The rounded date-time (with the original timezone).
"""
def round_by_mode() -> float | int:
if mode == "nearest": return round(int(total_seconds) / rounding_seconds) * rounding_seconds
if mode == "ceil": return math.ceil(int(total_seconds) / rounding_seconds) * rounding_seconds
if mode == "floor": return math.floor(int(total_seconds) / rounding_seconds) * rounding_seconds
# For timezone-naive cases:
if datetime_object.tzinfo is None:
total_seconds = datetime_object.timestamp()
rounded_seconds = round_by_mode()
rounded_datetime = datetime.fromtimestamp(rounded_seconds)
return rounded_datetime
# For timezone-aware cases:
else:
original_tz = datetime_object.tzinfo
_dt = to_timezone(datetime_object, timezone = original_tz)
total_seconds = _dt.timestamp()
rounded_seconds = round_by_mode()
rounded_datetime = datetime.fromtimestamp(rounded_seconds, tz = original_tz)
return rounded_datetime
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/goog/__init__.py
View File
utils_v2/goog/controllers/__init__.py
View File
utils_v2/goog/controllers/base.py
-479
View File
@@ -1,479 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 26th Nov., 2024
OBJECTIVE:
To provide a base class for common behaviour of Google's APIs.
REFERENCES:
1. Quickstart: https://developers.google.com/gmail/api/quickstart/python
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.date_time import date_time
from utils_v2.goog.models.api_call import GoogleApiResponse
from utils_v2.goog.models.auth_tokens import GoogleAuthTokens
# Related to Google:
from google_auth_oauthlib.flow import InstalledAppFlow
# To make API calls:
import httpx
# To work with date and time:
import datetime
# For working with datatypes:
from typing import Literal, List
# For debugging:
from icecream import IceCreamDebugger
import inspect
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncGoogleBase:
def __init__(
self,
service_name: str,
oauth_json: dict,
http_client: httpx.AsyncClient,
redirect_url: str = None,
debug = True,
debug_prefix = "Google | ",
debug_only_errors = True
):
"""
To initialize any Google API from one base class. The client's id and secret are available in the file
downloaded form https://console.cloud.google.com/apis/credentials (do not forget to select your app).
:param service_name: A string to identify this service.
:param oauth_json: The OAuth credentials downloaded from https://console.cloud.google.com/apis/credentials
:param http_client: An asynchronous HTTP client to make API calls.
:param redirect_url: Where you would like to receive the confirmation of the user authorization.
:param debug: Whether, or not, you would like to show debugging messages on the terminal.
:param debug_prefix: The prefix string to identify the debugging messages.
:param debug_only_errors: Whether you would like to show all debugging messages or just error messages.
"""
# Prepare the debugging utility:
self._debug_prefix = debug_prefix
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self._printer.disable()
self._debug_only_errors = debug_only_errors
# Accept the input configuration:
self._service_name = service_name
self._http_client = http_client
self._oauth_json = oauth_json
self._client_id = self._oauth_json["web"]["client_id"]
self._client_secret = self._oauth_json["web"]["client_secret"]
self._redirect_url = redirect_url
def enable_debug(self):
self._printer.enable()
def disable_debug(self):
self._printer.disable()
def debug_only_errors(self):
self._debug_only_errors = True
def debug_everything(self):
self._debug_only_errors = False
# ┏┓ •
# ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
# ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
# ┛
@property
def client_id(self):
return self._client_id
@property
def client_secret(self):
return self._client_secret
@property
def http_client(self):
return self._http_client
# ┏┓┏┓ ┓ ┏┓ ┏┓
# ┃┃┣┫┓┏╋┣┓ ┏┛ ┃┫
# ┗┛┛┗┗┻┗┛┗ ┗━•┗┛
async def get_authorization_url(
self,
scopes: List[str],
state: str = None,
access_type: Literal["online", "offline"] = "offline",
approval_prompt: Literal["auto", "force", "consent"] = "auto",
include_granted_scopes: Literal["true", "false"] = "true",
user_email: str = None
) -> str:
"""
TO get the OAuth2.0 authorization URL for one user.
DOCUMENTATION:
1. https://developers.google.com/identity/protocols/oauth2/web-server
:param scopes: The set of permission you want the user to give.
:param state: A unique identifier for your user. If not supplied, a random string will be generated.
:param access_type: Set the value to offline if your application needs to refresh access tokens when the user is
not present at the browser.
:param approval_prompt: "force" ensures that the consent screen is always shown to the user, regardless of
whether the user has previously granted consent for the requested scopes. It forces the user to re-approve
the app's access, which can be useful if the app is requesting new permissions or if the consent needs to be
explicitly confirmed. "consent" ensures the user's consent is required if they haven't approved the app's
requested permissions yet. "auto" allows Google to automatically determine whether the consent screen should
be shown.
:param include_granted_scopes: Enables applications to use incremental authorization to request access to
additional scopes in context. If you set this parameter's value to true and the authorization request is
granted, then the new access token will also cover any scopes to which the user previously granted the
application access.
:param user_email:
:return:
"""
# Create a flow:
flow = InstalledAppFlow.from_client_config(
self._oauth_json,
scopes = scopes,
redirect_uri = self._redirect_url
)
# Get an authorization URL:
auth_url, state = flow.authorization_url(
access_type = access_type,
approval_prompt = approval_prompt,
include_granted_scopes = include_granted_scopes,
login_hint = user_email,
state = state
)
# Done here:
return auth_url
async def get_authorization_tokens(
self,
scopes: List[str] | None,
redirect_url: str
) -> GoogleAuthTokens:
"""
When the user accepts or declines an authorization request, Google sends you an alert on your redirect URL. Pass
the URL as it is to this method to generate the authorization tokens that you can store in the database and
reuse for this user's activities.
:param scopes: The set of permissions the user granted.
:param redirect_url: The exact URL that was hit (with the query params) that Google hit when the user did
something on your authorization URL. Fortunately, this URL is readily available in Quart and Flask by
calling 'request.url'.
:return: The authorization tokens.
"""
# Create a flow:
flow = InstalledAppFlow.from_client_config(
self._oauth_json,
scopes = scopes,
redirect_uri = self._redirect_url
)
# Get the credentials:
credentials = flow.fetch_token(authorization_response = redirect_url)
ttl = credentials["expires_in"] - 60
return GoogleAuthTokens(
accessToken = credentials["access_token"],
refreshToken = credentials["refresh_token"],
expiresAt = date_time.get_current_utc_date_time() + datetime.timedelta(seconds = ttl),
scopes = credentials["scope"]
)
# ┏┓ ┳┓ ┓•
# ┣ ┏┓┏┓┏┓┏┓ ┃┃┏┓┏┏┓┏┫┓┏┓┏┓
# ┗┛┛ ┛ ┗┛┛ ┻┛┗ ┗┗┛┗┻┗┛┗┗┫
# ┛
@staticmethod
async def __get_error_message(api_response: GoogleApiResponse) -> str:
"""
To extract various kinds of error messages from Google's responses.
:param api_response: The formatted response from the API call.
:return: The message string.
"""
try: return (await api_response.get_json())["error"]["message"]
except: return api_response.response.reason_phrase
# ┏┓┏┓┳ ┏┓ ┓┓•
# ┣┫┃┃┃ ┃ ┏┓┃┃┓┏┓┏┓
# ┛┗┣┛┻ ┗┛┗┻┗┗┗┛┗┗┫
# ┛
async def get(
self,
url: str,
headers: dict = None,
params: dict = None
) -> GoogleApiResponse:
"""
To call an API using the GET method.
:param url: The URL to call.
:param headers: The headers to pass.
:param params: The params to send in the query string itself.
:return: A structured response that includes the raw response, the exception (if any), and so on.
"""
# Prepare the structure of the response:
api_response = GoogleApiResponse(
serviceName = self._service_name,
action = inspect.stack()[1].function,
url = url,
method = "GET"
)
try:
# Make the API call:
response = await self._http_client.get(
url = url,
headers = headers,
params = params
)
# Note down the results:
api_response.response = response
api_response.httpCode = response.status_code
api_response.message = await self.__get_error_message(api_response)
# If something goes wrong:
except Exception as exception:
api_response.exception = exception
api_response.message = str(exception)
self._printer(exception, api_response.url, api_response.method, headers, params)
# Done here:
return api_response
async def post(
self,
url: str,
headers: dict = None,
json: dict = None,
data: dict = None,
content: str | bytes = None
) -> GoogleApiResponse:
"""
To call an API using the POST method.
:param url: The URL to call.
:param headers: The headers to pass.
:param json: The params to send in the JSON body.
:param data: The params to send in the form-data in the body.
:param content: The raw content to be sent in the body (typically as an octet-stream).
:return: A structured response that includes the raw response, the exception (if any), and so on.
"""
# Prepare the structure of the response:
api_response = GoogleApiResponse(
serviceName = self._service_name,
action = inspect.stack()[1].function,
url = url,
method = "POST"
)
try:
# Make the API call:
response = await self._http_client.post(
url = url,
headers = headers,
json = json,
data = data,
content = content
)
# Note down the results:
api_response.response = response
api_response.httpCode = response.status_code
api_response.message = await self.__get_error_message(api_response)
# If something goes wrong:
except Exception as exception:
api_response.exception = exception
api_response.message = str(exception)
self._printer(exception, api_response.url, api_response.method, headers, json, data)
# Done here:
return api_response
async def put(
self,
url: str,
headers: dict = None,
json: dict = None,
data: dict = None
) -> GoogleApiResponse:
"""
To call an API using the PUT method.
:param url: The URL to call.
:param headers: The headers to pass.
:param json: The params to send in the JSON body.
:param data: The params to send in the form-data in the body.
:return: A structured response that includes the raw response, the exception (if any), and so on.
"""
# Prepare the structure of the response:
api_response = GoogleApiResponse(
serviceName = self._service_name,
action = inspect.stack()[1].function,
url = url,
method = "PUT"
)
try:
# Make the API call:
response = await self._http_client.put(
url = url,
headers = headers,
json = json,
data = data
)
# Note down the results:
api_response.response = response
api_response.httpCode = response.status_code
api_response.message = await self.__get_error_message(api_response)
# If something goes wrong:
except Exception as exception:
api_response.exception = exception
api_response.message = str(exception)
self._printer(exception, api_response.url, api_response.method, headers, json, data)
# Done here:
return api_response
async def delete(
self,
url: str,
headers: dict = None
) -> GoogleApiResponse:
"""
To call an API using the DELETE method.
:param url: The URL to call.
:param headers: The headers to pass.
:return: A structured response that includes the raw response, the exception (if any), and so on.
"""
# Prepare the structure of the response:
api_response = GoogleApiResponse(
serviceName = self._service_name,
action = inspect.stack()[1].function,
url = url,
method = "DELETE"
)
try:
# Make the API call:
response = await self._http_client.delete(
url = url,
headers = headers
)
# Note down the results:
api_response.response = response
api_response.httpCode = response.status_code
api_response.message = await self.__get_error_message(api_response)
# If something goes wrong:
except Exception as exception:
api_response.exception = exception
api_response.message = str(exception)
self._printer(exception, api_response.url, api_response.method, headers)
# Done here:
return api_response
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/goog/controllers/gmail/__init__.py
View File
utils_v2/goog/controllers/gmail/gmail_client.py
-977
View File
@@ -1,977 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Monday, 25th Nov., 2024
OBJECTIVE:
To manage e-mails in a GMail account.
REFERENCES:
1. GMail Quickstart: https://developers.google.com/gmail/api/quickstart/python
2. GMail Labels: https://developers.google.com/gmail/api/guides/labels
3. GMail Messages: https://developers.google.com/gmail/api/reference/rest/v1/users.messages
4. People Profile: https://developers.google.com/people/api/rest/v1/people/get
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
from utils_v2.mail import mail_parser
# My Google utils:
from utils_v2.goog.controllers.base import AsyncGoogleBase
from utils_v2.goog.models.auth_tokens import GoogleAuthTokens
from utils_v2.goog.models.api_call import GoogleApiResponse
from utils_v2.goog.controllers.gmail.gmail_message import GmailMessage
# Related to Google:
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
# To make API calls:
import httpx
# For asynchronous activities:
import asyncio
# To work with date and time:
import datetime
# For working with datatypes:
from typing import Dict, Literal, List, Any
# For debugging:
from icecream import IceCreamDebugger
import inspect
# For computational help:
import math
# For base64 encoding:
import base64
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Google Scopes:
SCOPES_GMAIL_MAIL_MANAGEMENT = [
r"https://www.googleapis.com/auth/gmail.modify",
r"https://www.googleapis.com/auth/gmail.labels",
# r"profile",
r"https://www.googleapis.com/auth/userinfo.profile"
]
SCOPES_GMAIL_FULL = [
r"https://mail.google.com/",
# r"profile",
r"https://www.googleapis.com/auth/userinfo.profile"
]
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncGmailClient(AsyncGoogleBase):
async def get_user_profile(
self,
tokens: GoogleAuthTokens,
user_id: str = "me",
) -> GoogleApiResponse:
"""
To get the list of labels of this user.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users/getProfile
2. https://developers.google.com/people/api/rest/v1/people/get
3. https://developers.google.com/people/api/rest/v1/people#Person
:param tokens: The object that holds the access token to the service.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the GMail API call:
if not self._debug_only_errors: self._printer("Getting User Profile.")
gmail_api_response = await self.get(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/profile",
headers = {"Authorization": f"Bearer {tokens.accessToken}"}
)
# If the call was successful:
if gmail_api_response.httpCode in [200]:
gmail_api_response.success = True
gmail_api_response.data = await gmail_api_response.get_json()
gmail_api_response.data["displayName"] = None
gmail_api_response.data["displayPictureUrl"] = None
# If the call failed:
else:
self._printer(
gmail_api_response.action,
gmail_api_response.method,
gmail_api_response.httpCode,
)
return gmail_api_response
# Make the People API call:
if not self._debug_only_errors: self._printer("Getting User Profile.")
people_api_response = await self.get(
url = f"https://people.googleapis.com/v1/people/me?personFields=names,photos,birthdays,phoneNumbers,genders,emailAddresses,addresses",
headers = {"Authorization": f"Bearer {tokens.accessToken}"}
)
# If the call was successful:
if people_api_response.httpCode in [200]:
people_api_response.success = True
people_api_response.data = await people_api_response.get_json()
for item in people_api_response.data.get("names", []):
if item["metadata"]["primary"]:
gmail_api_response.data["displayName"] = item.get("displayName")
for item in people_api_response.data.get("photos", []):
if item["metadata"]["primary"]:
gmail_api_response.data["displayPictureUrl"] = item.get("url")
# If the call failed:
else: self._printer(
people_api_response.action,
people_api_response.method,
people_api_response.httpCode,
)
# Done here:
return gmail_api_response
# ┓ ┓ ┓
# ┃ ┏┓┣┓┏┓┃┏
# ┗┛┗┻┗┛┗ ┗┛
async def list_labels(
self,
tokens: GoogleAuthTokens,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To get the list of labels of this user.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/list
2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
:param tokens: The object that holds the access token to the service.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Listing All Labels.", user_id)
api_response = await self.get(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels",
headers = {"Authorization": f"Bearer {tokens.accessToken}"}
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_json = await api_response.get_json()
api_response.data = {label["name"]: label for label in api_json.get("labels", [])}
# Done here:
return api_response
async def get_label(
self,
tokens: GoogleAuthTokens,
label_id: str,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To get one label of this user. the label will be identified by its id.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/get
2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
:param tokens: The object that holds the access token to the service.
:param label_id: The id that Google assigned to the label.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Getting One Label.", user_id)
api_response = await self.get(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels/{label_id}",
headers = {"Authorization": f"Bearer {tokens.accessToken}"}
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_response.data = await api_response.get_json()
# Done here:
return api_response
async def create_label(
self,
tokens: GoogleAuthTokens,
label_name: str,
label_visibility: Literal["labelShow", "labelShowIfUnread", "labelHide"] = "labelShow",
message_visibility: Literal["show", "hide"] = "show",
label_text_color: str = "#434343",
label_background_color: str = "#000000",
user_id: str = "me"
) -> GoogleApiResponse:
"""
Create one label for the user. Doesn't apply it to any mail, just creates it.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/create
2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
:param tokens: The object that holds the access token to the service.
:param label_name: The display name of the label.
:param label_visibility: Whether, or not, you would like to show the label in the web UI.
:param message_visibility: Whether, or not, you would like to show messages with this label in the web UI.
:param label_text_color: The colour of the text of the label.
:param label_background_color: The colour of the background/tag of the label.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Creating One Label.", user_id)
api_response = await self.post(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
json = {
"name": label_name,
"messageListVisibility": "show" if message_visibility else "hide",
"labelListVisibility": "labelShow" if label_visibility else "labelHide",
"color": {
"textColor": label_text_color.lower(),
"backgroundColor": label_background_color.lower()
}
}
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_response.data = await api_response.get_json()
# Done here:
return api_response
async def update_label(
self,
tokens: GoogleAuthTokens,
label_id: str,
label_name: str = None,
label_visibility: Literal["labelShow", "labelShowIfUnread", "labelHide"] = None,
message_visibility: Literal["show", "hide"] = None,
label_text_color: str = None,
label_background_color: str = None,
user_id: str = "me"
) -> GoogleApiResponse:
"""
Updates one label for the user.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/update
2. https://developers.google.com/gmail/api/reference/rest/v1/users.labels#Label
NOTE: Both or none of the colours must be updated. For this reason, a simple default will be chosen for the
other if only one is provided.
:param tokens: The object that holds the access token to the service.
:param label_id: The id that Google assigned to the label.
:param label_name: The display name of the label.
:param label_visibility: Whether, or not, you would like to show the label in the web UI.
:param message_visibility: Whether, or not, you would like to show messages with this label in the web UI.
:param label_text_color: The colour of the text of the label.
:param label_background_color: The colour of the background/tag of the label.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Format the JSON body:
json_body = {}
if label_name: json_body["name"] = label_name
if label_visibility: json_body["labelListVisibility"] = label_visibility
if message_visibility: json_body["messageListVisibility"] = message_visibility
if label_text_color or label_background_color:
json_body["color"] = {
"textColor": (label_text_color or "#434343").lower(),
"backgroundColor": (label_background_color or "#000000").lower()
}
# Make the API call:
if not self._debug_only_errors: self._printer("Updating One Label.", user_id)
api_response = await self.put(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels/{label_id}",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
json = json_body
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_response.data = await api_response.get_json()
# Done here:
return api_response
async def delete_label(
self,
tokens: GoogleAuthTokens,
label_id: str,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To delete one label of this user. the label will be identified by its id.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.labels/delete
:param tokens: The object that holds the access token to the service.
:param label_id: The id that Google assigned to the label.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Deleting One Label.", user_id)
api_response = await self.delete(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/labels/{label_id}",
headers = {"Authorization": f"Bearer {tokens.accessToken}"}
)
# If the call was successful:
if api_response.httpCode in [200, 204]:
api_response.success = True
# Done here:
return api_response
# ┳┳┓
# ┃┃┃┏┓┏┏┏┓┏┓┏┓┏
# ┛ ┗┗ ┛┛┗┻┗┫┗ ┛
# ┛
async def __list_messages_on_page(
self,
tokens: GoogleAuthTokens,
max_count: int = 100,
query: str = None,
label_ids: List[str] | str = None,
include_spam_and_trash: bool = False,
next_page_token: str = None,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To enlist messages on one page. Google allows at most 500 results on one page. This method respects that
pagination limit and returns only what Google gives. This method should be used internally by the class and the
class should expose another method that calls this one in loop to get any arbitrary no. of messages as the user
desires.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list
2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
:param tokens: The object that holds the access token to the service.
:param max_count: The no. of messages to fetch.
:param query: Any query filter that is supported by GMail.
:param label_ids: The list of labels' ids that the mails must have on them.
:param include_spam_and_trash: Whether, or not, you would like to include mails categorized as spam and trash.
:param next_page_token: The token to fetch the next set of results.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Build the needed params:
params_json = {
"maxResults": max_count,
"includeSpamTrash": include_spam_and_trash
}
if query: params_json["q"] = query
if next_page_token: params_json["pageToken"] = next_page_token
if label_ids: params_json["labelIds"] = label_ids if isinstance(label_ids, list) else [label_ids]
# Make the API call:
if not self._debug_only_errors: self._printer("Listing Messages for Page.", user_id, max_count, next_page_token)
api_response = await self.get(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
params = params_json
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_json = await api_response.get_json()
api_response.data = {
"messages": {m["id"]: m for m in api_json.get("messages", [])},
"nextPageToken": api_json.get("nextPageToken"),
"resultSizeEstimate": api_json["resultSizeEstimate"],
}
# Done here:
return api_response
async def list_messages(
self,
tokens: GoogleAuthTokens,
max_count: int = 100,
query: str = None,
label_ids: List[str] | str = None,
include_spam_and_trash: bool = False,
next_page_token: str = None,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To enlist mail messages from a user's account.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/list
2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
:param tokens: The object that holds the access token to the service.
:param max_count: The no. of messages to fetch.
:param query: Any query filter that is supported by GMail.
:param label_ids: The list of labels' ids that the mails must have on them.
:param include_spam_and_trash: Whether, or not, you would like to include mails categorized as spam and trash.
:param next_page_token: The token to fetch the next set of results.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# We create a variable that will hold the results.
# We must supply the URL, Method and a few other params here due to the custom looping functionality:
all_messages = GoogleApiResponse(
serviceName = self._service_name,
action = inspect.stack()[0].function,
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages",
method = "GET",
data = {
"messages": {},
"nextPageToken": None,
"resultSizeEstimate": 0
}
)
# Let's figure out how many times we'll have to loop through the process to retrieve the target no. of
# messages. Google allows you to fetch info about at most 500 messages in one go.
max_per_call = 500 # ... because Google allows at most 500 entries in one call.
iterations_needed = int(math.ceil(max_count / max_per_call))
last_iteration_count = max_count - int((max_per_call * (iterations_needed - 1)))
# Run the loop those many times:
results_size_estimate = 0
for iteration_no in range(iterations_needed):
# Figure out the count for this page:
if iterations_needed > 1:
if iteration_no < (iterations_needed - 1): iteration_count = max_per_call
else: iteration_count = last_iteration_count
else: iteration_count = max_count
# Retrieve the messages for this page:
iteration_response = await self.__list_messages_on_page(
tokens = tokens,
max_count = iteration_count,
query = query,
label_ids = label_ids,
include_spam_and_trash = include_spam_and_trash,
next_page_token = next_page_token
)
# Check if no data was received:
if (
iteration_response.data is None or
not iteration_response.data.get("messages")
): break
# Now that we know that messages were received:
for k, v in iteration_response.data["messages"].items(): all_messages.data["messages"][k] = v
results_size_estimate += iteration_response.data["resultSizeEstimate"]
# Also copy the API call params:
all_messages.httpCode = iteration_response.httpCode
all_messages.message = iteration_response.message
all_messages.success = iteration_response.success
# If there is no next page after this, we break out of the loop:
next_page_token = iteration_response.data["nextPageToken"]
if next_page_token is None: break
# Format the final response:
all_messages.data["nextPageToken"] = next_page_token
all_messages.data["resultSizeEstimate"] = results_size_estimate
# Done here:
return all_messages
async def get_message(
self,
tokens: GoogleAuthTokens,
message_id: str,
return_raw: bool = False,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To get one message of this user. The message will be identified by its id. Note that, if you choose to return
the raw message, the message body will be compliant with RFC 5322 and RFC 2045 (among others).
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/get
2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
2. https://developers.google.com/gmail/api/reference/rest/v1/Format
:param tokens: The object that holds the access token to the service.
:param message_id: The id that Google assigned to the message.
:param return_raw: Whether you want the raw message or the formatted message.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Getting One Message.", user_id)
api_response = await self.get(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/{message_id}",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
params = {"format": "raw"}
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_json = await api_response.get_json()
raw_message = base64.urlsafe_b64decode(api_json["raw"]).decode()
if return_raw: api_response.data = raw_message
else:
parsed_message = mail_parser.parse(raw_message)
parsed_message["labels"] = api_json["labelIds"]
parsed_message["messageId"] = api_json["id"]
parsed_message["threadId"] = api_json["threadId"]
parsed_message["historyId"] = api_json["historyId"]
parsed_message["snippet"] = api_json["snippet"]
parsed_message["sizeEstimate"] = api_json["sizeEstimate"]
api_response.data = parsed_message
# Done here:
return api_response
async def modify_messages(
self,
tokens: GoogleAuthTokens,
message_ids: List[str] | str,
add_label_ids: List[str] | str = None,
remove_label_ids: List[str] | str = None,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To add or remove labels from one or more messages.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/batchModify
:param tokens: The object that holds the access token to the service.
:param message_ids: One or more message ids (assigned by Google).
:param add_label_ids: One or more label ids (not the display name of the label).
:param remove_label_ids: One or more label ids (not the display name of the label).
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Build the needed JSON:
body_json = {"ids": message_ids if isinstance(message_ids, list) else [message_ids]}
if add_label_ids: body_json["addLabelIds"] = add_label_ids if isinstance(add_label_ids, list) else [add_label_ids]
if remove_label_ids: body_json["removeLabelIds"] = remove_label_ids if isinstance(remove_label_ids, list) else [remove_label_ids]
# Make the API call:
if not self._debug_only_errors: self._printer("Modifying Message(s).", user_id)
api_response = await self.post(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/batchModify",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
json = body_json
)
# If the call was successful:
if api_response.httpCode in [200, 204]:
api_response.success = True
# Done here:
return api_response
async def delete_messages(
self,
tokens: GoogleAuthTokens,
message_ids: List[str] | str,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To PERMANENTLY delete one or more messages.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/batchDelete
:param tokens: The object that holds the access token to the service.
:param message_ids: One or more message ids (assigned by Google).
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Deleting Message(s).", user_id)
api_response = await self.post(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/batchDelete",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
json = {"ids": message_ids if isinstance(message_ids, list) else [message_ids]}
)
# If the call was successful:
if api_response.httpCode in [200, 204]:
api_response.success = True
# Done here:
return api_response
async def trash_message(
self,
tokens: GoogleAuthTokens,
message_id: str,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To move one message to trash.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/trash
2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
:param tokens: The object that holds the access token to the service.
:param message_id: The id of the message (assigned by Google).
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Trashing One Message.", user_id)
api_response = await self.post(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/{message_id}/trash",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
)
# If the call was successful:
if api_response.httpCode in [200, 204]:
api_response.success = True
api_response.data = await api_response.get_json()
# Done here:
return api_response
async def untrash_message(
self,
tokens: GoogleAuthTokens,
message_id: str,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To move one message to trash.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/trash
2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
:param tokens: The object that holds the access token to the service.
:param message_id: The id of the message (assigned by Google).
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Make the API call:
if not self._debug_only_errors: self._printer("Un-Trashing One Message.", user_id)
api_response = await self.post(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/{message_id}/untrash",
headers = {"Authorization": f"Bearer {tokens.accessToken}"},
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_response.data = await api_response.get_json()
# Done here:
return api_response
async def send_message(
self,
tokens: GoogleAuthTokens,
message: GmailMessage,
thread_id: str = None,
user_id: str = "me"
) -> GoogleApiResponse:
"""
To send one message.
NOTE: If you want to apply a custom label to your outgoing mails, this API endpoint doesn't allow you to do that
in one go. Instead, you should note down the 'id' field from a successful response and use the 'modify_messages'
method of this class to immediately apply that label to the mail in a separate call.
DOCUMENTATION:
1. https://developers.google.com/gmail/api/reference/rest/v1/users.messages/send
2. https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
:param tokens: The object that holds the access token to the service.
:param message: The object that has the content of the message to be sent.
:param thread_id: Replies to an existing mail if the correct thread-id is specified. If not specified, a new
mail with a new thread-id is created.
:param user_id: The id of the user for which this service must be run. Typically, the e-mail id itself or "me".
:return: A structured response where the list of labels will be in the 'data' variable.
"""
# Ensure that the tokens are valid:
await tokens.arefresh(
http_client = self._http_client,
client_id = self._client_id,
client_secret = self._client_secret,
force_refresh = False
)
# Construct the JSON body:
json_body = {"raw": message.get_raw_message(as_base64 = True)}
if thread_id: json_body["threadId"] = thread_id
# Make the API call:
if not self._debug_only_errors: self._printer("Sending One Message.", user_id)
api_response = await self.post(
url = f"https://gmail.googleapis.com/gmail/v1/users/{user_id}/messages/send",
headers = {
"Authorization": f"Bearer {tokens.accessToken}",
"Content-Type": "message/rfc822"
},
json = json_body
)
# If the call was successful:
if api_response.httpCode in [200]:
api_response.success = True
api_response.data = await api_response.get_json()
# Done here:
return api_response
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
# Create an HTTP client:
test_client = httpx.AsyncClient(
limits = httpx.Limits(
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
),
timeout = httpx.Timeout(
connect = 2.5, # ... Shorter connection timeout.
read = 2.5, # ...... Like what EasyEcom gives.
write = 10.0, # .... Time to wait for sending data.
pool = 120.0 # ..... Time to wait for a free connection from the pool.
)
)
# Read the secrets that give you access to the app:
secrets_file = r"../../../../creds/goog/app/google_tcaoff_test_oauth_20241125.json"
secrets_dict = json.from_file(secrets_file)
async def main():
# Create an instance of the client:
my_gmail = AsyncGmailClient(
service_name = "gmail",
oauth_json = secrets_dict,
http_client = test_client,
redirect_url = r"https://api.thecaoffice.com/converse/mail/callback/gmail",
debug = True,
debug_prefix = "GMail (M) | ",
debug_only_errors = False
)
# Request Auth:
print("AUTH URL:", await my_gmail.get_authorization_url(
scopes = SCOPES_GMAIL_MAIL_MANAGEMENT,
state = "Bhopli",
approval_prompt = "force"
))
# Get tokens from callback:
test_tokens = await my_gmail.get_authorization_tokens(
scopes = SCOPES_GMAIL_MAIL_MANAGEMENT,
redirect_url = input("Paste the redirect URL here: ")
)
print("TOKENS:", test_tokens)
# Test some feature:
response = await my_gmail.get_user_profile(tokens = test_tokens)
print("SUCCESS:", response.success)
print("SUMMARY:", response.to_markdown())
print("\n\n---\n\n")
print("DATA:", json.to_string(response.data, default = str))
if not response.success:
print("\n\n---\n\n")
asyncio.run(main())
utils_v2/goog/controllers/gmail/gmail_message.py
-349
View File
@@ -1,349 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 16th Jul, 2024
OBJECTIVE:
To be able to send out mails from code.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For working with mails:
from email.mime.multipart import MIMEMultipart
from email.mime.image import MIMEImage
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
# For random strings:
import string
import random
# For system-level activities:
import os
# For working with files in RAM:
import io
# To work with Base64 encoding:
import base64
# To work with datatypes:
from typing import List
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class GmailMessage:
def __init__(
self,
from_email: str,
to_email: str | List[str],
subject: str,
cc_emails: str | List[str] = None,
bcc_emails: str | List[str] = None
):
"""
Create an instance of the message that you would like to send.
:param from_email: The EMail ID of the sender.
:param to_email: The EMail ID of the recipient.
:param subject: The subject of the mail.
:param cc_emails: A list of recipients to add to the CC section.
:param bcc_emails: A list of recipients to add to the BCC section.
"""
# Create the instance of the message:
self.message = MIMEMultipart()
self.message["From"] = from_email
self.message["To"] = ",".join(to_email if isinstance(to_email, list) else [to_email])
self.message["Subject"] = subject
if cc_emails: self.message["CC"] = ",".join(cc_emails if isinstance(cc_emails, list) else [cc_emails])
if bcc_emails: self.message["BCC"] = ",".join(bcc_emails if isinstance(bcc_emails, list) else [bcc_emails])
# Note down the values for accessing later:
self.__from = from_email
self.__to = to_email
self.__cc = cc_emails,
self.__bcc = bcc_emails
self.__subject = subject
# ┏┓ •
# ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
# ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
# ┛
@property
def from_mail(self):
return self.__from
@property
def to_mail(self):
return self.__to
@property
def cc_mails(self):
return self.__cc
@property
def bcc_mails(self):
return self.__bcc
@property
def subject(self):
return self.__subject
# ┏┓ ┓ ┓ ┏┓
# ┣┫┏┫┏┫ ┃ ┏┓┏┓╋┏┓┏┓╋
# ┛┗┗┻┗┻ ┗┛┗┛┛┗┗┗ ┛┗┗
def add_text(self, text) -> None:
"""
Add plain-text to the mail body.
:param text: The text to add to the mail body.
:return: None.
"""
self.message.attach(MIMEText(text, "plain"))
def add_html(self, html_text) -> None:
"""
Add HTML text to the mail body.
:param html_text: The HTML text to add to the mail body.
:return: None.
"""
self.message.attach(MIMEText(html_text, "html"))
def add_inline_image(
self,
image_file: str | io.BytesIO,
file_name: str = None,
content_id: str = None
) -> None:
"""
Add an inline image to the body of the mail.
NOTE: This is NOT the same as sending an image as an attachment.
:param image_file: The image data to attach to the mail body.
:param file_name: The name of the file. This is the same name by which it will be downloaded. You need not
specify this if the input file is specified as a path. Needed when you give the input file as a buffer.
:param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not
specified, I will generate a random string. You may write a custom value here if you know what you are
doing. For most use cases, please ignore this field.
:return: None.
"""
# Read the image as bytes:
image_bytes = None
if type(image_file) is str:
file_name = file_name or os.path.split(image_file)[-1]
with open(image_file, "rb") as opened_image_file:
image_bytes = opened_image_file.read()
if type(image_file) is io.BytesIO:
image_file.seek(0)
image_bytes = image_file.getvalue()
# Declare the part to be attached to the multipart message:
if image_bytes is not None:
# Create the HTML block if the image pointer is blank:
if content_id is None:
content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
self.add_html(f"""
<html>
<body>
<p><img src="cid:{content_id}"></p>
</body>
</html>
""")
# Then add the image:
image_part = MIMEImage(image_bytes)
image_part.add_header(
"Content-ID", f"<{content_id}>"
)
image_part.add_header(
"Content-Disposition",
f"inline; filename=\"{file_name}\"",
)
self.message.attach(image_part)
def add_attachment(
self,
attachment_file: str | io.BytesIO,
file_name: str = None
) -> None:
"""
Add a file as an attachment to the mail. This file, even if possible, will not be rendered on the screen in-line
with the body. It will be made available as a download.
:param attachment_file: The file that you would like to attach.
:param file_name: The name of the file. This is the same name by which it will be downloaded. You need not
specify this if the input file is specified as a path. Needed when you give the input file as a buffer.
:return: None.
"""
# Declare the part to be attached to the multipart message:
part = MIMEBase("application", "octet-stream")
# If the attachment is a file stored in the local disk:
if isinstance(attachment_file, str):
file_name = file_name or os.path.split(attachment_file)[-1]
with open(attachment_file, "rb") as attachment:
part.set_payload(attachment.read())
# If the file is held in RAM:
elif isinstance(attachment_file, io.BytesIO):
attachment_file.seek(0)
part.set_payload(attachment_file.read())
# Encode and attach the file:
encoders.encode_base64(part)
part.add_header(
"Content-Disposition",
f"attachment; filename=\"{file_name}\"",
)
self.message.attach(part)
# ┏┓
# ┣ ┏┓┏┓┏┳┓┏┓╋
# ┻ ┗┛┛ ┛┗┗┗┻┗
def get_raw_message(
self,
as_base64: bool = False
) -> str:
"""
Get the raw string dump from the current contents of the message. This text will be compliant with RFC 5322 and
RFC 2045 (among others).
:param
:param as_base64: If set to True, the response will be a URL-safe B64 output, else it'll be a raw string.
:return: The standardized raw text dump. either as a raw string or as a Base64 (url-safe) string.
"""
if as_base64: return base64.urlsafe_b64encode(self.message.as_bytes()).decode()
else: return self.message.as_string()
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
# Create an instance of a mail message:
my_mail = GmailMessage(
from_email = "sender@gmail.com",
to_email = "recipient@gmail.com",
subject = "Bhopli is the best!",
cc_emails = None,
bcc_emails = None
)
# Add content to it:
my_mail.add_html(
"""
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Sample HTML String</title>
<style>
.heading {
color: #ff9025;
}
.sub-heading {
color: #000000;
}
</style>
</head>
<body>
<h1 class="heading">Hello, Bhopli!</h1>
<h2 class="sub-heading">Bhopli is the best, most well-behaved cat in the known universe.</h2>
</body>
</html>
"""
)
my_mail.add_text("This is how you should pet her 👇")
my_mail.add_inline_image(r"../../../../data/images/cat_petting.png")
my_mail.add_attachment(r"../../../../data/pdf/sample_label.pdf")
# Get the raw contents of the message:
mail_raw = my_mail.get_raw_message(as_base64 = False)
print("RAW TYPE:", type(mail_raw))
# Try parsing it:
from utils_v2.string import json
from utils_v2.mail import mail_parser
mail_json = mail_parser.parse(mail_raw)
print("PARSED MAIL (JSON):", json.to_string(mail_json, default = str))
utils_v2/goog/controllers/gmail/json/messages/chatgpt_suggested_message_format.json
-69
View File
@@ -1,69 +0,0 @@
{
"_id": ObjectId("..."), // MongoDB auto-generated ID
"message_id": "unique-message-id", // Unique identifier for the email
"thread_id": "thread-id", // Optional: Group of related messages
"subject": "Subject of the email",
"from": {
"name": "Sender Name", // Name of the sender
"email": "sender@example.com" // Sender's email address
},
"to": [ // List of recipients
{
"name": "Recipient Name",
"email": "recipient@example.com"
}
],
"cc": [ // List of CC recipients (optional)
{
"name": "CC Name",
"email": "cc@example.com"
}
],
"bcc": [ // List of BCC recipients (optional)
{
"name": "BCC Name",
"email": "bcc@example.com"
}
],
"date": ISODate("2024-11-26T12:00:00Z"), // Date the email was sent
"headers": { // Raw headers from the email
"X-Priority": "3",
"Content-Type": "multipart/alternative; boundary=\"boundary\"",
"X-Mailer": "Mailer XYZ"
},
"body": { // The body of the email, with parts if multipart
"text": "Plain text body content", // Plain text part (if any)
"html": "<p>HTML body content</p>", // HTML part (if any)
"parts": [ // List of parts for multipart emails
{
"content_type": "text/plain",
"content_transfer_encoding": "base64",
"content": "base64-encoded-content-here"
},
{
"content_type": "text/html",
"content_transfer_encoding": "base64",
"content": "base64-encoded-html-content-here"
}
]
},
"attachments": [ // Attachments in the email
{
"filename": "file1.pdf",
"content_type": "application/pdf",
"content_transfer_encoding": "base64",
"content": "base64-encoded-file-content"
},
{
"filename": "image1.jpg",
"content_type": "image/jpeg",
"content_transfer_encoding": "base64",
"content": "base64-encoded-image-content"
}
],
"flags": { // Optional flags for internal tracking
"read": false,
"spam": false
},
"received_timestamp": ISODate("2024-11-26T12:01:00Z") // Time the email was received (optional)
}
utils_v2/goog/controllers/gmail/json/messages/formatted_message_list_20241126.json
-36
View File
@@ -1,36 +0,0 @@
{
"messages": {
"193669615aa33694": {
"threadId": "193669615aa33694"
},
"19365d0239aa1aaf": {
"threadId": "19365d0239aa1aaf"
},
"193641e1379da64d": {
"threadId": "193641e1379da64d"
},
"193640e74d93dc9b": {
"threadId": "193640e74d93dc9b"
},
"19363d6a7f50225a": {
"threadId": "19363d6a7f50225a"
},
"19363ba8e2582133": {
"threadId": "19363ba8e2582133"
},
"19362f9983e58ea6": {
"threadId": "19362f9983e58ea6"
},
"19360aabcb976cfc": {
"threadId": "19360aabcb976cfc"
},
"193604ad97c9fe62": {
"threadId": "193604ad97c9fe62"
},
"1935e92672e67f22": {
"threadId": "1935e92672e67f22"
}
},
"nextPageToken": "11954226706764006151",
"resultSizeEstimate": 402
}
utils_v2/goog/controllers/gmail/json/messages/raw_message_01_193669615aa33694_no_attachment_20241126.json
-148
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_02_19367930033154ca_with_attachment_20241126.json
-258
View File
@@ -1,258 +0,0 @@
{
"id": "19367930033154ca",
"threadId": "19367930033154ca",
"labelIds": [
"UNREAD",
"IMPORTANT",
"CATEGORY_PERSONAL",
"INBOX"
],
"snippet": "Sample content. Inline graphics done. Sample monospaced How about some Headers? Hello, World! These are quotes. How about some emojis? \ud83d\ude1a\ud83d\ude05\ud83d\ude05\ud83e\udd70\ud83d\ude18\ud83d\ude01\ud83d\ude01 There&#39;s also an attachment!",
"payload": {
"partId": "",
"mimeType": "multipart/mixed",
"filename": "",
"headers": [
{
"name": "Delivered-To",
"value": "pskhushal@gmail.com"
},
{
"name": "Received",
"value": "by 2002:a5d:4205:0:b0:382:44a8:2102 with SMTP id n5csp1327351wrq; Tue, 26 Nov 2024 00:25:08 -0800 (PST)"
},
{
"name": "X-Received",
"value": "by 2002:a05:690c:6111:b0:6e3:fd6:6ccb with SMTP id 00721157ae682-6eee08c3459mr156525787b3.13.1732609507838; Tue, 26 Nov 2024 00:25:07 -0800 (PST)"
},
{
"name": "ARC-Seal",
"value": "i=1; a=rsa-sha256; t=1732609507; cv=none; d=google.com; s=arc-20240605; b=VDostCh8Kr1QGeQoXgLjYyrkymtwKEVHTyPaNeSjy30Q0B4GTTIBIPbKgkL6UazG4T s92UnE+Br4HJUSsqbtOdKTV1Pwt9lI3EzASDMX3OVGccdViTyUgGjgNuagV2fUVQugYi BJWxLrWZXVVtKkLNqRDOEQgiPi6t869FTB4877V8dKxQs99Q8IBncekOq3f6+SmsFlIl U3l6EUgaV/7MFzNRofsmXFGEG0TLto+Xgx5bMQ8B0OpjSFXR2Va/1xF5BmrzJTLgL6N5 zct3VIRLMxkf+8v3oniTG7yaaCQMPLxOq1pKXhP/MfKzgvdo5r14/kAzsj05rQ/L2/rT 2OrQ=="
},
{
"name": "ARC-Message-Signature",
"value": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605; h=cc:to:subject:message-id:date:from:mime-version:dkim-signature; bh=9NSLykdU3XzuxItzVTTQJjRRBR6yhJqdgYRarHa+VdY=; fh=KtxgdfxhT3MZUXDfdB/Mxp08o9TbOvoJMvz1V2wYTKk=; b=Xqtl2jfteUY2I+wdSwdllaqPKg1QP8hUyJpP/644zwGrOBdLbjCO8Po5C9KJfxUh7i H7UMWfEWkulB52WFTDU/54UY8lnX7iGZEMtT/4twKUGToWPPWeG/wmh/I4W9ZEmA/qXG OiRrBlJI3NJowsrnfoqXrpV/W5mz1SNuJBkj5k71+3vrPVlEmGwGmz8/MCG+RF2yV4nz QEVg613QkuThPEEsgYdHbfWDlF8jnGbAS+Lyt8ZdixVvO4/Gq5swuGUHihfeV+Fxt5Kv L2Wbqiq7UFJF9YJEtezJOdkdrTxs8dz6bTUZTrpWtRYxoBuH76HB+f9yrhOIRFTxor+9 18dA==; dara=google.com"
},
{
"name": "ARC-Authentication-Results",
"value": "i=1; mx.google.com; dkim=pass header.i=@gmail.com header.s=20230601 header.b=\"Adx/Ch2H\"; spf=pass (google.com: domain of orangebhopli@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=orangebhopli@gmail.com; dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com; dara=pass header.i=@gmail.com"
},
{
"name": "Return-Path",
"value": "<orangebhopli@gmail.com>"
},
{
"name": "Received",
"value": "from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41]) by mx.google.com with SMTPS id 00721157ae682-6eee01062easor78223267b3.12.2024.11.26.00.25.07 (Google Transport Security); Tue, 26 Nov 2024 00:25:07 -0800 (PST)"
},
{
"name": "Received-SPF",
"value": "pass (google.com: domain of orangebhopli@gmail.com designates 209.85.220.41 as permitted sender) client-ip=209.85.220.41;"
},
{
"name": "Authentication-Results",
"value": "mx.google.com; dkim=pass header.i=@gmail.com header.s=20230601 header.b=\"Adx/Ch2H\"; spf=pass (google.com: domain of orangebhopli@gmail.com designates 209.85.220.41 as permitted sender) smtp.mailfrom=orangebhopli@gmail.com; dmarc=pass (p=NONE sp=QUARANTINE dis=NONE) header.from=gmail.com; dara=pass header.i=@gmail.com"
},
{
"name": "DKIM-Signature",
"value": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20230601; t=1732609507; x=1733214307; dara=google.com; h=cc:to:subject:message-id:date:from:mime-version:from:to:cc:subject :date:message-id:reply-to; bh=9NSLykdU3XzuxItzVTTQJjRRBR6yhJqdgYRarHa+VdY=; b=Adx/Ch2H1Vw5fzCEcIPf/EuHGviFMr1/Gw93WwruoabampXxraQ5hkcJNx0yi+zM2/ 9wlmVqcDwiWM1dFrkc1hPNMyyH4h7hCXjGsDbK8hkUE/Fk1AR51fhjliB9AZFkps+PUR 21CYTDMzWWlHjKlBpnM1axU9suvv04NqDZ8Hr7LAXysd9eF1ypoTWjwWMJ5QGKpoQo6W qf3m5yuqSXLAtbTKZN6K3qUO+S9ZRjUahw0eke7ieQ2uKR8dFwaoS899KhVBY5bZpLzr gEjYkNbzZGN4bp/LIJcHmgueZ5J0L6V+zaYMIqNRp3wOwAe41nZSAf6PvLxixLZoTvyH TmmA=="
},
{
"name": "X-Google-DKIM-Signature",
"value": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1732609507; x=1733214307; h=cc:to:subject:message-id:date:from:mime-version:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=9NSLykdU3XzuxItzVTTQJjRRBR6yhJqdgYRarHa+VdY=; b=F5yE7IslZpO2UHCSg9jCYE645+q7dW9WO9S0DIb4oRcTlz/qgI1fwnnB0vlWvgXQxJ Af8/WUW6AJm/8mJ//SNXSDwE28OZiZqR+i/SYc0yontGepFawQAZg9xFRLspp7O/2UAL EHrkssz9noe2wPPdcPGYanXfoiSLtX/3XnrGCg1P5Y4jdy7p6624f87Xx+Mno1t8q2xh Odh2QrURKad50BqY+UB20+um9ommh4J0BOsD9WiVCfu5isvTCfwZYAtjYUJ48UVWLh3b kAPAaSic4g1zQzlOychUnqFB95VrrODUNUACfKBuIR0YbUdsgjMzn9p4RQ1LAgr9NfvY 62kw=="
},
{
"name": "X-Forwarded-Encrypted",
"value": "i=1; AJvYcCVe+1Kjt1hhMEjYG2UyrzV/DvH4xn376Ya7lsLKkTIVIAJGmG8xKSkvvJAmXIAXRyYuWSNpPVXGyjSKXTgsLlU=@gmail.com"
},
{
"name": "X-Gm-Message-State",
"value": "AOJu0Ywb3oN+4RLkTx99fCaTMOlGwSTJR7AWBy7IvxJyBYNwLV6Of7c2 vtYqtaRl1CcUgUXmGnXqjfQsSX6DT4LLNPGCsrrzk4VXfV26FJA1iaDFm/ey7pRaAucm5dGPHRO 9oG7H+96wM0pzmY3Tlb9LUEY3NIdSR9H2"
},
{
"name": "X-Gm-Gg",
"value": "ASbGncuwgQwpQZrb7exEtts26PEAwkklAu9DD/4RvvBSiHYGDUD7+909oeyrMtfk66q IuA94Z8WcFFe3TabyhyXIjGTCEF5+Qg=="
},
{
"name": "X-Google-Smtp-Source",
"value": "AGHT+IFy7wXSaJCDrrXclKRzxMFtyv1HrgyWSfgyomoDnvBS4NPu9KBdzHjvcekKjw9uKzJAj5R2iOl+mZuCt5sjIws="
},
{
"name": "X-Received",
"value": "by 2002:a05:690c:6701:b0:6ee:b5a6:a67a with SMTP id 00721157ae682-6eee0a402acmr176495297b3.28.1732609506221; Tue, 26 Nov 2024 00:25:06 -0800 (PST)"
},
{
"name": "MIME-Version",
"value": "1.0"
},
{
"name": "From",
"value": "TheBhopli <orangebhopli@gmail.com>"
},
{
"name": "Date",
"value": "Tue, 26 Nov 2024 13:54:54 +0530"
},
{
"name": "Message-ID",
"value": "<CAA-t6761d=GwN3to+252S34rA=gjv+Ao+u5WVbz+Gj+5oiwb1g@mail.gmail.com>"
},
{
"name": "Subject",
"value": "Test Mail for GMail API."
},
{
"name": "To",
"value": "\"pskhushal@gmail.com\" <pskhushal@gmail.com>"
},
{
"name": "Cc",
"value": "\"khushal@easyfi.net.in\" <khushal@easyfi.net.in>, \"bhushan.thakkar@gmail.com\" <bhushan.thakkar@gmail.com>"
},
{
"name": "Content-Type",
"value": "multipart/mixed; boundary=\"00000000000027d46c0627cc96ea\""
}
],
"body": {
"size": 0
},
"parts": [
{
"partId": "0",
"mimeType": "multipart/related",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "multipart/related; boundary=\"00000000000027d46c0627cc96e9\""
}
],
"body": {
"size": 0
},
"parts": [
{
"partId": "0.0",
"mimeType": "multipart/alternative",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "multipart/alternative; boundary=\"00000000000027d46c0627cc96e8\""
}
],
"body": {
"size": 0
},
"parts": [
{
"partId": "0.0.0",
"mimeType": "text/plain",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "text/plain; charset=\"UTF-8\""
},
{
"name": "Content-Transfer-Encoding",
"value": "quoted-printable"
}
],
"body": {
"size": 236,
"data": "U2FtcGxlIGNvbnRlbnQuDQoNCltpbWFnZTogY2hhdC5wbmddDQoNCklubGluZSBncmFwaGljcyBkb25lLg0KDQpTYW1wbGUgbW9ub3NwYWNlZA0KDQoqSG93IGFib3V0IHNvbWUgSGVhZGVycz8qDQoNCkhlbGxvLCBXb3JsZCEgVGhlc2UgYXJlIHF1b3Rlcy4NCg0KDQpIb3cgYWJvdXQgc29tZSBlbW9qaXM_IPCfmJrwn5iF8J-YhfCfpbDwn5iY8J-YgfCfmIENClRoZXJlJ3MgYWxzbyBhbiAqYXR0YWNobWVudCohDQo="
}
},
{
"partId": "0.0.1",
"mimeType": "text/html",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "text/html; charset=\"UTF-8\""
},
{
"name": "Content-Transfer-Encoding",
"value": "quoted-printable"
}
],
"body": {
"size": 675,
"data": "PGRpdiBkaXI9Imx0ciI-U2FtcGxlIGNvbnRlbnQuPGRpdj48YnI-PGRpdj48aW1nIHNyYz0iY2lkOmlpX20zeTZ0dTg1MCIgYWx0PSJjaGF0LnBuZyIgd2lkdGg9IjIyMiIgaGVpZ2h0PSIyMjIiIHN0eWxlPSJtYXJnaW4tcmlnaHQ6IDBweDsiPjwvZGl2PjxkaXY-PGJyPjwvZGl2PjxkaXY-SW5saW5lIGdyYXBoaWNzIGRvbmUuPC9kaXY-PGRpdj48YnI-PC9kaXY-PGRpdj48Zm9udCBmYWNlPSJtb25vc3BhY2UiPlNhbXBsZSBtb25vc3BhY2VkPC9mb250PjwvZGl2PjxkaXY-PGJyPjwvZGl2PjxkaXY-PGI-PGZvbnQgc2l6ZT0iNiI-SG93IGFib3V0IHNvbWUgSGVhZGVycz88L2ZvbnQ-PC9iPjwvZGl2PjxkaXY-PGJyPjwvZGl2PjxibG9ja3F1b3RlIGNsYXNzPSJnbWFpbF9xdW90ZSIgc3R5bGU9Im1hcmdpbjowcHggMHB4IDBweCAwLjhleDtib3JkZXItbGVmdDoxcHggc29saWQgcmdiKDIwNCwyMDQsMjA0KTtwYWRkaW5nLWxlZnQ6MWV4Ij5IZWxsbywgV29ybGQhIFRoZXNlIGFyZSBxdW90ZXMuPC9ibG9ja3F1b3RlPjxkaXY-PGJyPjwvZGl2PjxkaXY-SG93IGFib3V0IHNvbWUgZW1vamlzP8Kg8J-YmvCfmIXwn5iF8J-lsPCfmJjwn5iB8J-YgTxicj48L2Rpdj48L2Rpdj48ZGl2PlRoZXJlJiMzOTtzIGFsc28gYW4gPGk-PHU-YXR0YWNobWVudDwvdT48L2k-ITwvZGl2PjwvZGl2Pg0K"
}
}
]
},
{
"partId": "0.1",
"mimeType": "image/png",
"filename": "chat.png",
"headers": [
{
"name": "Content-Type",
"value": "image/png; name=\"chat.png\""
},
{
"name": "Content-Disposition",
"value": "inline; filename=\"chat.png\""
},
{
"name": "Content-Transfer-Encoding",
"value": "base64"
},
{
"name": "Content-ID",
"value": "<ii_m3y6tu850>"
},
{
"name": "X-Attachment-Id",
"value": "ii_m3y6tu850"
}
],
"body": {
"attachmentId": "ANGjdJ88SNHhnjq5-Alk71WYYaL72CUhJn6_geE_GP7JZKZdcSIR56LPGVklvFO_IuMFqeDkDT2U4r8SXTSMb-uBnNrio-NeXiW2p8vPWnV2yq-YaZxG0xBD6WILt5Qwg_43lv9_TGnjSvWn5QrGzusOttPluVRIWtv0-dksvYXrHDbS5MC0IYX4qpv8dbvGto5rgH_iIsIMav2aXbm5_o_mmxlxO9GavC8G9ZWD9BbuLlKKMJQAXgGXlfOlWA1Ukr5Cr0UO5LZbeTtVX3dZMYs3QfMwlGkVLT7Gam6aZfW8JduWhObFP9ghg2haF4tooXcUFNOPX7e2Axto-OmYAXLaPZRh-xpSzEObOeQv8gAHihoXlkA5Y_A2m45-emjdSiwff2Ui0VRZs96_DMGO",
"size": 17466
}
}
]
},
{
"partId": "1",
"mimeType": "image/png",
"filename": "chat.png",
"headers": [
{
"name": "Content-Type",
"value": "image/png; name=\"chat.png\""
},
{
"name": "Content-Disposition",
"value": "attachment; filename=\"chat.png\""
},
{
"name": "Content-Transfer-Encoding",
"value": "base64"
},
{
"name": "Content-ID",
"value": "<f_m3y6xx8e1>"
},
{
"name": "X-Attachment-Id",
"value": "f_m3y6xx8e1"
}
],
"body": {
"attachmentId": "ANGjdJ-XMpv18VYpIKT1zc45Su99KXSI3r3YaVquSLfKE3DqgQ7XjO7p4iHRHvc47lIr70rLBk2H8lLlMifTCFxFLMy0q_uFgsCHASCDeq-opjPkeDxC3BaxACeyeNWaZYANXgUCFYqCygTLkIoeggJDBxmzPn99baxvoJqcRLXROO1mSWGB4K0PLJcP35r8Y2D9lVRjSAJupzf7HLYLzimY1Q-X6Ge2qNCNnree-rS47FNXQgvFfVdV_x7jcvLE7JJl5cBBIyus-TtTulMk0RrFbcn5zKH7AniW6IF7v-mLOUzpDVye9tIiVqc6W4EdLeP3wmIaMVy4shsAdHkKtbjwLxicycke8Vi5hAV50s0pCllN26rSHakAA9zswG9q5z3cbS5LNMdEUpgWt6ON",
"size": 17466
}
}
]
},
"sizeEstimate": 55270,
"historyId": "528587",
"internalDate": "1732609494000"
}
utils_v2/goog/controllers/gmail/json/messages/raw_message_03_19365d0239aa1aaf_20241126.json
-175
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_04_193640e74d93dc9b_20241126.json
-179
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_05_19363d6a7f50225a_20241126.json
-183
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_06_19363ba8e2582133_20241126.json
-183
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_07_19362f9983e58ea6_20241126.json
-210
View File
@@ -1,210 +0,0 @@
{
"id": "19362f9983e58ea6",
"threadId": "19362f9983e58ea6",
"labelIds": [
"CATEGORY_PROMOTIONS",
"UNREAD",
"INBOX"
],
"snippet": "INDIAN FILM &amp; TELEVISION DIRECTORS&#39; ASSOCIATION G- 8/9/10, Crescent Towers, Near Morya House, Andheri (W), Mumbai - 400 053 Tel: 022 46088096 Mob: 9892885346/ 7021494476 www.directorsiftda.com",
"payload": {
"partId": "",
"mimeType": "multipart/related",
"filename": "",
"headers": [
{
"name": "Delivered-To",
"value": "pskhushal@gmail.com"
},
{
"name": "Received",
"value": "by 2002:a5d:4205:0:b0:382:44a8:2102 with SMTP id n5csp832066wrq; Mon, 25 Nov 2024 02:59:05 -0800 (PST)"
},
{
"name": "X-Forwarded-Encrypted",
"value": "i=2; AJvYcCWTJ16JRatSfV54WvhqvY7z3mW9LUmKJsBYYVJp5gas68X0OXWAugWTJyyEQE0ESerQAquXl7ZIugA=@gmail.com"
},
{
"name": "X-Received",
"value": "by 2002:a17:906:3188:b0:a99:7bc0:bca9 with SMTP id a640c23a62f3a-aa50990b300mr1091084466b.3.1732532345051; Mon, 25 Nov 2024 02:59:05 -0800 (PST)"
},
{
"name": "ARC-Seal",
"value": "i=1; a=rsa-sha256; t=1732532345; cv=none; d=google.com; s=arc-20240605; b=cGPTlt9iJZomIZrEX1YxFZt7CeiQG3g/9XJ/DHe6g6ghGuBopYyGtWRIq/b0u5ezsK 3JyasTQwmCnqAmvhDEgUZQM/f2Bk8PshykVmLfSUzhP0vUS2wfg90N1jmaMTefDGFvlF lSjmHBlgpl4Vhn4BKGBnLuN3ttrMXvyZ6Vi4xGil9yDgW+KjwrL3fN2jdlFsOvhxODZR EE7q5SC0waIenSAllTd+0CcG4TrH7Q0ZrQYR0yVH98JZ4UJN6v5xqhwBmTMtlPx3mWw2 d8H5HXwevTXeYzYf/F7iKAlaaWu2IPM7k/e3HAatqQk/pKHfh6cmYqN1G/MzENERvOsk HmUQ=="
},
{
"name": "ARC-Message-Signature",
"value": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605; h=to:subject:message-id:date:from:mime-version:dkim-signature; bh=52sadrERSC8TqD31OwCOGWYBr43KUD3rPyq2rw8jz2Y=; fh=3tsGDR7RWsLBUvHMNbctop2/zoZmFmKl/B1BkNJ2jsE=; b=N9JHVDUsSOQO9HPax1y++VDKEbumii3FfXCPi2ec+NDD0lV4w+48dqXsrPO9k8XO03 FgGp+qnSgZps8TogBykk8a/bFPO4VJ0AZgt5b7v4i08Y+auiO+ywcP37yeZjB52iGVSU U0TcBLTtqQSBtlirHN4lG5t4CvNTgDhKDwr9UjvOqtLgValnrsySlSpoQQ+CRhaxC3Pq bEVn4CKh8dGlJWVvOA037UDZFBQtR6ygY7IbiXBEIFZsC49aH4lZbpyzA+Xh3H3X0Xtg LrM/0hxRNj33PLOvbm5gdS7F32djBbaoIXRl057QZV0JQbPa1fvEeDOwJCv8H6rECfHK m0UQ==; dara=google.com"
},
{
"name": "ARC-Authentication-Results",
"value": "i=1; mx.google.com; dkim=pass header.i=@directorsiftda-com.20230601.gappssmtp.com header.s=20230601 header.b=h3FDhhUH; spf=fail (google.com: domain of contact@directorsiftda.com does not designate 209.85.220.41 as permitted sender) smtp.mailfrom=contact@directorsiftda.com; dara=pass header.i=@gmail.com"
},
{
"name": "Return-Path",
"value": "<contact@directorsiftda.com>"
},
{
"name": "Received",
"value": "from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41]) by mx.google.com with SMTPS id a640c23a62f3a-aa560bdd42csor237566b.0.2024.11.25.02.59.04 for <pskhushal@gmail.com> (Google Transport Security); Mon, 25 Nov 2024 02:59:04 -0800 (PST)"
},
{
"name": "Received-SPF",
"value": "fail (google.com: domain of contact@directorsiftda.com does not designate 209.85.220.41 as permitted sender) client-ip=209.85.220.41;"
},
{
"name": "Authentication-Results",
"value": "mx.google.com; dkim=pass header.i=@directorsiftda-com.20230601.gappssmtp.com header.s=20230601 header.b=h3FDhhUH; spf=fail (google.com: domain of contact@directorsiftda.com does not designate 209.85.220.41 as permitted sender) smtp.mailfrom=contact@directorsiftda.com; dara=pass header.i=@gmail.com"
},
{
"name": "DKIM-Signature",
"value": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=directorsiftda-com.20230601.gappssmtp.com; s=20230601; t=1732532344; x=1733137144; dara=google.com; h=to:subject:message-id:date:from:mime-version:from:to:cc:subject :date:message-id:reply-to; bh=52sadrERSC8TqD31OwCOGWYBr43KUD3rPyq2rw8jz2Y=; b=h3FDhhUHelnzSe5+as+rP+HnwiQlFmbqO/XDImaXOHroIQfkSRt1MAwH2o8MiNS07l fzAKjxqRmO+p+fy9FyRuQ7d0/TDALivdP4HW1CpU7nHUqqp8BamVDC06zS1RI2PCVo7s qp4cxTIJsFTVaMauaTWQCIj9xstz9lGMa7DuhyJ8deSJU6zIAn0rFOhLISFZfZkVtZEy Q4J4MBtf2DZVvaxqUNr/DqkaownkPsTWaxfXTE8Ztq41a9kb+2IfX+6kk6M80DF4pm8I 5Wju6mVd6xpms0UXJPtRPfRBFKEjEApYq6dypd37ETr+tkUynPrv10xfuSUvrZdzU73L 1uEQ=="
},
{
"name": "X-Google-DKIM-Signature",
"value": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1732532344; x=1733137144; h=to:subject:message-id:date:from:mime-version:x-gm-message-state :from:to:cc:subject:date:message-id:reply-to; bh=52sadrERSC8TqD31OwCOGWYBr43KUD3rPyq2rw8jz2Y=; b=tQU+3IcgiGI90Bknb75WOMN2IpL9BIunrIP5KwfkJMvrRTWq9GzjUl0t6gUocRNiPn 1fJ2VgBd/nAvzUpg/2lcypTBbfp7019CxuW5aqungvCD5iWFzMUcOoudN0O4kMx0KrMX DFThj9pD951evE/jteODFBQt0886+8Ofo8KIWaCsSiJ7uerf968pWkkJm6lkHsMCFf97 wgiA4kN8X8i9d+ZFs2yo5MxZykY1Ll2YYmMozgIS6jsXLb0pveLtA1L0SFxbqRU+/Njf n6ibGo42xOWOM92qDEbJnDXNtlnOEVOAKA3sgazkG8pDpC9yf2RqMPWex9xCNAbbVNQE KTog=="
},
{
"name": "X-Forwarded-Encrypted",
"value": "i=1; AJvYcCWtfq6Mi9XsVN+r9oOb64yoh/F6ROqg/9C0UMWDuXU/fz1A74BwcwwZQy0hyIykg2dwtgxUFBLIz0w=@gmail.com"
},
{
"name": "X-Gm-Message-State",
"value": "AOJu0YxUy7pf/My+et+YkWa4bqQ26C2PeklIXRKdME1VkRVdK9A8dtmN p34rPukc6VEwoEvOLsIh42JHO+nLEOYgcdIMTfklpdIp+2SaamAG+j/zSQFpfdFhWov4PO2vbqx aOztICe4xCfaWEiv4jKQ3arZkHmUWwrT6xb1sOQ=="
},
{
"name": "X-Gm-Gg",
"value": "ASbGncsXwRmXGJJAyTvAKY+pMS2y7/Mf8mmUI9AmHaycDBvZnbQkc20j125cXkjxmOG o3M/KYTcTZDKyHLOi5xRqMfOv1Qs8lwFelQ=="
},
{
"name": "X-Google-Smtp-Source",
"value": "AGHT+IFopa4irNtf4tkqOS0N6Tc13BHGQQ6fNDsmqoW9klWPS/JD7cYNG9eMDQkP9k7DPZCzWaTq4bPYSLxo6IJY2OU="
},
{
"name": "X-Received",
"value": "by 2002:a17:906:1daa:b0:aa5:3853:5532 with SMTP id a640c23a62f3a-aa53853589dmr630025266b.43.1732532343546; Mon, 25 Nov 2024 02:59:03 -0800 (PST)"
},
{
"name": "MIME-Version",
"value": "1.0"
},
{
"name": "From",
"value": "Iftda India <contact@directorsiftda.com>"
},
{
"name": "Date",
"value": "Mon, 25 Nov 2024 16:28:50 +0530"
},
{
"name": "Message-ID",
"value": "<CAAPQQGQPeA-z38EPhBSyMPmhyuWzc2uRyibVMg6uLd5V8hr5VQ@mail.gmail.com>"
},
{
"name": "Subject",
"value": "Sad Demise of Mr. Jalaj Dhir"
},
{
"name": "To",
"value": "undisclosed-recipients:;"
},
{
"name": "Content-Type",
"value": "multipart/related; boundary=\"000000000000e67bea0627ba9e63\""
},
{
"name": "Bcc",
"value": "pskhushal@gmail.com"
}
],
"body": {
"size": 0
},
"parts": [
{
"partId": "0",
"mimeType": "multipart/alternative",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "multipart/alternative; boundary=\"000000000000e67be90627ba9e62\""
}
],
"body": {
"size": 0
},
"parts": [
{
"partId": "0.0",
"mimeType": "text/plain",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "text/plain; charset=\"UTF-8\""
}
],
"body": {
"size": 672,
"data": "W2ltYWdlOiBjb25kb2xlbmNlcyBNci4gSkFMQUogREhJUi5qcGddDQoNCg0KKklORElBTioqIEZJTE0gJiBURUxFVklTSU9OIERJUkVDVE9SUycgQVNTT0NJQVRJT04qDQpHLSA4LzkvMTAsIENyZXNjZW50IFRvd2VycywNCk5lYXIgTW9yeWEgSG91c2UsDQpBbmRoZXJpIChXKSwgTXVtYmFpIC0gNDAwIDA1Mw0KVGVsOiAwMjIgNDYwODgwOTYNCk1vYjogOTg5Mjg4NTM0Ni8gNzAyMTQ5NDQ3Ng0Kd3d3LmRpcmVjdG9yc2lmdGRhLmNvbSB8IEZhY2Vib29rL2RpcmVjdG9yc2lmdGRhDQoNCipESVNDTEFJTUVSOiogVGhpcyBlLW1haWwgbWF5IGJlIHByaXZpbGVnZWQgYW5kL29yIGNvbmZpZGVudGlhbCwgYW5kIHRoZQ0Kc2VuZGVyIGRvZXMgbm90IHdhaXZlIGFueSByZWxhdGVkIHJpZ2h0cyBhbmQgb2JsaWdhdGlvbnMuIEFueSBkaXN0cmlidXRpb24sDQp1c2Ugb3IgY29weWluZyBvZiB0aGlzIGUtbWFpbCBvciB0aGUgaW5mb3JtYXRpb24gaXQgY29udGFpbnMgYnkgb3RoZXIgdGhhbg0KYW4gaW50ZW5kZWQgcmVjaXBpZW50KHMpIGlzIHVuYXV0aG9yaXplZC4gSWYgeW91IHJlY2VpdmVkIHRoaXMgZS1tYWlsIGluDQplcnJvciwgcGxlYXNlIGFkdmlzZSB1cyAoYnkgcmV0dXJuIGUtbWFpbCBvciBvdGhlcndpc2UpIGltbWVkaWF0ZWx5IGFuZA0KZGVsZXRlIHRoaXMgZS1tYWlsLg0K"
}
},
{
"partId": "0.1",
"mimeType": "text/html",
"filename": "",
"headers": [
{
"name": "Content-Type",
"value": "text/html; charset=\"UTF-8\""
},
{
"name": "Content-Transfer-Encoding",
"value": "quoted-printable"
}
],
"body": {
"size": 2187,
"data": "PGRpdiBkaXI9Imx0ciI-PGRpdj48ZGl2IGNsYXNzPSJnbWFpbF9kZWZhdWx0IiBzdHlsZT0iZm9udC1mYW1pbHk6JnF1b3Q7dHJlYnVjaGV0IG1zJnF1b3Q7LHNhbnMtc2VyaWY7Y29sb3I6cmdiKDY4LDY4LDY4KSI-PC9kaXY-PGltZyBzcmM9ImNpZDppaV9tM3d4MGo5YzAiIGFsdD0iY29uZG9sZW5jZXMgTXIuIEpBTEFKIERISVIuanBnIiB3aWR0aD0iNDcyIiBoZWlnaHQ9IjMzNCI-PGJyPjxiciBjbGVhcj0iYWxsIj48L2Rpdj48ZGl2PjxkaXYgZGlyPSJsdHIiIGNsYXNzPSJnbWFpbF9zaWduYXR1cmUiIGRhdGEtc21hcnRtYWlsPSJnbWFpbF9zaWduYXR1cmUiPjxkaXYgZGlyPSJsdHIiPjxkaXY-PGRpdiBkaXI9Imx0ciI-PGRpdj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2IGRpcj0ibHRyIj48ZGl2PjxzcGFuPjxzcGFuIHN0eWxlPSJjb2xvcjpyZ2IoNjgsNjgsNjgpIj48c3Bhbj48Yj48Zm9udCBzaXplPSI0Ij48YnI-PC9mb250PjwvYj48L3NwYW4-PC9zcGFuPjwvc3Bhbj48ZGl2Pjxmb250IGZhY2U9InRhaG9tYSwgc2Fucy1zZXJpZiI-PGZvbnQ-PGI-SU5ESUFOPC9iPjwvZm9udD48Yj7CoEZJTE0gJmFtcDsgVEVMRVZJU0lPTiBESVJFQ1RPUlMmIzM5OyBBU1NPQ0lBVElPTjwvYj48L2ZvbnQ-PC9kaXY-PGRpdj48Zm9udCBmYWNlPSJ0YWhvbWEsIHNhbnMtc2VyaWYiPkctIDgvOS8xMCwgQ3Jlc2NlbnQgVG93ZXJzLMKgPGJyPjwvZm9udD48L2Rpdj48ZGl2IGRpcj0ibHRyIj48ZGl2Pjxmb250IGZhY2U9InRhaG9tYSwgc2Fucy1zZXJpZiI-TmVhciBNb3J5YSBIb3VzZSw8L2ZvbnQ-PC9kaXY-PGRpdj48Zm9udCBmYWNlPSJ0YWhvbWEsIHNhbnMtc2VyaWYiPkFuZGhlcmkgKFcpLCBNdW1iYWkgLSA0MDAgMDUzPC9mb250PjwvZGl2PjxkaXY-PHNwYW4gc3R5bGU9InRleHQtYWxpZ246Y2VudGVyIj5UZWw6wqA8L3NwYW4-PHNwYW4gc3R5bGU9InRleHQtYWxpZ246Y2VudGVyIj4wMjIgNDYwODgwOTY8L3NwYW4-PGZvbnQgZmFjZT0idGFob21hLCBzYW5zLXNlcmlmIj48YnI-PC9mb250PjwvZGl2PjxkaXY-PGZvbnQgZmFjZT0idGFob21hLCBzYW5zLXNlcmlmIj5Nb2I6IDxzcGFuIHN0eWxlPSJ0ZXh0LWFsaWduOmNlbnRlciI-OTg5Mjg4NTM0Ni8gNzAyMTQ5NDQ3Njwvc3Bhbj48L2ZvbnQ-PC9kaXY-PC9kaXY-PC9kaXY-PGRpdj48ZGl2IGRpcj0ibHRyIj48ZGl2PjxzcGFuIHN0eWxlPSJjb2xvcjpyZ2IoNjgsNjgsNjgpIj48Zm9udCBmYWNlPSJ0YWhvbWEsIHNhbnMtc2VyaWYiPjxmb250IHNpemU9IjIiPnd3dy5kaXJlY3RvcnNpZnRkYTwvZm9udD4uY29tIHwgRmFjZWJvb2svZGlyZWN0b3JzaWZ0ZGE8L2ZvbnQ-PGJyPjxicj48c3BhbiBzdHlsZT0iZm9udC1zaXplOjEyLjhweCI-PGI-PGk-PHU-PHNwYW4gc3R5bGU9ImZvbnQtc2l6ZTo4cHQiIGxhbmc9IkVOLVVTIj5ESVNDTEFJTUVSOjwvc3Bhbj48L3U-PC9pPjwvYj48c3BhbiBzdHlsZT0iZm9udC1zaXplOjhwdCI-PC9zcGFuPjxzcGFuIHN0eWxlPSJmb250LXNpemU6OHB0O2JhY2tncm91bmQ6d2hpdGUgbm9uZSByZXBlYXQgc2Nyb2xsIDAlIDAlIj4gVGhpcw0KIGUtbWFpbCBtYXkgYmUgcHJpdmlsZWdlZCBhbmQvb3IgY29uZmlkZW50aWFsLCBhbmQgdGhlIHNlbmRlciBkb2VzIG5vdCANCndhaXZlIGFueSByZWxhdGVkIHJpZ2h0cyBhbmQgb2JsaWdhdGlvbnMuIEFueSBkaXN0cmlidXRpb24sIHVzZSBvciANCmNvcHlpbmcgb2YgdGhpcyBlLW1haWwgb3IgdGhlIGluZm9ybWF0aW9uIGl0IGNvbnRhaW5zIGJ5IG90aGVyIHRoYW4gYW4gDQppbnRlbmRlZCByZWNpcGllbnQocykgaXMgdW5hdXRob3JpemVkLiBJZiB5b3UgcmVjZWl2ZWQgdGhpcyBlLW1haWwgaW4gDQplcnJvciwgcGxlYXNlIGFkdmlzZSB1cyAoYnkgcmV0dXJuIGUtbWFpbCBvciBvdGhlcndpc2UpIGltbWVkaWF0ZWx5IGFuZCANCmRlbGV0ZSB0aGlzIGUtbWFpbC48L3NwYW4-PHNwYW4gc3R5bGU9ImZvbnQtc2l6ZTo4cHQiPjwvc3Bhbj48L3NwYW4-PC9zcGFuPjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2PjwvZGl2Pg0K"
}
}
]
},
{
"partId": "1",
"mimeType": "image/jpeg",
"filename": "condolences Mr. JALAJ DHIR.jpg",
"headers": [
{
"name": "Content-Type",
"value": "image/jpeg; name=\"condolences Mr. JALAJ DHIR.jpg\""
},
{
"name": "Content-Disposition",
"value": "inline; filename=\"condolences Mr. JALAJ DHIR.jpg\""
},
{
"name": "Content-Transfer-Encoding",
"value": "base64"
},
{
"name": "Content-ID",
"value": "<ii_m3wx0j9c0>"
},
{
"name": "X-Attachment-Id",
"value": "ii_m3wx0j9c0"
}
],
"body": {
"attachmentId": "ANGjdJ8zCv8y-pC8ecFP7IVoYsW4sTkayvcSepjUwiCDIrSUTJPBua7Ack8YvSuFPYcM-np_PeWRTl_wq5dYpyz3k6Hc5e-tp4RZjT6t2ITBDpkqJhPn17eR36sVN41ojkJ63Wzd9HD60LDk3o4GiL8C6iyMlsHMhadvOXKYK5zITlU9mvtdMoye6fofiiLwRsoNuT9dEpBbk9r2gh-vmuYpbu0waybh_EKefEejDvuOl7cIFU1Y3Dv8kxWYQwtZ1kx1LhHF24TzisZjvBaF6cno1T6f9aTdNd3x5T0_YsgJNilfU3xdWbCYilbVACU8axpIN5mC9CwhD45ogliA9kLadSpig8KmW9MNJJxr1vMnOAI4G-zr1SAhkzm6ZXcHllk9WpncS8RRRaLilLxo",
"size": 1736748
}
}
]
},
"sizeEstimate": 2385712,
"historyId": "527104",
"internalDate": "1732532330000"
}
utils_v2/goog/controllers/gmail/json/messages/raw_message_08_19360aabcb976cfc_20241126.json
-175
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_09_193604ad97c9fe62_20241126.json
-183
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/controllers/gmail/json/messages/raw_message_10_1935e92672e67f22_20241126.json
-183
View File
File diff suppressed because one or more lines are too long
utils_v2/goog/models/__init__.py
View File
utils_v2/goog/models/api_call.py
-132
View File
@@ -1,132 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Wednesday, 30th Oct., 2024.
OBJECTIVE:
To provide a data model for describing the API response from Google's APIs.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator, model_validator
from typing import Optional, Literal, Union, Dict, List, Any
# My utils:
from utils_v2.string import json
from utils_v2.string import regex
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class GoogleApiResponse(BaseModel):
serviceName: str = Field(frozen = True, default = None)
action: str = Field(frozen = True, default = None)
url: str = Field(frozen = True)
method: str = Field(frozen = True)
response: Any = None
httpCode: int = None
success: bool = False
message: str = None
data: Any = None
exception: Any = None
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "forbid"
# ┏┓ ┏┓
# ┃ ┓┏┏╋┏┓┏┳┓ ┣ ┓┏┏┓┏┏
# ┗┛┗┻┛┗┗┛┛┗┗ ┻ ┗┻┛┗┗┛
def to_markdown(self):
if self.exception: message = "❌ *GOOGLE API EXCEPTION:* ❌\n\n"
else: message = "*GOOGLE API RESPONSE:*\n\n"
message += f"*SERVICE:*\n`{self.serviceName}`\n\n"
message += f"*ACTION:*\n`{self.action}`\n\n"
message += f"*URL:*\n`{self.url}`\n\n"
message += f"*METHOD:*\n`{self.method}`\n\n"
message += f"*RESPONSE:*\n`{self.response}`\n\n"
message += f"*MESSAGE:*\n`{self.message}`\n\n"
message += f"*EXCEPTION:*\n`{self.exception.__class__.__name__}: {str(self.exception)}`\n\n"
return message
async def get_json(self):
try: return self.response.json()
except: return {}
async def get_content(self):
try: return self.response.content
except: return b""
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/goog/models/auth_tokens.py
-243
View File
@@ -1,243 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Wednesday, 30th Oct., 2024.
OBJECTIVE:
To provide a data model for describing the tokens to be used for Google's APIs.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For making data behaviour_models:
from pydantic import BaseModel, Field, field_validator, model_validator, AwareDatetime
from typing import Optional, Literal, Union, Dict, List, Any
# Related to Google:
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
# My utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with date and time:
import datetime
import dateparser
# To make API calls:
import httpx
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class GoogleAuthTokens(BaseModel):
accessToken: str = Field(description = "the main 'bearer' token")
refreshToken: str = Field(description = "token to be used to refresh the access token")
expiresAt: AwareDatetime = Field(description = "the time (utc) at which the token will expire")
scopes: List[str] = Field(description = "the list of permissions", default = [])
email: str | None = Field(description = "the email id of the user", default = None)
displayName: str | None = Field(description = "the display name of the user", default = None)
displayPictureUrl: str | None = Field(description = "the url to the display picture of the user", default = None)
# ┏┓ ┏•
# ┃ ┏┓┏┓╋┓┏┓
# ┗┛┗┛┛┗┛┗┗┫
# ┛
class Config:
extra = "allow"
# ┓┏ ┓• ┓ •
# ┃┃┏┓┃┓┏┫┏┓╋┓┏┓┏┓
# ┗┛┗┻┗┗┗┻┗┻┗┗┗┛┛┗
@field_validator("expiresAt", mode = "before")
def parse_dates(cls, value):
if not isinstance(value, datetime.datetime):
parsed = date_time.parse_date_time(value, date_formats = ["%Y%m%d", "%Y-%m-%d"])
value = parsed if isinstance(parsed, datetime.datetime) else dateparser.parse(value)
if isinstance(value, datetime.datetime): value = date_time.to_timezone(value, date_time.TIMEZONE_UTC)
return value
# ┏┓ •
# ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
# ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
# ┛
@property
def client_user_id(self):
return {"email": self.email}
@property
def expired(self):
return True if date_time.get_current_utc_date_time() >= self.expiresAt else False
@property
def ttl(self):
return (self.expiresAt - date_time.get_current_utc_date_time()).total_seconds()
# ┏┓ ┏┓
# ┃ ┓┏┏╋┏┓┏┳┓ ┣ ┓┏┏┓┏┏
# ┗┛┗┻┛┗┗┛┛┗┗ ┻ ┗┻┛┗┗┛
def refresh(
self,
client_id: str,
client_secret: str,
force_refresh: bool = False
) -> bool:
"""
Synchronously (blocking) refreshes the existing access tokens in place.
:param client_id: The id of the client app (OAuth JSON) for which these tokens were granted.
:param client_secret: The secret of the client app (OAuth JSON) for which these tokens were granted.
:param force_refresh: Whether you want to refresh the token even if it hasn't expired.
:return: True if refreshed, else False.
"""
# Start by assuming failure:
success = False
try:
# Go ahead only if either the token has expired,
# or the user has asked to forcefully refresh the tokens:
if self.expired or force_refresh:
# Create the credentials:
credentials = Credentials.from_authorized_user_info(
info = {
"client_id": client_id,
"client_secret": client_secret,
"refresh_token": self.refreshToken,
"expires_at": self.expiresAt
}
)
# Request a refresh:
credentials.refresh(Request())
# Note down the new credentials:
if credentials.token != self.accessToken:
success = True
self.accessToken = credentials.token
self.refreshToken = credentials.refresh_token
self.expiresAt = date_time.as_if_timezone(credentials.expiry, timezone = date_time.TIMEZONE_UTC)
# In case something goes wrong:
except Exception as exception:
success = False
# Done here:
return success
async def arefresh(
self,
http_client: httpx.AsyncClient,
client_id: str,
client_secret: str,
force_refresh: bool = False
) -> bool:
"""
Asynchronously refreshes the existing access tokens in place.
:param http_client: The HTTP client to use to make the refresh request.
:param client_id: The id of the client (OAuth JSON) for which these tokens were granted.
:param client_secret: The secret of the client (OAuth JSON) for which these tokens were granted.
:param force_refresh: Whether you want to refresh the token even if it hasn't expired.
:return: True if refreshed, else False.
"""
# Currently we don't really know how to refresh tokens through low-level API calls,
# so we will pass on the intent to the regular, synchronous function.
return self.refresh(
client_id = client_id,
client_secret = client_secret,
force_refresh = force_refresh
)
def has_scopes(self, scopes: List[str]) -> bool:
"""
Checks if all the specified scopes were granted.
:param scopes: The list of scopes to check. These are the permissions you need.
:return: True if all specified scoped are present, else False.
"""
# Start by assuming success:
has_scopes = True
# Now loop through the needed scopes and check:
for scope in scopes:
if scope not in self.scopes:
has_scopes = False
break
# Done here:
return has_scopes
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/image/__init__.py
View File
utils_v2/image/scanner/__init__.py
View File
utils_v2/image/scanner/scanner.py
-410
View File
@@ -1,410 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 19th Sept., 2024
OBJECTIVE:
To provide a way to easily scan documents like printouts and visiting cards.
REFERENCES:
01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085
02. https://github.com/JaidedAI/EasyOCR
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
from diffusers.utils.import_utils import candidates
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# To work with image processing:
import cv2
import imutils
import numpy as np
# To run OCR:
import easyocr
# To download images from the web:
import requests
# My utils:
from utils_v2.string import json
from utils_v2.ai.object_detection.yolo import YoloDetect
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class DocumentScanner:
def __init__(
self,
layout_detection_yolo = None,
whitelisted_yolo_classes = None,
ocr_languages = None
):
self.__yolo = YoloDetect(
model_file = layout_detection_yolo,
debug = False
)
self.__whitelisted_yolo_classes = whitelisted_yolo_classes or []
self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"])
pass
@staticmethod
def open_image(image):
"""
Opens an image from various sources.
:param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
:return: The image as a CV2 object.
"""
# When the image is provided as a string,
# it could be either as a URL or a path to a local file:
if isinstance(image, str):
if image.startswith("https://") or image.startswith("http://"):
image = io.BytesIO(requests.get(image).content)
else: image = cv2.imread(image)
# If the image is provided as a io.BytesIO buffer:
if isinstance(image, io.BytesIO):
image.seek(0)
image = np.asarray(bytearray(image.read()), dtype = np.uint8)
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
# Done here:
return image
@staticmethod
def polygon_area(points):
"""
Computes the area occupied by a shape described by the array of points.
Example input: [[x1, y1], [x2, y2], [x3, y3], [x4, y4]]
:return: The area of the shape.
"""
n = len(points)
area = 0.0
for i in range(n):
j = (i + 1) % n # next vertex
area += points[i][0] * points[j][1]
area -= points[j][0] * points[i][1]
return abs(area) / 2.0
@staticmethod
def show(image, title = "Preview", wait = True):
"""
Just a quick wrapper to show the image in a window.
:param image: The image that you want to show.
:param title: The title of the window.
:param wait: Set this to True when you want to show the window(s). This is useful when you want to show many
windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the
very last one.
:return: None.
"""
cv2.imshow(title, image)
if wait:
cv2.waitKey()
cv2.destroyAllWindows()
@staticmethod
def enhance(image):
"""
Applies a form of contrast boost to make the edges more easily visible.
:param image: The OpenCV image that needs to be enhanced.
:return: The enhanced CV2 image.
"""
return cv2.detailEnhance(
src = image,
sigma_s = 20,
sigma_r = 0.15
)
@staticmethod
def get_edges(image):
"""
Gets the edges in an image.
:param image: A CV2 image.
:return: The image with the edges detected.
"""
# Denoise the image:
processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
processed_image = cv2.GaussianBlur(
processed_image,
ksize = (5, 5),
sigmaX = 0
)
# Detect edges in the image:
processed_image = cv2.Canny(
processed_image,
threshold1 = 50,
threshold2 = 200
)
# Close small gaps in the edges:
kernel = np.ones((5, 5), np.uint8)
processed_image = cv2.dilate(processed_image, kernel, iterations = 1)
# processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel)
# processed_image = cv2.erode(processed_image, kernel, iterations = 1)
# Done here:
return processed_image
def scan(self, image, min_area = 0.125):
"""
Looks for a rectangle in an image and flattens it out. No colour filters are applied here.
:param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
:param min_area: The minimum area occupied by the document in the image.
:return: The flattened image if any, else None.
"""
# Open the image and read the data:
image = self.open_image(image)
if image is None: return None
height, width = image.shape[:2]
image_area = height * width
# Enhance the image to have better visibility of edges:
# processed_image = self.enhance(image.copy())
# Detect edges in the image:
processed_image = self.get_edges(image.copy())
# Find the contours in the edges,
# and sort them in ascending order:
contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
contours = imutils.grab_contours(contours)
contours = sorted(contours, key = cv2.contourArea, reverse = True)
# We try to get clean and passable contours.
# For our purposes, 'clean' is when it can be approximated to exactly 4 sides,
# and 'passable' is when the approximation comes close to it:
clean_rects = []
passable_rects = []
for contour in contours:
perimeter = cv2.arcLength(contour, closed = True)
approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True)
if len(approximation) == 4: clean_rects.append(approximation)
elif len(approximation) <= 32:
candidate_contour = cv2.minAreaRect(approximation)
candidate_contour = cv2.boxPoints(candidate_contour)
candidate_contour = np.intp(candidate_contour)
passable_rects.append(candidate_contour)
clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True)
passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True)
# Now we decide between the best candidate:
document_outline = None
if len(clean_rects) > 0:
candidate_rect = np.array([p[0] for p in clean_rects[0]])
rectangle_area = cv2.contourArea(candidate_rect)
if rectangle_area / image_area >= min_area: document_outline = candidate_rect
if document_outline is None and len(passable_rects) > 0:
candidate_rect = cv2.minAreaRect(passable_rects[0])
candidate_rect = cv2.boxPoints(candidate_rect)
candidate_rect = np.intp(candidate_rect)
rectangle_area = cv2.contourArea(candidate_rect)
if rectangle_area / image_area >= min_area: document_outline = candidate_rect
# If there is no rectangular contour,
# we exit with failure:
if document_outline is None: return None
# Since we did get the best candidate for the document,
# we figure out which point n the contour is which corner of the document:
document_rectangle = np.zeros((4, 2), dtype = "float32")
sum_points = document_outline.sum(axis = 1)
document_rectangle[0] = document_outline[np.argmin(sum_points)]
document_rectangle[2] = document_outline[np.argmax(sum_points)]
diff_points = np.diff(document_outline, axis = 1)
document_rectangle[1] = document_outline[np.argmin(diff_points)]
document_rectangle[3] = document_outline[np.argmax(diff_points)]
(top_left, top_right, bottom_right, bottom_left) = document_rectangle
# let's compute the dimensions of the document in the image:
left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
max_height = max(int(left_height), int(right_height))
max_width = max(int(top_width), int(bottom_width))
# We compute the destination of the transform:
destination = np.array(
object = [
[0, 0],
[max_width - 1, 0],
[max_width - 1, max_height - 1],
[0, max_height - 1]
],
dtype = "float32"
)
# We apply the transform that flattens out the document:
matrix = cv2.getPerspectiveTransform(document_rectangle, destination)
flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height))
# Done here:
return flattened_image
def extract_text(
self,
image,
min_confidence = 0.5,
margin = 0.05
):
"""
To run OCR on the input image.
:param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
:param min_confidence: The minimum amount of confidence for detected text to be considered.
:param margin: The extra pixels to include when cropping into a section for OCR.
:return: The extracted text.
"""
# Make a variable that will hold the final result:
extracted_text = {}
# Open the image and read the data:
image = self.open_image(image)
height, width = image.shape[:2]
# Run OCR on the entire canvas:
ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence]
extracted_text["fullDoc"] = " ".join(ocr_result)
# Get the doc layout in the image:
document_sections = self.__yolo.predict(image)
document_sections = document_sections["boxes"]
# For each section, we run the OCR process individually:
section_results = []
for section in document_sections:
if section["class"] in self.__whitelisted_yolo_classes:
section_width = section["x2"] - section["x1"]
section_height = section["y2"] - section["y1"]
x1 = max(int(section["x1"] - (margin * section_width)), 0)
y1 = max(int(section["y1"] - (margin * section_height)), 0)
x2 = min(int(section["x2"] + (margin * section_width)), width)
y2 = min(int(section["y2"] + (margin * section_height)), height)
sub_image = image[y1:y2, x1:x2]
ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence]
section_results.append({
"text": " ".join(ocr_result),
"class": section["class"],
"className": section["className"],
"x1": x1,
"y1": y1,
"x2": x2,
"y2": y2
})
extracted_text["bySection"] = section_results
# Done here:
return extracted_text
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import time
my_scanner = DocumentScanner(
layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
# whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
ocr_languages = ["en"]
)
image = my_scanner.open_image(r"/home/developer/Downloads/talkaholics_card.jpg")
document_image = my_scanner.scan(image)
if document_image is not None:
cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image)
start_time = time.time()
document_text = my_scanner.extract_text(
document_image,
min_confidence = 0.5
)
print("OCR RESULT:")
print(document_text)
print(f"FINISHED IN {time.time() - start_time} SECONDS!")
else: print("No image")
utils_v2/image/scanner/scanner_v2.py
-379
View File
@@ -1,379 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 24th Sept., 2024
OBJECTIVE:
To provide a way to easily scan documents like printouts and visiting cards.
REFERENCES:
01. https://medium.com/@victorolufemi/build-a-document-scanner-with-opencv-ff9f645a4085
02. https://github.com/JaidedAI/
03. https://github.com/criistian14/flutter_document_scanner/blob/master/flutter_document_scanner_android/android/src/main/kotlin/com/christian/flutterDocumentScanner/OpenCVPlugin.kt
04. https://www.geeksforgeeks.org/python-bilateral-filtering/
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# To work with image processing:
import cv2
import imutils
import numpy as np
# To run OCR:
import easyocr
# To download images from the web:
import requests
# My utils:
from utils_v2.string import json
from utils_v2.ai.object_detection.yolo import YoloDetect
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class DocumentScanner:
def __init__(
self,
layout_detection_yolo = None,
whitelisted_yolo_classes = None,
ocr_languages = None
):
self.__yolo = YoloDetect(
model_file = layout_detection_yolo,
debug = False
)
self.__whitelisted_yolo_classes = whitelisted_yolo_classes or []
self.__ocr_engine = easyocr.Reader(ocr_languages or ["en"])
pass
@staticmethod
def open_image(image):
"""
Opens an image from various sources.
:param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
:return: The image as a CV2 object.
"""
# When the image is provided as a string,
# it could be either as a URL or a path to a local file:
if isinstance(image, str):
if image.startswith("https://") or image.startswith("http://"):
image = io.BytesIO(requests.get(image).content)
else: image = cv2.imread(image)
# If the image is provided as a io.BytesIO buffer:
if isinstance(image, io.BytesIO):
image.seek(0)
image = np.asarray(bytearray(image.read()), dtype = np.uint8)
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
# Done here:
return image
@staticmethod
def show(image, title = "Preview", wait = True):
"""
Just a quick wrapper to show the image in a window.
:param image: The image that you want to show.
:param title: The title of the window.
:param wait: Set this to True when you want to show the window(s). This is useful when you want to show many
windows at once. Suppose you want to show a lot of windows, you set this to False for all calls except the
very last one.
:return: None.
"""
cv2.imshow(title, image)
if wait:
cv2.waitKey()
cv2.destroyAllWindows()
@staticmethod
def get_edges(image):
"""
Apply all the preprocessing filters on the image before sending it off for contour-finding.
:param image: The image to pre-process.
:return: The pre-processed image.
"""
# Denoise the image:
processed_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
processed_image = cv2.bilateralFilter(
processed_image,
d = 9,
sigmaColor = 75,
sigmaSpace = 75
)
# Detect edges in the image:
processed_image = cv2.Canny(
processed_image,
threshold1 = 75,
threshold2 = 200
)
# Close small gaps in the edges:
kernel = np.ones((5, 5), np.uint8)
processed_image = cv2.dilate(processed_image, kernel, iterations = 1)
# processed_image = cv2.morphologyEx(processed_image, cv2.MORPH_CLOSE, kernel)
# processed_image = cv2.erode(processed_image, kernel, iterations = 1)
# Done here:
return processed_image
def scan(self, image, min_area = 0.125):
"""
Looks for a rectangle in an image and flattens it out. No colour filters are applied here.
:param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
:param min_area: The minimum area occupied by the document in the image.
:return: The flattened image if any, else None.
"""
# Open the image and read the data:
image = self.open_image(image)
if image is None: return None
height, width = image.shape[:2]
image_area = height * width
# image = self.enhance(image)
processed_image = self.get_edges(image)
# self.show(processed_image)
# Find the contours in the edges,
# and sort them in ascending order:
contours = cv2.findContours(processed_image, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
contours = imutils.grab_contours(contours)
contours = sorted(contours, key = cv2.contourArea, reverse = True)
# We try to get clean and passable contours.
# For our purposes, 'clean' is when it can be approximated to exactly 4 sides,
# and 'passable' is when the approximation comes close to it:
clean_rects = []
passable_rects = []
for contour in contours:
perimeter = cv2.arcLength(contour, closed = True)
approximation = cv2.approxPolyDP(contour, 0.025 * perimeter, closed = True)
if len(approximation) == 4: clean_rects.append(approximation)
elif len(approximation) <= 32:
candidate_contour = cv2.minAreaRect(approximation)
candidate_contour = cv2.boxPoints(candidate_contour)
candidate_contour = np.intp(candidate_contour)
passable_rects.append(candidate_contour)
clean_rects = sorted(clean_rects, key = cv2.contourArea, reverse = True)
passable_rects = sorted(passable_rects, key = cv2.contourArea, reverse = True)
# Now we decide between the best candidate:
document_outline = None
if len(clean_rects) > 0:
candidate_rect = np.array([p[0] for p in clean_rects[0]])
rectangle_area = cv2.contourArea(candidate_rect)
if rectangle_area / image_area >= min_area: document_outline = candidate_rect
if document_outline is None and len(passable_rects) > 0:
candidate_rect = cv2.minAreaRect(passable_rects[0])
candidate_rect = cv2.boxPoints(candidate_rect)
candidate_rect = np.intp(candidate_rect)
rectangle_area = cv2.contourArea(candidate_rect)
if rectangle_area / image_area >= min_area: document_outline = candidate_rect
# If there is no rectangular contour,
# we exit with failure:
if document_outline is None: return None
# Since we did get the best candidate for the document,
# we figure out which point n the contour is which corner of the document:
document_rectangle = np.zeros((4, 2), dtype = "float32")
sum_points = document_outline.sum(axis = 1)
document_rectangle[0] = document_outline[np.argmin(sum_points)]
document_rectangle[2] = document_outline[np.argmax(sum_points)]
diff_points = np.diff(document_outline, axis = 1)
document_rectangle[1] = document_outline[np.argmin(diff_points)]
document_rectangle[3] = document_outline[np.argmax(diff_points)]
(top_left, top_right, bottom_right, bottom_left) = document_rectangle
# let's compute the dimensions of the document in the image:
left_height = np.sqrt(((top_left[0] - bottom_left[0]) ** 2) + ((top_left[1] - bottom_left[1]) ** 2))
right_height = np.sqrt(((top_right[0] - bottom_right[0]) ** 2) + ((top_right[1] - bottom_right[1]) ** 2))
top_width = np.sqrt(((top_right[0] - top_left[0]) ** 2) + ((top_right[1] - top_left[1]) ** 2))
bottom_width = np.sqrt(((bottom_right[0] - bottom_left[0]) ** 2) + ((bottom_right[1] - bottom_left[1]) ** 2))
max_height = max(int(left_height), int(right_height))
max_width = max(int(top_width), int(bottom_width))
# We compute the destination of the transform:
destination = np.array(
object = [
[0, 0],
[max_width - 1, 0],
[max_width - 1, max_height - 1],
[0, max_height - 1]
],
dtype = "float32"
)
# We apply the transform that flattens out the document:
matrix = cv2.getPerspectiveTransform(document_rectangle, destination)
flattened_image = cv2.warpPerspective(image, matrix, dsize = (max_width, max_height))
# Done here:
return flattened_image
def extract_text(
self,
image,
min_confidence = 0.5,
margin = 0.05
):
"""
To run OCR on the input image.
:param image: The image either as a path to a file on the local disk, or a URL, or an io.BytesIO buffer.
:param min_confidence: The minimum amount of confidence for detected text to be considered.
:param margin: The extra pixels to include when cropping into a section for OCR.
:return: The extracted text.
"""
# Make a variable that will hold the final result:
extracted_text = {}
# Open the image and read the data:
image = self.open_image(image)
height, width = image.shape[:2]
# Run OCR on the entire canvas:
ocr_result = [r[1] for r in self.__ocr_engine.readtext(image) if r[-1] >= min_confidence]
extracted_text["fullDoc"] = " ".join(ocr_result)
# Get the doc layout in the image:
document_sections = self.__yolo.predict(image)
document_sections = document_sections["boxes"]
# For each section, we run the OCR process individually:
section_results = []
for section in document_sections:
if section["class"] in self.__whitelisted_yolo_classes:
section_width = section["x2"] - section["x1"]
section_height = section["y2"] - section["y1"]
x1 = max(int(section["x1"] - (margin * section_width)), 0)
y1 = max(int(section["y1"] - (margin * section_height)), 0)
x2 = min(int(section["x2"] + (margin * section_width)), width)
y2 = min(int(section["y2"] + (margin * section_height)), height)
sub_image = image[y1:y2, x1:x2]
ocr_result = [r[1] for r in self.__ocr_engine.readtext(sub_image) if r[-1] >= min_confidence]
section_results.append({
"text": " ".join(ocr_result),
"class": section["class"],
"className": section["className"],
"x1": x1,
"y1": y1,
"x2": x2,
"y2": y2
})
extracted_text["bySection"] = section_results
# Done here:
return extracted_text
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import time
my_scanner = DocumentScanner(
layout_detection_yolo = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt",
whitelisted_yolo_classes = [0, 1, 3, 4, 5, 7, 9, 10],
# whitelisted_yolo_classes = [0, 1, 3, 4, 5, 6, 7, 8, 9, 10],
ocr_languages = ["en"]
)
image = my_scanner.open_image(r"/home/developer/Downloads/sushmita_card.jpg")
# image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card.jpg")
# image = my_scanner.open_image(r"/home/developer/Downloads/niranjan_card_2.jpg")
# image = my_scanner.open_image(r"/home/developer/Downloads/card_square.jpg")
document_image = my_scanner.scan(image)
if document_image is not None:
cv2.imwrite(r"/home/developer/Downloads/flattened_image.jpg", document_image)
start_time = time.time()
document_text = my_scanner.extract_text(
document_image,
min_confidence = 0.5
)
print("OCR RESULT:")
print(document_text)
print(f"FINISHED IN {time.time() - start_time} SECONDS!")
else: print("No image")
utils_v2/logging/__init__.py
View File
utils_v2/logging/__pycache__/__init__.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/logging/__pycache__/model.cpython-310.pyc
BIN
View File
Binary file not shown.
utils_v2/logging/context.py
-468
View File
@@ -1,468 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Friday, 2nd Aug., 2024
OBJECTIVE:
To provide an easy way to log all system activities by way of managing the context of what is going on.
REFERENCES:
N/A
DOWNLOADS:
N/A
NOTES:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To manage context:
import contextvars
from contextlib import contextmanager, asynccontextmanager
# To make decorators:
from functools import wraps
# For system-level activities:
import os
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
from utils_v2.security import sanitizers
from utils_v2.api.codes import StatusCodes, HttpCodes
from utils_v2.api.log import APILogModel
from utils_v2.api.response import ResponseModel
# The needed data models:
from utils_v2.logging.model import GeneralLogModel
# To work with date and time:
import time
# For random strings:
import random
# For debugging:
import traceback
import string
# To work with datatypes:
from types import NoneType
import pandas as pd
# To work with Pydantic objects:
from pydantic import BaseModel
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Chars to choose from for random strings:
ALPHANUMERIC_CHARS = string.ascii_letters + string.digits
# To capture system information:
PROCESS_ID = os.getppid()
PARENT_PROCESS_ID = os.getppid()
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** EXCEPTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def describe_exception(exc):
"""
Describes the exception in detail. It extracts the type of exception, a brief message, and even the entire
traceback. Useful for debugging in details without the terminal. You could either log the resultant dict or send it
to the dev team over some service like WhatsApp/Telegram.
:param exc: The exception that occurred.
:return: The dict that explains the exception.
"""
exc_desc = {
"type": type(exc).__name__,
"msg": str(exc),
"tb": [str(exc_tb) for exc_tb in traceback.format_exception(exc, value = exc, tb = exc.__traceback__)]
}
return exc_desc
# *****************************************************************************************************************
# ***** ****
# *** Classes ***
# ***** ****
# *****************************************************************************************************************
class AsyncMongoLogger:
def __init__(self, db_conn, collection = "logs"):
"""
This class uses an instance of 'AsyncMongo' and makes it usable as a logger.
:param db_conn: The instance of 'AsyncMongo' to use.
:param collection: The collection to write the log into.
"""
self.__db_conn = db_conn
self.__collection = collection
async def log(self, log_json):
"""
Log something to the database using the connection provided when the object was made.
:param log_json: The dict to log.
:return: True if logged successfully, else False.
"""
asyncio.create_task(self.__db_conn.insert_one(
collection = self.__collection,
document = log_json,
raise_exception = False
))
# ---------------------------------------------------------------------------------------------------------------------
class AsyncLoggerContext:
# Create the context-aware variable(s):
logger = contextvars.ContextVar("logger", default = None)
log_chain = contextvars.ContextVar("log_chain", default = None)
@classmethod
@asynccontextmanager
async def logging_context(cls, logger, log_chain = None):
"""
This function makes the context manager that makes the value of the log chain available to everything that is
called within the scope of the context.
:param logger: The object which is to be used to write the log. It should have a 'log' method which should take
in a dict as its input.
:param log_chain: The value of the log chain to be made available within the scope.
:return: None.
"""
# Set the context:
token_logger = cls.logger.set(logger)
token_log_chain = cls.log_chain.set(log_chain)
# Make the objects available within the context:
try: yield
# Release the objects when the context is over:
finally:
cls.logger.reset(token_logger)
cls.log_chain.reset(token_log_chain)
@staticmethod
def generate_log_id(count = 8):
return "".join(random.choice(ALPHANUMERIC_CHARS) for _ in range(min(8, count)))
@classmethod
def get_logger(cls):
return cls.logger.get()
@classmethod
def get_log_chain(cls):
return cls.log_chain.get()
@staticmethod
def summarize(
value,
str_limit = 100,
expand: bool | int = False,
sensitive_keys: list[str] = None
):
"""
To summarize an input value to capture the essence without hoarding to much data.
:param value: Anything that you want to summarize.
:param str_limit: The max. no. of chars of a string to retain.
:param expand: Set to True for full expansion, False for no expansion, and an integer for a specific level of
expansion. Applicable on iterables and dicts. The smaller this number, the more concise the summary will be,
and vice versa.
:param sensitive_keys: The list of keys (of a dict) to obscure when summarizing.
:return: The summarized version of the input.
"""
# If the input is a Pydantic class:
if isinstance(value, BaseModel): value = value.model_dump()
# Check the sensitive keys:
if sensitive_keys is None: sensitive_keys = []
# Handle datatypes that you don't want to modify:
if isinstance(value, (int, float, bool, NoneType)): pass
# When the value is a list or similar iterable:
elif isinstance(value, (list, tuple, set)):
if expand:
if not isinstance(expand, bool): expand -= 1
value = [AsyncLoggerContext.summarize(
v,
expand = expand,
sensitive_keys = sensitive_keys
) for v in value]
else: value = f"array of {len(value)} item(s)"
# If the value is a dict:
elif isinstance(value, dict):
if expand:
if not isinstance(expand, bool): expand -= 1
value = {
k: AsyncLoggerContext.summarize(
v,
expand = expand,
sensitive_keys = sensitive_keys
) if k not in sensitive_keys else "********"
for k, v in value.items()
}
else: value = f"object of {len(value.keys())} field(s) [{', '.join(value.keys())}]"
# When a dataframe is passed:
elif isinstance(value, pd.DataFrame):
cols = value.columns.to_list()
value = f"table with {len(cols)} col(s) [{', '.join(cols)}] and {len(value)} row(s)"
str_limit = 999
# If the input is some form of non-standard object:
else: value = str(value)
# Handle strings:
if isinstance(value, str):
if len(value) > str_limit: value = value[:str_limit] + "..."
# Done here:
return value
@classmethod
def log_it(
cls,
api_version: str = None,
project: str = None,
log_type: str = None,
operation: str = None,
log_input: bool | int = True,
log_output: bool | int = True,
sensitive_keys: list = None
):
"""
A decorator factor that can be used to log the results of functions automatically.
:param api_version: A string that indicates the version code of the function being decorated.
:param project: A hint about which project is being worked on.
:param log_type: A hint about which module is being worked on.
:param operation: A hint about which action in a particular module is being worked on.
:param log_input: Set to True to capture everything that went into the function, False to capture the least
info, and set it to an integer to capture a certain depth of the input (applicable on iterables and dicts.
:param log_output: The same as 'log_input', but applicable to the response from the function.
:param sensitive_keys: Keys of a dict whose values must be obscured even if that depth is being captured.
:return: A decorator with the configuration.
"""
def decorator(func):
@wraps(func)
async def wrapper(*args, **kwargs):
# Make variables and extract available info.:
exception = None
response = None
request_ts = date_time.get_current_utc_date_time()
start_ts = time.perf_counter()
cpu_start_ts = time.process_time()
# Execute the function that is being wrapped:
try: response = await func(*args, **kwargs)
except Exception as exc: exception = exc
# Do the next steps only if within the logging context:
if cls.get_logger() is not None:
# Create the log:
if not args: args = None
if not kwargs: kwargs = None
func_log = GeneralLogModel(
pid = PROCESS_ID,
ppid = PARENT_PROCESS_ID,
project = project,
log = log_type or func.__name__,
operation = operation or func.__name__,
apiVer = api_version,
logId = cls.generate_log_id(),
logChain = cls.get_log_chain(),
ts = request_ts,
tat = time.perf_counter() - start_ts,
cpuTime = time.process_time() - cpu_start_ts,
func = func.__name__,
args = cls.summarize(args, expand = log_input, sensitive_keys = sensitive_keys),
kwargs = cls.summarize(kwargs, expand = log_input, sensitive_keys = sensitive_keys),
exception = None if exception is None else describe_exception(exception),
response = cls.summarize(response, expand = log_output, sensitive_keys = sensitive_keys),
).model_dump()
# Write the log:
await cls.get_logger().log(func_log)
# Done here:
if exception is not None: raise exception
return response
return wrapper
return decorator
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
from utils_v2.database.async_mongo_v2 import AsyncMongo
@AsyncLoggerContext.log_it(
api_version = "0.0.1",
project = "testProj",
log_type = "work",
operation = "someWork",
log_input = True,
log_output = True,
sensitive_keys = None
)
async def some_work(*args, **kwargs):
print("SOME WORK:", AsyncLoggerContext.get_log_chain())
await asyncio.sleep(max(2.0 * random.random(), 1.0))
total = sum(args)
return total
@AsyncLoggerContext.log_it(
api_version = "0.0.1",
project = "testProj",
log_type = "work",
operation = "moreWork",
log_input = True,
log_output = True,
sensitive_keys = ["password", "sessionToken"]
)
async def more_work(*args, **kwargs):
print("MORE WORK:", AsyncLoggerContext.get_log_chain())
await asyncio.sleep(max(2.0 * random.random(), 1.0))
return {"success": True, "sessionToken": "1234567890"}
@AsyncLoggerContext.log_it(
api_version = "0.0.1",
project = "testProj",
log_type = "work",
operation = "moreWork",
log_input = True,
log_output = True,
sensitive_keys = None
)
async def last_work(*args, **kwargs):
print("LAST WORK:", AsyncLoggerContext.get_log_chain())
await asyncio.sleep(max(2.0 * random.random(), 1.0))
async def main(chain = None):
# Connect to MongoDB:
mongo = AsyncMongo(
connection_string = r"mongodb://del.ditscentre.in:27017,wtt.ditscentre.in:27017,mum.arh.001.ditscentre.in:27017/admin?tls=true&tlsCAFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_ca.pem&tlsCertificateKeyFile=%2Fetc%2Fssl%2Fcerts%2Fmongo_data_cert.pem&replicaSet=dits_mongod_rep&readPreference=primary&authMechanism=MONGODB-X509&authSource=%24external",
database_name = "converse",
max_connections = 10,
debug = True
)
# Convert the connection to a logger instance that can be injected
# into the context as a dependency:
mongo_logger = AsyncMongoLogger(
db_conn = mongo,
collection = "logs"
)
# Initialize the context:
async with AsyncLoggerContext.logging_context(
logger = mongo_logger,
log_chain = chain
):
# Run some functions within the context:
await some_work(1, 2, 3, 4, 5)
await more_work(username = "john.doe@domain.com", password = "mySecretPass")
# Run something outside the context:
await last_work()
# async def multi_main():
# tasks = [
# main(chain = "kPRwXdItb1"),
# main(chain = "456")
# ]
# await asyncio.gather(*tasks)
asyncio.run(main(chain = "00wGHRFYPY123"))
utils_v2/logging/model.py
-131
View File
@@ -1,131 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Friday, 11th Oct., 2024
OBJECTIVE:
To have a structure for the logs maintained for regular function calls.
This is different from the logs maintained for API calls.
REFERENCES:
01. https://chatgpt.com/share/6708b6c9-6ba4-800f-9ea9-00ec35067512
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# System-level activities:
import distro
import socket
import platform
# For data-modelling:
from pydantic import BaseModel, Field
from typing import Any, Optional, List, Literal
# To work with date and time:
import datetime
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# Info for logging that will stay constant during runtime:
SERVER_HOSTNAME = str(socket.gethostname())
PLATFORM_INFO = platform.uname()
HOST_OS = str(distro.name(True))
HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})"
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class GeneralLogModel(BaseModel):
# To identify the machine the code is running on.
# DO NOT MODIFY THESE:
hostname: str = SERVER_HOSTNAME
os: str = HOST_OS
cpu: str = HOST_CPU
# Can modify these:
pid: Optional[Any] = None
ppid: Optional[Any] = None
# To identify the project and actions:
project: Optional[str] = None
log: str
operation: Optional[str] = None
apiVer: Optional[str] = None
logId: Optional[str] = None
logChain: Optional[str] = None
# Timing metrics:
ts: datetime.datetime
tat: float
cpuTime: float
# To understand the inputs:
func: str
args: Optional[Any] = None
kwargs: Optional[Any] = None
# To understand the outputs:
exception: Optional[Any] = None
response: Optional[Any] = None
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/mail/__init__.py
View File
utils_v2/mail/async_smpt.py
-443
View File
@@ -1,443 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 16th Jul, 2024
OBJECTIVE:
To be able to send out mails from code.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For working with mails:
import aiosmtplib
from email.mime.multipart import MIMEMultipart
from email.mime.image import MIMEImage
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
# For random strings:
import string
import random
# For system-level activities:
import os
# For working with files in RAM:
import io
# For debugging:
from icecream import IceCreamDebugger
# To work with datatypes:
from typing import List
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class SMTPMessage:
def __init__(
self,
to_email: str,
subject: str,
cc_emails: List[str] = None,
bcc_emails: List[str] = None
):
"""
Create an instance of the message that you would like to send.
:param to_email: The EMail ID of th recipient.
:param subject: The subject of the mail.
:param cc_emails: A list of recipients to add to the CC section.
:param bcc_emails: A list of recipients to add to the BCC section.
"""
self.message = MIMEMultipart()
self.message["To"] = to_email
self.message["Subject"] = subject
if cc_emails: self.message["CC"] = ",".join(cc_emails)
if bcc_emails: self.message["BCC"] = ",".join(bcc_emails)
def add_text(self, text):
"""
Add plain-text to the mail body.
:param text: The text to add to the mail body.
:return: None.
"""
self.message.attach(MIMEText(text, "plain"))
def add_html(self, html_text):
"""
Add HTML text to the mail body.
:param html_text: The HTML text to add to the mail body.
:return: None.
"""
self.message.attach(MIMEText(html_text, "html"))
def add_inline_image(self, image_file, content_id = None):
"""
Add an inline image to the body of the mail.
NOTE: This is NOT the same as sending an image as an attachment.
:param image_file: The image data to attach to the mail body.
:param content_id: Inline images are inserted via HTML bocks. This field identifies the image resource. If not
specified, I will generate a random string. You may write a custom value here if you know what you are
doing. For most use cases, please ignore this field.
:return: None.
"""
# Read the image as bytes:
image_bytes = None
if type(image_file) is str:
with open(image_file, "rb") as opened_image_file:
image_bytes = opened_image_file.read()
if type(image_file) is io.BytesIO:
image_file.seek(0)
image_bytes = image_file.getvalue()
# Declare the part to be attached to the multipart message:
if image_bytes is not None:
# Create the HTML block if the image pointer is blank:
if content_id is None:
content_id = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
self.add_html(f"""
<html>
<body>
<p><img src="cid:{content_id}"></p>
</body>
</html>
""")
# Then add the image:
image_part = MIMEImage(image_bytes)
image_part.add_header("Content-ID", f"<{content_id}>")
self.message.attach(image_part)
def add_attachment(self, attachment_file, file_name = None):
# Declare the part to be attached to the multipart message:
part = MIMEBase("application", "octet-stream")
# If the attachment is a file stored in the local disk:
if isinstance(attachment_file, str):
file_name = file_name or os.path.split(attachment_file)[-1]
with open(attachment_file, "rb") as attachment:
part.set_payload(attachment.read())
# If the file is held in RAM:
elif isinstance(attachment_file, io.BytesIO):
attachment_file.seek(0)
part.set_payload(attachment_file.read())
# Encode and attach the file:
encoders.encode_base64(part)
part.add_header(
"Content-Disposition",
f"attachment; filename= {file_name}",
)
self.message.attach(part)
def get_message(self):
return self.message
# ---------------------------------------------------------------------------------------------------------------------
class AsyncSMTPClient:
# Constants:
SMTP_TLS_PORT = 587
SMTP_SSL_PORT = 465
# variables:
__smtp = None
def __init__(
self,
email,
password,
server,
port = 587,
rate_limiters = None,
wait_for_turn = True,
debug = True,
debug_prefix = "Mail (C) | "
):
"""
Set up the mail client.
:param email: The Email ID to use when sending out mails.
:param password: The password of the EMail ID that is being used.
:param server: The EMail server.
:param port: The port number to connect to the host.
:param rate_limiters: The rate limiters to use. Must have "get_turn" and "has_turn" methods. "get_turn" method
must wait for the turn, and "has_turn" method must only check if a turn is available.
:param wait_for_turn: To wait for turn if the rate limit has been exceeded, or to return with failure.
:param debug: Whether, or not, you want to print debugging messages.
:param debug_prefix: The prefix to identify the debugging messages.
"""
# Initialize the debugger:
self.__printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self.__printer.disable()
# Note down the credentials and other details:
self.__email = email
self.__password = password
self.__server = server
self.__port = port
self.__rate_limiters = rate_limiters if type(rate_limiters) is list else ([rate_limiters] if rate_limiters is not None else [])
self.__wait_for_turn = wait_for_turn
def enable_debug(self):
self.__printer.enable()
def disable_debug(self):
self.__printer.disable()
async def login(self):
"""
To connect to the mail server and authenticate the user.
:return: True if authenticated, else False.
"""
# Initialize the SMTP connection,
# and return with success if all goes well:
try:
self.__smtp = aiosmtplib.SMTP(
hostname = self.__server,
port = self.__port,
use_tls = False,
start_tls = False
)
await self.__smtp.connect()
await self.__smtp.starttls()
await self.__smtp.login(self.__email, self.__password)
return True
# Return with failure if something goes wrong:
except Exception as exception:
self.__printer(exception)
try: await self.__smtp.quit()
except Exception as exception: self.__printer(exception)
self.__smtp = None
return False
async def ensure_connection(self):
"""
Can be run before the sending operation to confirm that we are yet connected to the server.
If not connected, this code will reattempt to connect and log-in.
:return: True if connected, else False.
"""
# If the login had failed initially, the object will be set to null.
# In such a case, we make an attempt to login:
if self.__smtp is None:
return await self.login()
# If the login was successful, we check if the connection is active.
# If not, we try to re-login:
if self.__smtp.is_connected:
return True
else:
try:
await self.__smtp.connect()
await self.__smtp.starttls()
await self.__smtp.login(self.__email, self.__password)
return True
except Exception as exception:
self.__printer(exception)
try: await self.__smtp.quit()
except Exception as exception: self.__printer(exception)
self.__smtp = None
return False
async def logout(self):
"""
Closes the connection to the SMTP client.
:return: True by default.
"""
if self.__smtp is not None:
try: await self.__smtp.quit()
except Exception as exception: self.__printer(exception)
self.__smtp = None
return True
async def send(self, mail: SMTPMessage):
"""
Send out the mail.
:param mail: The instance of 'MailMessage' with all the content populated.
:return: A dict with 'success' and 'message'.
"""
# Return with failure if we aren't connected,
# and our attempt to (re)connect fails:
if not await self.ensure_connection():
return {
"success": False,
"message": "login failed"
}
# Comply with the rate-limit:
for rate_limiter in self.__rate_limiters:
if not self.__wait_for_turn:
if not await rate_limiter.has_turn(): return False
got_turn = await rate_limiter.get_turn()
if not got_turn:
return {
"success": False,
"message": "rate-limit wait timeout"
}
# Try to send the message:
try:
mail.message["From"] = self.__email
response = await self.__smtp.send_message(mail.message)
return {
"success": True,
"message": f"mail accepted - {response[-1]}"
}
# If something goes wrong:
except Exception as exception:
self.__printer(exception)
return {
"success": False,
"message": str(exception)
}
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
import asyncio
from utils_v2.string import json
async def test():
mail_client = AsyncSMTPClient(
email = "sender@gmail.com",
password = "zcaf nmqy ncfz fave",
server = "smtp.gmail.com",
rate_limiters = None
)
my_mail = SMTPMessage(
to_email = "orangebhopli@gmail.coms",
subject = "Bhopli is the best!",
cc_emails = None,
bcc_emails = None
)
my_mail.add_html(
"""
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>Sample HTML String</title>
<style>
.heading {
color: #ff9025;
}
.sub-heading {
color: #000000;
}
</style>
</head>
<body>
<h1 class="heading">Hello, Bhopli!</h1>
<h2 class="sub-heading">Bhopli is the best, most well-behaved cat in the known universe.</h2>
</body>
</html>
"""
)
my_mail.add_text("This is how you should pet her 👇")
my_mail.add_inline_image(r"/path/to/image/cat_petting.png")
my_mail.add_attachment(r"/path/to/file/sample_label.pdf")
await mail_client.login()
result = await mail_client.send(my_mail)
print("MAIL RESULT:", json.to_string(result))
await mail_client.logout()
asyncio.run(test())
utils_v2/mail/mail_parser.py
-266
View File
@@ -1,266 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Tuesday, 10th Dec., 2024
OBJECTIVE:
To parse raw mail bodies and give a structure that is suitable for storing in No-SQL databases like MongoDB. The
raw mail's text is expected to be compliant with standard defined in RFC 5322, RFC 2045, and maybe a few more.
REFERENCES:
1. GitHub: https://github.com/SpamScope/mail-parser
2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
4. StackOverflow: https://stackoverflow.com/questions/17874360/python-how-to-parse-the-body-from-a-raw-email-given-that-raw-email-does-not
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# To work with mails:
import email
from email.message import Message
from email.utils import parsedate_tz
from email.utils import parseaddr
# To parse the HTML content in the mail:
from bs4 import BeautifulSoup
# To work with datatypes:
from typing import Any, Dict, List, Literal
# To work with various encodings:
import base64
import quopri
# To work with date and time:
import datetime
import pytz
import time
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def parse_addr(addr_header: str) -> List[Dict[str, str]]:
# If the field is null, we return null:
if addr_header is None: return []
# Create an empty variable that will hold the results:
addrs = []
# Iterate through the addresses and parse them:
for a in addr_header.split(","):
n, e = parseaddr(a.strip())
addrs.append({
"name": n.strip() or e.strip(),
"email": e.strip()
})
# Done here:
return addrs
# ---------------------------------------------------------------------------------------------------------------------
def parse_date(date_header: str) -> datetime.datetime | None:
# Try to parse the date header:
date_tuple = parsedate_tz(date_header)
# If the date header was parsed successfully, we assemble
# the parts to get an aware object in UTC timezone:
if date_tuple:
dt = datetime.datetime(*date_tuple[:6], tzinfo = pytz.FixedOffset(int(date_tuple[-1] / 60)))
dt = date_time.to_timezone(dt, date_time.TIMEZONE_UTC)
return dt
# In case of an invalid date header:
else: return None
# ---------------------------------------------------------------------------------------------------------------------
def decode_payload(
raw_payload: str | bytes,
content_main_type: str,
content_charset: str | None,
content_transfer_encoding: Literal[None, "base64", "quoted-printable"]
) -> str | bytes:
# Start by assuming nothing needs to be done:
payload = raw_payload
# We decode various kinds of parts:
match content_transfer_encoding:
# This is just unencoded plaintext:
case None:
pass
# Typically see with attachments:
case "base64":
charset = content_charset or "utf-8"
payload = raw_payload
payload = base64.b64decode(payload)
if content_main_type == "text": payload = payload.decode(charset)
# Typically seen with HTML parts:
case "quoted-printable":
charset = content_charset or "utf-8"
payload = raw_payload.encode(charset)
payload = quopri.decodestring(payload)
if content_main_type == "text": payload = payload.decode(charset)
# Done here:
return payload
# ---------------------------------------------------------------------------------------------------------------------
def parse_part(
part: Message | List[Message]
) -> Dict[str, Any]:
# Start by extracting basic details:
part_json = {
"boundary": part.get_boundary(),
"contentType": part.get_content_type(),
"contentMainType": part.get_content_maintype(),
"contentSubType": part.get_content_subtype(),
"contentCharset": part.get_content_charset(),
"contentTransferEncoding": part.get("Content-Transfer-Encoding"),
"contentDisposition": part.get_content_disposition(),
"filename": part.get_filename(),
"contentId": part.get("Content-ID")
}
# Process the payload of this part:
if part_json["contentMainType"] == "multipart":
part_json["payload"] = [parse_part(sub_part) for sub_part in part.get_payload(decode = False)]
else:
part_json["payload"] = decode_payload(
raw_payload = part.get_payload(decode = False),
content_main_type = part_json["contentMainType"],
content_charset = part_json["contentCharset"],
content_transfer_encoding = part_json["contentTransferEncoding"]
)
# Done here;
return part_json
# ---------------------------------------------------------------------------------------------------------------------
def parse(raw_mail: str | bytes) -> Dict[str, Any]:
"""
To parse the raw mail text to a usable JSON that can even be stored on a No-SQL database like MongoDB.
DOCUMENTATION:
1. GitHub: https://github.com/SpamScope/mail-parser
2. RFC 5322: https://datatracker.ietf.org/doc/html/rfc5322
3. RFC 2045: https://datatracker.ietf.org/doc/html/rfc2045
:param raw_mail: The raw mail body that adheres to RFC 5322 and RFC 2045 (among others).
:return: The parsed JSON format (dict) of the mail.
"""
# Parse the raw format:
if isinstance(raw_mail, str): parsed_mail = email.message_from_string(raw_mail)
else: parsed_mail = email.message_from_bytes(raw_mail)
# Extract the most basic details:
mail_json = {
"ts": parse_date(parsed_mail["Date"]),
"headers": {k: v for k, v in parsed_mail.items()},
"from": parse_addr(parsed_mail["From"]),
"to": parse_addr(parsed_mail["To"]),
"cc": parse_addr(parsed_mail["Cc"]),
"bcc": parse_addr(parsed_mail["Bcc"]),
"subject": parsed_mail["Subject"],
"payload": None
}
# Iterate through each part of the mail for multipart mails:
if parsed_mail.is_multipart(): mail_json["payload"] = parse_part(parsed_mail)
# When the mails are not multipart, just plaintext:
else: mail_json["payload"] = parsed_mail.get_payload()
# Done here:
return mail_json
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
from utils_v2.system import files
mail_string_raw = files.read_file(r"/home/developer/Downloads/raw_mail_test.txt")
parse_results = parse(mail_string_raw)
print(json.to_string(parse_results, default = str))
utils_v2/network/__init__.py
View File
utils_v2/network/ip.py
-173
View File
@@ -1,173 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Friday, 12th Jul., 2024
OBJECTIVE:
To provide an easy way to get geolocation information of an IP address.
REFERENCES:
1) https://medium.com/@tubelwj/how-to-retrieve-ip-geolocation-information-in-python-929e15041e3e
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# For working with IP Addresses:
import ipaddress
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def ipv4_to_int(ip_string):
"""
Converts an IP (v4) string to an integer value.
:param ip_string: The IP address (v4) that you want to convert to integer format.
:return: An integer representation of the IP (v4) address.
"""
ip_numerical = int(ipaddress.IPv4Address(ip_string))
return ip_numerical
# ---------------------------------------------------------------------------------------------------------------------
def int_to_ipv4(ip_numerical):
"""
Interprets the IP (v4) value from the given integer value.
:param ip_numerical: The integer value that represents an IP (v4) address.
:return:
"""
ip_string = str(ipaddress.IPv4Address(ip_numerical))
return ip_string
# ---------------------------------------------------------------------------------------------------------------------
def ipv4_to_bin(ip_string):
"""
Converts an input IP (v4) address to the binary string that represents the 32 bits.
:param ip_string: The IP (v4) string in a format like "192.168.0.1"
:return: The binary representation (as a string) of the input IP address.
"""
ip_binary = bin(int(ipaddress.IPv4Address(ip_string)))[2:].zfill(32)
return ip_binary
# ---------------------------------------------------------------------------------------------------------------------
def bin_to_ipv4(ip_binary):
"""
Interprets the IP (v4) value from the given binary string.
:param ip_binary: The string of 1s and 0s that represents the IP (v4) address.
:return: The IP (v4) address as a string.
"""
ip_string = str(ipaddress.IPv4Address(int(ip_binary, 2)))
return ip_string
# ---------------------------------------------------------------------------------------------------------------------
def get_ipv4_range(ip_string, as_string = True):
"""
Given a network description in the format "88.95.100.128/25", this function tells you the first and last IP
addresses of that network. Useful for determining if an IP address lies in a network.
:param ip_string: The input network description in the format "88.95.100.128/25"
:param as_string: To select between integer and string formats for the IP range output.
:return: The first and last IP addresses of the input network, and the count.
"""
# Extract the components of the string:
ip_components = ip_string.split("/")
ip_addr = ipv4_to_int(ip_components[0])
ip_bits = int(ip_components[1])
# Convert the mask number to binary representation:
ip_mask = (1 << ip_bits) - 1
ip_mask = ip_mask << (32 - ip_bits)
inv_ip_mask = (~ip_mask) & 0xFFFF
# Figure out the start and end IP addresses:
start_ip = ip_addr & ip_mask
end_ip = ip_addr | inv_ip_mask
count = end_ip - start_ip + 1
# If the IPs are needed as strings, we perform the conversion:
if as_string:
start_ip = int_to_ipv4(start_ip)
end_ip = int_to_ipv4(end_ip)
# Done here:
return start_ip, end_ip, count
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
print(ipv4_to_int("255.255.255.255"))
print(ipv4_to_int("x.x.x.x"))
utils_v2/network/ping.py
-201
View File
@@ -1,201 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Friday, 28th Jun, 2024
OBJECTIVE:
To provide a way to ping a server and get the traceroute dump.
REFERENCES:
1) https://www.geeksforgeeks.org/traceroute-implementation-on-python/
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# Utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# For networking:
import socket
from scapy.all import *
# For running the script from the terminal:
import argparse
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
def get_name_and_addr(destination):
# If the user provided the IP address:
if (
regex.match(destination, regex.REGEX_IPV4) or
regex.match(destination, regex.REGEX_IPV6)
):
try: destination_name = socket.gethostbyaddr(destination)[0]
except Exception as exception: destination_name = "*"
destination_ip = destination
# If the provided destination was the domain name:
else:
destination_name = destination
try: destination_ip = socket.gethostbyname(destination)
except Exception as exception: destination_ip = "*"
# Done here:
return destination_name, destination_ip
# ---------------------------------------------------------------------------------------------------------------------
def tracert(
destination,
max_hops = 30,
timeout = 2.0,
port = 33434
):
# Initialize the variables:
destination_name, destination_ip = get_name_and_addr(destination)
full_trace = []
ttl = 1
# Keep noting hops till the limit is reached:
while ttl <= max_hops:
# Create a JSON for this stage:
this_hop = {
"destAddr": destination_ip,
"destName": destination_name,
"hopNo": ttl - 1,
"isDest": False,
"hopAddr": None,
"hopName": None,
"ts": None
}
# Create the IP and UDP headers and combine them:
ip_packet = IP(dst = destination, ttl = ttl)
udp_packet = UDP(dport = port)
trace_packet = ip_packet / udp_packet
# Send the packet and receive a reply and note down the timestamp:
reply = sr1(trace_packet, timeout = timeout, verbose = 0)
this_hop["ts"] = date_time.get_current_utc_date_time(as_string = True)
# No response:
if reply is None: this_hop["hopAddr"] = this_hop["hopName"] = "*"
# If some response was received, we note the values and break out if this was the destination hop:
else:
this_hop["hopName"], this_hop["hopAddr"] = get_name_and_addr(f"{reply.src}")
if reply.type == 3:
this_hop["isDest"] = True
full_trace.append(this_hop)
break
# Carry on to the next hop:
full_trace.append(this_hop)
ttl += 1
# Done here:
return full_trace
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
parser = argparse.ArgumentParser(description = "Traceroute Implementation in Python!")
parser.add_argument(
"dest",
help = "Destination (Name or IP address)."
)
parser.add_argument(
"-m",
"--max-hops",
type = int,
default = 30,
help =
"Maximum number of hops (default: 30)."
)
parser.add_argument(
"-t",
"--timeout",
type = float,
default = 2.0,
help = "Timeout for each packet in seconds (default: 2.0)."
)
parser.add_argument(
"-p",
"--port",
type = int,
default = 33434,
help = "The port to probe at the destination (default: 33434)."
)
args = parser.parse_args()
trace = tracert(
destination = args.dest,
max_hops = args.max_hops,
timeout = args.timeout,
port = args.port
)
print("TRACE:", json.to_string(trace))
utils_v2/nse/__init__.py
View File
utils_v2/nse/controllers/__init__.py
View File
utils_v2/nse/controllers/base.py
-443
View File
@@ -1,443 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 28th Nov., 2024
OBJECTIVE:
To provide a base class for common behaviour of NSE's APIs.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
from utils_v2.nse.models.api_call import NSEApiResponse
# To make API calls:
import httpx
# To work with date and time:
import datetime
# For working with datatypes:
from typing import Literal, List
# For debugging:
from icecream import IceCreamDebugger
import inspect
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** CLASSES ***
# ***** ****
# *****************************************************************************************************************
class AsyncNSEBase:
def __init__(
self,
base_url: str,
data_url: str,
http_client: httpx.AsyncClient = None,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE | ",
debug_only_errors = True
):
"""
To be used as a base class for all sorts of NSE scraping.
:param base_url: The base URL (that you open in your browser) for your targeted module.
:param data_url: The internal URL (that NSE's script calls by itself) for your targeted module.
:param http_client: An HTTP client to use to make API calls.
:param cookies_refresh_interval: The no. of seconds after which the cookies should get refreshed.
:param debug: Whether, or not, you want to show debugging messages.
:param debug_prefix: The prefix string to use to recognize the module that is printing the debugging text.
:param debug_only_errors: Whether you want to show all debugging messages, or just error messages.
"""
# Prepare the debugging utility:
self._debug_prefix = debug_prefix
self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
if not debug: self._printer.disable()
self._debug_only_errors = debug_only_errors
# Accept the input configuration:
self._base_url = base_url
self._data_url = data_url
self._http_client = http_client or httpx.AsyncClient(
limits = httpx.Limits(
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
),
timeout = httpx.Timeout(
pool = 120.0, # .... Time to wait for a free connection from the pool.
connect = 5.0, # ... Time to wait for establishing a connection to the server.
write = 10.0, # .... Time to wait for sending data.
read = 60.0 # ...... Time to wait for receiving data.
)
)
# Make some provisions for later:
self._cookies = None
self._cookies_refresh_interval = cookies_refresh_interval
self._last_cookies_refresh = datetime.datetime.fromtimestamp(0, tz = date_time.TIMEZONE_UTC)
# For locking asynchronous tasks:
self._exclusive_lock = asyncio.Semaphore(1)
def enable_debug(self):
self._printer.enable()
def disable_debug(self):
self._printer.disable()
def debug_only_errors(self):
self._debug_only_errors = True
def debug_everything(self):
self._debug_only_errors = False
# ┏┓ •
# ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
# ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
# ┛
@property
def cookies(self):
return self._cookies
# ┓┏ ┓
# ┣┫┏┓┃┏┓┏┓┏┓┏
# ┛┗┗ ┗┣┛┗ ┛ ┛
# ┛
@staticmethod
def parse_datetime_string(
dt_str: str,
dt_formats: str | List[str] = None,
input_tz: str = date_time.TIMEZONE_IST,
output_tz: str = date_time.TIMEZONE_UTC
) -> datetime.datetime | None:
"""
Handles the date-time conversion and normalization for strings received from NSE.
:param dt_str: The string value as received from NSE.
:param dt_formats: The format(s) in which to attempt to parse the date string.
:param input_tz: The timezone in which the input must be assumed to be.
:param output_tz: The timezone the output must be converted to.
:return: The parsed datetime object, or null if parsing fails.
"""
# Basic input formatting:
dt_formats = dt_formats or ["%d-%b-%Y"]
# Try to parse the date-time string:
if isinstance(dt_formats, list):
dt_obj = date_time.parse_date_time(
input_value = dt_str,
date_formats = dt_formats
)
else:
try: dt_obj = datetime.datetime.strptime(dt_str, dt_formats)
except: dt_obj = None
# If the parsing failed:
if not dt_obj: return None
# If parsed successfully:
return date_time.to_timezone(
datetime_object = date_time.as_if_timezone(
datetime_object = dt_obj,
timezone = input_tz
),
timezone = output_tz
)
# ┏┓ ┓ • ┳┳┓ ┓ •
# ┃ ┏┓┏┓┃┏┓┏┓┏ ┃┃┃┏┓┃┏┓┏┓┏┓
# ┗┛┗┛┗┛┛┗┗┗ ┛ ┛ ┗┗┻┛┗┗┛┗┗┫
# ┛
@property
def seconds_since_cookies_refreshed(self):
return (date_time.get_current_utc_date_time() - self._last_cookies_refresh).total_seconds()
async def refresh_cookies(
self,
force_refresh = True,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> bool:
"""
To get cookies for the given base URL. NSE has some sort of strict cookies and origin policy that is beyond my
current understanding. But I have noticed that if you furnish the right base URL and then use that, NSE will
respond just fine.
USAGE: Call this right after you initialize your class and then call it every so often after that.
:param force_refresh: Set to True to ignore the time criterion and refresh the cookie even if it is new.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The cookies in the 'data' field of the
"""
# Start by assuming failure:
success = False
http_code = None
# Ensure that only one cookie-refresh attempt is being made at a time:
async with self._exclusive_lock:
# We refresh the cookies only if the time has elapsed,
# or we have been asked to forcefully refresh the cookies:
if (
force_refresh or
self._cookies is None or
self.seconds_since_cookies_refreshed > self._cookies_refresh_interval
):
# Make the API call:
api_response = await self.get(
url = self._base_url,
headers = {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Upgrade-Insecure-Requests": "1",
"Sec-Ch-Ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"128\"",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
},
refresh_cookies = False,
force_refresh_cookies = False,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
http_code = api_response.httpCode
# If the API call succeeds:
if api_response.httpCode in [200]:
self._last_cookies_refresh = date_time.get_current_utc_date_time(as_string = False)
self._cookies = api_response.response.cookies
success = True
# Done here:
if not success or not self._debug_only_errors: self._printer(success, http_code)
return success
# ┏┓┏┓┳ ┏┓ ┓┓•
# ┣┫┃┃┃ ┃ ┏┓┃┃┓┏┓┏┓
# ┛┗┣┛┻ ┗┛┗┻┗┗┗┛┗┗┫
# ┛
async def _get(
self,
url: str = None,
headers: dict = None,
params: dict = None,
cookies: httpx.Cookies | dict = None
) -> (httpx.Response | None, Exception | None):
"""
To call an API once using the GET method.
:param url: A custom URL to hit. If not specified, the data URL will be hit.
:param headers: Custom headers to use. If not specified, default values will be used.
:param params: The params to send in the query string itself.
:param cookies: The cookies to use when calling the API.
:return: The raw response from the API call and any exception that occurred.
"""
# Start by assuming failure:
response = None
excp = None
try:
# Make the API call:
response = await self._http_client.get(
url = url or self._data_url,
headers = headers or {
"Accept": "*/*",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Host": "www.nseindia.com",
"Referer": self._base_url,
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"Sec-GPC": "1",
"Upgrade-Insecure-Requests": "1",
"TE": "trailers",
"Sec-Ch-Ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"128\"",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
},
params = params,
cookies = cookies
)
# If something goes wrong:
except Exception as exception:
excp = exception
self._printer(exception, url, params)
# Done here:
return response, excp
async def get(
self,
url: str = None,
headers: dict = None,
params: dict = None,
cookies: httpx.Cookies | dict = None,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
Call an API using the GET method.
:param url: A custom URL to hit. If not specified, the data URL will be hit.
:param headers: Custom headers to use. If not specified, default values will be used.
:param params: The params to send in the query string itself.
:param cookies: Any custom cookies to use when calling the API. If not given, the class's internal cookies will
be used by default. Use this only to override the default one.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: A structured response that includes the raw response, the exception (if any), and so on.
"""
# Select and refresh the cookies as needed:
if cookies: use_cookies = cookies
else:
if refresh_cookies: await self.refresh_cookies(force_refresh = force_refresh_cookies)
use_cookies = self._cookies
# Prepare the structure of the response:
frame = inspect.stack()[1]
caller_locals = frame[0].f_locals
calling_class = caller_locals.get("self", None)
calling_class = calling_class.__class__.__name__ if calling_class else ""
calling_function = inspect.stack()[1].function
api_response = NSEApiResponse(
action = f"{calling_class}.{calling_function}",
url = url or self._data_url,
method = "GET",
inputs = {"headers": headers, "params": params}
)
# Try as many times as asked:
for attempt in range(retry_count):
# Simple debugging statement:
if not self._debug_only_errors:
self._printer("API call.", attempt)
# Make the API call:
raw_response, exception = await self._get(
url = url,
headers = headers,
params = params,
cookies = use_cookies
)
# Populate the response model:
if raw_response is not None:
api_response.response = raw_response
api_response.httpCode = raw_response.status_code
api_response.message = raw_response.reason_phrase
if raw_response.status_code in [200]: api_response.success = True
if exception is not None:
api_response.exception = exception
api_response.message = str(exception)
# If the API call succeeded:
if api_response.success: break
# If the API call failed:
await asyncio.sleep(backoff_seconds)
backoff_seconds *= backoff_multiplier
# Done here:
return api_response
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
pass
utils_v2/nse/controllers/bhavcopy/__init__.py
View File
utils_v2/nse/controllers/bhavcopy/equities.py
-326
View File
@@ -1,326 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Saturday, 4th Jan., 2025.
OBJECTIVE:
To provide a way to fetch the daily BhavCopy from NSE.
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
import pytz
# To work with datatypes:
from typing import Any, List
# To work with tabulated dataL
import pandas as pd
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSEEquitiesBhavCopy(AsyncNSEBase):
def __init__(
self,
http_client: httpx.AsyncClient,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (Eq. BhavCopy) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/all-reports",
data_url = r"https://nsearchives.nseindia.com/products/content/sec_bhavdata_full_03012025.csv",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
@staticmethod
def parse_float(value):
try: return float(value)
except ValueError: return None
@staticmethod
def parse_int(value):
try: return int(value)
except ValueError: return None
async def get_data(
self,
target_date: datetime.datetime,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the corporate event calendar.
:param target_date: The starting date (inclusive) from which the events must be fetched.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Figure out the file name:
date_time.to_timezone(target_date, date_time.TIMEZONE_IST)
file_name = f"sec_bhavdata_full_{target_date.strftime('%d%m%Y')}.csv"
# Make the API call:
api_response = await self.get(
url = f"https://nsearchives.nseindia.com/products/content/{file_name}",
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
},
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw:
api_response.data = io.BytesIO(await api_response.get_content())
api_response.data.seek(0)
else:
try: api_response.data = self.format_data(
raw_data = await api_response.get_content(),
timestamp = date_time.get_current_utc_date_time(as_string = False),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
def format_data(
self,
raw_data: bytes | io.BytesIO,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[dict] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_data: The raw data as scraped from NSE.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_data is None: return raw_data
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Ensure that we have a BytesIO object,
# and that we are pointing at the start:
if isinstance(raw_data, bytes):
raw_data = io.BytesIO(raw_data)
raw_data.seek(0)
# Read the data into Pandas:
bhav_df = pd.read_csv(raw_data)
bhav_df.rename(
columns = {c: c.strip() for c in bhav_df.columns.to_list()},
inplace = True
)
bhav_df.rename(
columns = {
"SYMBOL": "symbol",
"SERIES": "segment",
"DATE1": "date",
"PREV_CLOSE": "prevClose",
"OPEN_PRICE": "open",
"HIGH_PRICE": "high",
"LOW_PRICE": "low",
"CLOSE_PRICE": "close",
"LAST_PRICE": "ltp",
"AVG_PRICE": "vwap",
"TTL_TRD_QNTY": "totVol",
"TURNOVER_LACS": "totCash",
"NO_OF_TRADES": "tradeCount",
"DELIV_QTY": "deliveryVol",
"DELIV_PER": "deliveryPct"
},
inplace = True
)
# Format the data:
for column in ["symbol", "segment", "date"]:
bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip())
for column in ["prevClose", "open", "high", "low", "close", "ltp", "vwap", "totCash", "deliveryPct"]:
bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x))
for column in ["totVol", "deliveryVol"]:
bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x))
bhav_df["totCash"] *= 1_00_000
# Give the date as a timestamp:
bhav_df["scrapeTs"] = timestamp
bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string(
dt_str = x,
dt_formats = ["%d-%b-%Y"],
input_tz = date_time.TIMEZONE_IST,
output_tz = date_time.TIMEZONE_UTC
))
bhav_df["tz"] = "Asia/Kolkata"
# Keep what is needed, in an order that makes intuitive sense:
bhav_df = bhav_df[[
"symbol", "segment",
"prevClose", "open", "high", "low", "close", "ltp", "vwap",
"totVol", "totCash", "deliveryVol", "deliveryPct",
"date", "ts", "tz", "scrapeTs"
]]
# Done here:
formatted_data = bhav_df.to_dict(orient = "records")
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an HTTP client:
test_client = httpx.AsyncClient(
limits = httpx.Limits(
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
),
timeout = httpx.Timeout(
pool = 120.0, # .... Time to wait for a free connection from the pool.
connect = 2.5, # ... Time to wait for establishing a connection to the server.
write = 10.0, # .... Time to wait for sending data.
read = 2.5 # ....... Time to wait for receiving data.
)
)
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSEEquitiesBhavCopy(http_client = test_client)
# Get and show the data:
target_date = datetime.datetime.strptime("20250103", "%Y%m%d")
api_response = await my_nse.get_data(
target_date = target_date,
return_raw = False
)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str))
asyncio.run(main())
utils_v2/nse/controllers/bhavcopy/fno.py
-325
View File
@@ -1,325 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Monday, 6th Jan., 2025.
OBJECTIVE:
To provide a way to fetch the daily BhavCopy from NSE.
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
import pytz
# To work with datatypes:
from typing import Any, List
# To work with tabulated dataL
import pandas as pd
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSEFNOBhavCopy(AsyncNSEBase):
def __init__(
self,
http_client: httpx.AsyncClient,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (FNO BhavCopy) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/all-reports",
data_url = r"https://nsearchives.nseindia.com/content/trdops/FNO_BC02012025.DAT",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
@staticmethod
def parse_float(value):
try: return float(value)
except ValueError: return None
@staticmethod
def parse_int(value):
try: return int(value)
except ValueError: return None
async def get_data(
self,
target_date: datetime.datetime,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the corporate event calendar.
:param target_date: The starting date (inclusive) from which the events must be fetched.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Figure out the file name:
date_time.to_timezone(target_date, date_time.TIMEZONE_IST)
file_name = f"FNO_BC{target_date.strftime('%d%m%Y')}.DAT"
# Make the API call:
api_response = await self.get(
url = f"https://nsearchives.nseindia.com/content/trdops/{file_name}",
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
},
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw:
api_response.data = io.BytesIO(await api_response.get_content())
api_response.data.seek(0)
else:
try: api_response.data = self.format_data(
raw_data = await api_response.get_content(),
timestamp = date_time.get_current_utc_date_time(as_string = False),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
def format_data(
self,
raw_data: bytes | io.BytesIO,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[dict] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_data: The raw data as scraped from NSE.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_data is None: return raw_data
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Ensure that we have a BytesIO object,
# and that we are pointing at the start:
if isinstance(raw_data, bytes):
raw_data = io.BytesIO(raw_data)
raw_data.seek(0)
# Read the data into Pandas:
bhav_df = pd.read_csv(
raw_data,
header = None,
names = [
"symbol", "segment", "underlying", "expiry", "strike",
"type", "G?", "H?", "I?", "prevClose",
"open", "high", "low", "close", "O?",
"totVol", "totCash", "oi", "oiChg", "T?",
"U?", "date", "W?", "X?", "Y?",
"Z?"
]
)
# Format the data:
for column in ["symbol", "segment", "type", "underlying", "expiry", "date"]:
bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip())
for column in ["strike", "prevClose", "open", "high", "low", "close", "totCash"]:
bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x))
for column in ["totVol", "oi", "oiChg"]:
bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x))
# Handle segment, type and other normalizations:
bhav_df["isIndex"] = bhav_df["segment"].apply(lambda x: True if x.endswith("IDX") else False)
bhav_df["segment"] = bhav_df["segment"].apply(lambda x: x[:3])
bhav_df["type"] = bhav_df.apply(lambda x: None if x["segment"] == "FUT" else x["type"], axis = 1)
bhav_df["strike"] = bhav_df.apply(lambda x: None if x["segment"] == "FUT" else x["strike"], axis = 1)
# Give the expiry as a timestamp:
bhav_df["expiryTs"] = bhav_df["expiry"].apply(lambda x: self.parse_datetime_string(
dt_str = f"{int(x):0>8}",
dt_formats = ["%d%m%Y"],
input_tz = date_time.TIMEZONE_IST,
output_tz = date_time.TIMEZONE_UTC
))
bhav_df["expiryTz"] = "Asia/Kolkata"
# Give the data recording date as a timestamp:
bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string(
dt_str = x,
dt_formats = ["%d%m%Y", "%d-%b-%Y", "%d/%b/%Y"],
input_tz = date_time.TIMEZONE_IST,
output_tz = date_time.TIMEZONE_UTC
))
bhav_df["tz"] = "Asia/Kolkata"
bhav_df["scrapeTs"] = timestamp
# Keep what is needed, in an order that makes intuitive sense:
bhav_df = bhav_df[[
"symbol", "segment", "type", "underlying", "isIndex", "expiry", "expiryTs", "expiryTz", "strike",
"prevClose", "open", "high", "low", "close", "totVol", "totCash", "oi", "oiChg",
"date", "ts", "tz", "scrapeTs"
]]
# Done here:
formatted_data = bhav_df.to_dict(orient = "records")
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an HTTP client:
test_client = httpx.AsyncClient(
limits = httpx.Limits(
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
),
timeout = httpx.Timeout(
pool = 120.0, # .... Time to wait for a free connection from the pool.
connect = 2.5, # ... Time to wait for establishing a connection to the server.
write = 10.0, # .... Time to wait for sending data.
read = 2.5 # ....... Time to wait for receiving data.
)
)
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSEFNOBhavCopy(http_client = test_client)
# Get and show the data:
target_date = datetime.datetime.strptime("20250103", "%Y%m%d")
api_response = await my_nse.get_data(
target_date = target_date,
return_raw = False
)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str))
asyncio.run(main())
utils_v2/nse/controllers/calendar/__init__.py
View File
utils_v2/nse/controllers/calendar/corporate_actions.py
-282
View File
@@ -1,282 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 28th Nov., 2024
OBJECTIVE:
To provide a way to retrieve dates of important events like financial-results, stock-splits, fund-raising, etc.
from NSE's portal.
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
# data models:
from utils_v2.nse.models.calendar import NSECorporateAction
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
import pytz
# To work with datatypes:
from typing import Any, List
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSECorporateActionsCalendar(AsyncNSEBase):
# The types of corporate actions/entities:
KIND_EQUITIES = "equities"
KIND_MUTUAL_FUNDS = "mf"
KIND_DEBT = "debt"
KIND_SME = "sme"
def __init__(
self,
http_client: httpx.AsyncClient = None,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (C.Act.) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/companies-listing/corporate-filings-actions",
data_url = r"https://www.nseindia.com/api/corporates-corporateActions",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
async def get_data(
self,
kind: str = KIND_EQUITIES,
from_date: datetime.datetime = None,
to_date: datetime.datetime = None,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the corporate event calendar.
:param kind: The option between 'equities", "mf", "debt", and "sme".
:param from_date: The starting date (inclusive) from which the events must be fetched.
:param to_date: The ending date (inclusive) till which the events must be fetched.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Prepare the inputs:
now = date_time.get_current_utc_date_time(as_string = False)
from_date = date_time.to_timezone(from_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
to_date = date_time.to_timezone(to_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
# Make the API call:
api_response = await self.get(
params = {
"index": kind,
"from_date": from_date,
"to_date": to_date,
},
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw: api_response.data = await api_response.get_json()
else:
try: api_response.data = self.format_data(
raw_json = await api_response.get_json(),
kind = kind,
timestamp = date_time.get_current_utc_date_time(as_string = True),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
def format_data(
self,
raw_json: List[dict],
kind: str = None,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[NSECorporateAction] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_json: The raw data as scraped from NSE.
:param kind: The kind of data pulled using the 'get_data' method.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_json is None: return raw_json
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Format the data:
formatted_data = []
for action in raw_json:
formatted_data.append(NSECorporateAction(**{
"scrapeTs": timestamp,
"kind": kind,
"symbol": action["symbol"],
"name": action["comp"],
"isin": action["isin"],
"series": action["series"],
"ind": action["ind"],
"faceVal": float(action["faceVal"]),
"action": action["subject"],
"exTs": self.parse_datetime_string(action["exDate"], ["%d-%b-%Y"]),
"recTs": self.parse_datetime_string(action["recDate"]),
"bcStartTs": self.parse_datetime_string(action["bcStartDate"]),
"bcEndTs": self.parse_datetime_string(action["bcEndDate"]),
"ndStartTs": self.parse_datetime_string(action["ndStartDate"]),
"ndEndTs": self.parse_datetime_string(action["ndEndDate"]),
"caBroadcastTs": self.parse_datetime_string(action["caBroadcastDate"])
}))
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSECorporateActionsCalendar()
# Get and show the data:
now = date_time.get_current_ist_date_time()
to_date = now + datetime.timedelta(days = 31)
from_date = now - datetime.timedelta(days = 31)
print("FROM :", from_date)
print("TO :", to_date)
api_response = await my_nse.get_data(
kind = NSECorporateActionsCalendar.KIND_EQUITIES,
from_date = from_date,
to_date = to_date,
return_raw = False
)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("CORPORATE ACTIONS:", json.to_string(api_response.data, default = str))
else: raise api_response.exception
asyncio.run(main())
utils_v2/nse/controllers/calendar/corporate_events.py
-273
View File
@@ -1,273 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 28th Nov., 2024
OBJECTIVE:
To provide a way to retrieve dates of important events like financial-results, stock-splits, fund-raising, etc.
from NSE's portal.
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
# Data models:
from utils_v2.nse.models.calendar import NSECorporateEvent
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
# To work with datatypes:
from typing import Any, List
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSECorporateEventsCalendar(AsyncNSEBase):
# Index types:
KIND_EQUITIES = "equities"
KIND_SME = "sme"
def __init__(
self,
http_client: httpx.AsyncClient = None,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (CECal) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/companies-listing/corporate-filings-event-calendar",
data_url = r"https://www.nseindia.com/api/event-calendar",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
async def get_data(
self,
kind: str = KIND_EQUITIES,
from_date: datetime.datetime = None,
to_date: datetime.datetime = None,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the corporate event calendar.
:param kind: The type of index you want. A choice between "equities" and "sme"
:param from_date: The starting date (inclusive) from which the events must be fetched.
:param to_date: The ending date (inclusive) till which the events must be fetched.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Prepare the inputs:
now = date_time.get_current_utc_date_time(as_string = False)
from_date = date_time.to_timezone(from_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
to_date = date_time.to_timezone(to_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
# Make the API call:
api_response = await self.get(
params = {
"index": kind,
"from_date": from_date,
"to_date": to_date
},
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw: api_response.data = await api_response.get_json()
else:
try: api_response.data = self.format_data(
raw_json = await api_response.get_json(),
kind = kind,
timestamp = date_time.get_current_utc_date_time(as_string = False),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
@staticmethod
def format_data(
raw_json: List[dict],
kind: str = None,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[NSECorporateEvent] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_json: The raw data as scraped from NSE.
:param kind: The kind of data pulled using the 'get_data' method.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_json is None: return raw_json
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Format the data:
formatted_data = [
{
"scrapeTs": timestamp,
"kind": kind,
"symbol": event.get("symbol", event.get("bm_symbol")),
"name": event.get("company", event.get("sm_name")),
"ts": date_time.to_timezone(
date_time.parse_date_time(
event.get("date", event.get("bm_date")),
timezone = date_time.TIMEZONE_IST
),
timezone = date_time.TIMEZONE_UTC
),
"tz": "Asia/Kolkata",
"purpose": event.get("purpose", event.get("bm_purpose")),
"description": event.get("bm_desc"),
"attachment": event.get("attachment")
} for event in raw_json
]
formatted_data = [NSECorporateEvent(**d) for d in formatted_data]
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSECorporateEventsCalendar()
# Get and show the data:
api_response = await my_nse.get_data(
kind = NSECorporateEventsCalendar.KIND_EQUITIES,
from_date = datetime.datetime(year = 2024, month = 1, day = 1),
to_date = datetime.datetime(year = 2025, month = 1, day = 1),
return_raw = False
)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("CORPORATE EVENT CALENDAR:", json.to_string(api_response.data, default = str))
asyncio.run(main())
utils_v2/nse/controllers/calendar/examples/20241128_formatted_nse_corporate_event_calendar.json
-178
View File
@@ -1,178 +0,0 @@
[
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "AAREYDRUGS",
"company": "Aarey Drugs & Pharmaceuticals Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Fund Raising/Other business matters",
"brief": "To consider Fund Raising and other business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "ASTERDM",
"company": "Aster DM Healthcare Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider other business matters pertaining to Preferential Issue."
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "EROSMEDIA",
"company": "Eros International Media Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Financial Results/Other business matters",
"brief": "To consider and approve the financial results for the quarter and year ended March 31, 2024 and other business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "MUFIN",
"company": "Mufin Green Finance Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Fund Raising",
"brief": "To consider Fund Raising by Issue of Secured Unlisted Non Convertible Debenture."
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "ONMOBILE",
"company": "OnMobile Global Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Fund Raising",
"brief": "To consider Fund Raising"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "PRIVISCL",
"company": "Privi Speciality Chemicals Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider and Approve \"Privi Employee Stock Option Scheme - 2024\""
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "SOUTHBANK",
"company": "The South Indian Bank Limited",
"eventDate": "2024-11-28 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider, decide on exercise of call option of Bank s Non-convertible, fully paid-up, unsecured, perpetual, Basel III Compliant, Tier I Bonds, listed in BSE"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "BCG",
"company": "Brightcom Group Limited",
"eventDate": "2024-11-29 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider other business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "BCG",
"company": "Brightcom Group Limited",
"eventDate": "2024-11-29 18:07:00+00:00",
"eventType": "Financial Results",
"brief": "To consider and approve the financial results for the period ended Jun 30, 2024"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "SIKKO",
"company": "Sikko Industries Limited",
"eventDate": "2024-11-29 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider discuss and decide terms and conditions such as determination of the rights issue price, rights entitlement ratio, record date and other matters incidental orconnected therewith."
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "AGIIL",
"company": "Agi Infra Limited",
"eventDate": "2024-12-01 18:07:00+00:00",
"eventType": "Stock Split",
"brief": "To consider stock split of equity shares of the Company"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "MANAKCOAT",
"company": "Manaksia Coated Metals & Industries Limited",
"eventDate": "2024-12-01 18:07:00+00:00",
"eventType": "Fund Raising",
"brief": "To consider Fund Raising"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "NTL",
"company": "Neueon Towers Limited",
"eventDate": "2024-12-01 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider the enclosed business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "PAR",
"company": "Par Drugs and Chemicals Limited",
"eventDate": "2024-12-01 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider other business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "FIBERWEB",
"company": "Fiberweb (India) Limited",
"eventDate": "2024-12-02 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider and discuss about the expansion plans"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "IITL",
"company": "Industrial Investment Trust Limited",
"eventDate": "2024-12-02 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider other business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "SWIGGY",
"company": "Swiggy Limited",
"eventDate": "2024-12-02 18:07:00+00:00",
"eventType": "Financial Results",
"brief": "To consider and approve the financial results for the quarter and half year ended September 30, 2024"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "GBGLOBAL",
"company": "GB Global Limited",
"eventDate": "2024-12-03 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider other business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "MOTOGENFIN",
"company": "The Motor & General Finance Limited",
"eventDate": "2024-12-03 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "Intimation Regarding Independent Directors Meeting will be held on 04.12.2024."
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "DIL",
"company": "Debock Industries Limited",
"eventDate": "2024-12-04 18:07:00+00:00",
"eventType": "Financial Results",
"brief": "DIL : 05-Dec-2024 : The Company has informed the Exchange that a Board meeting to be held on November 27, 2024 has been re-scheduled. Further, the Company has informed the Exchange that the meeting of the Board of Directors of the Company will be held on December 05, 2024, To consider and approve the financial results for the period ended September 30, 2024"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "NTL",
"company": "Neueon Towers Limited",
"eventDate": "2024-12-08 18:07:00+00:00",
"eventType": "Other business matters",
"brief": "To consider enclosed business matters"
},
{
"ts": "2024-11-28 15:11:22.793824+05:30",
"symbol": "EXIDEIND",
"company": "Exide Industries Limited",
"eventDate": "2025-01-27 18:07:00+00:00",
"eventType": "Financial Results",
"brief": "To consider and approve the financial results for the period ended December 31, 2024"
}
]
utils_v2/nse/controllers/calendar/examples/20241128_raw_nse_corporate_event_calendar.json
-156
View File
@@ -1,156 +0,0 @@
[
{
"symbol": "AAREYDRUGS",
"company": "Aarey Drugs & Pharmaceuticals Limited",
"purpose": "Fund Raising/Other business matters",
"bm_desc": "To consider Fund Raising and other business matters",
"date": "29-Nov-2024"
},
{
"symbol": "ASTERDM",
"company": "Aster DM Healthcare Limited",
"purpose": "Other business matters",
"bm_desc": "To consider other business matters pertaining to Preferential Issue.",
"date": "29-Nov-2024"
},
{
"symbol": "EROSMEDIA",
"company": "Eros International Media Limited",
"purpose": "Financial Results/Other business matters",
"bm_desc": "To consider and approve the financial results for the quarter and year ended March 31, 2024 and other business matters",
"date": "29-Nov-2024"
},
{
"symbol": "MUFIN",
"company": "Mufin Green Finance Limited",
"purpose": "Fund Raising",
"bm_desc": "To consider Fund Raising by Issue of Secured Unlisted Non Convertible Debenture.",
"date": "29-Nov-2024"
},
{
"symbol": "ONMOBILE",
"company": "OnMobile Global Limited",
"purpose": "Fund Raising",
"bm_desc": "To consider Fund Raising",
"date": "29-Nov-2024"
},
{
"symbol": "PRIVISCL",
"company": "Privi Speciality Chemicals Limited",
"purpose": "Other business matters",
"bm_desc": "To consider and Approve \"Privi Employee Stock Option Scheme - 2024\"",
"date": "29-Nov-2024"
},
{
"symbol": "SOUTHBANK",
"company": "The South Indian Bank Limited",
"purpose": "Other business matters",
"bm_desc": "To consider, decide on exercise of call option of Bank s Non-convertible, fully paid-up, unsecured, perpetual, Basel III Compliant, Tier I Bonds, listed in BSE",
"date": "29-Nov-2024"
},
{
"symbol": "BCG",
"company": "Brightcom Group Limited",
"purpose": "Other business matters",
"bm_desc": "To consider other business matters",
"date": "30-Nov-2024"
},
{
"symbol": "BCG",
"company": "Brightcom Group Limited",
"purpose": "Financial Results",
"bm_desc": "To consider and approve the financial results for the period ended Jun 30, 2024",
"date": "30-Nov-2024"
},
{
"symbol": "SIKKO",
"company": "Sikko Industries Limited",
"purpose": "Other business matters",
"bm_desc": "To consider discuss and decide terms and conditions such as determination of the rights issue price, rights entitlement ratio, record date and other matters incidental orconnected therewith.",
"date": "30-Nov-2024"
},
{
"symbol": "AGIIL",
"company": "Agi Infra Limited",
"purpose": "Stock Split",
"bm_desc": "To consider stock split of equity shares of the Company",
"date": "02-Dec-2024"
},
{
"symbol": "MANAKCOAT",
"company": "Manaksia Coated Metals & Industries Limited",
"purpose": "Fund Raising",
"bm_desc": "To consider Fund Raising",
"date": "02-Dec-2024"
},
{
"symbol": "NTL",
"company": "Neueon Towers Limited",
"purpose": "Other business matters",
"bm_desc": "To consider the enclosed business matters",
"date": "02-Dec-2024"
},
{
"symbol": "PAR",
"company": "Par Drugs and Chemicals Limited",
"purpose": "Other business matters",
"bm_desc": "To consider other business matters",
"date": "02-Dec-2024"
},
{
"symbol": "FIBERWEB",
"company": "Fiberweb (India) Limited",
"purpose": "Other business matters",
"bm_desc": "To consider and discuss about the expansion plans",
"date": "03-Dec-2024"
},
{
"symbol": "IITL",
"company": "Industrial Investment Trust Limited",
"purpose": "Other business matters",
"bm_desc": "To consider other business matters",
"date": "03-Dec-2024"
},
{
"symbol": "SWIGGY",
"company": "Swiggy Limited",
"purpose": "Financial Results",
"bm_desc": "To consider and approve the financial results for the quarter and half year ended September 30, 2024",
"date": "03-Dec-2024"
},
{
"symbol": "GBGLOBAL",
"company": "GB Global Limited",
"purpose": "Other business matters",
"bm_desc": "To consider other business matters",
"date": "04-Dec-2024"
},
{
"symbol": "MOTOGENFIN",
"company": "The Motor & General Finance Limited",
"purpose": "Other business matters",
"bm_desc": "Intimation Regarding Independent Directors Meeting will be held on 04.12.2024.",
"date": "04-Dec-2024"
},
{
"symbol": "DIL",
"company": "Debock Industries Limited",
"purpose": "Financial Results",
"bm_desc": "DIL : 05-Dec-2024 : The Company has informed the Exchange that a Board meeting to be held on November 27, 2024 has been re-scheduled. Further, the Company has informed the Exchange that the meeting of the Board of Directors of the Company will be held on December 05, 2024, To consider and approve the financial results for the period ended September 30, 2024",
"date": "05-Dec-2024"
},
{
"symbol": "NTL",
"company": "Neueon Towers Limited",
"purpose": "Other business matters",
"bm_desc": "To consider enclosed business matters",
"date": "09-Dec-2024"
},
{
"symbol": "EXIDEIND",
"company": "Exide Industries Limited",
"purpose": "Financial Results",
"bm_desc": "To consider and approve the financial results for the period ended December 31, 2024",
"date": "28-Jan-2025"
}
]
utils_v2/nse/controllers/calendar/examples/20241205_formatted_nse_trading_holiday_calendar.json
-542
View File
@@ -1,542 +0,0 @@
[
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-01-21 18:30:00+00:00",
"weekDay": "Monday",
"event": "Special Holiday",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-01-25 18:30:00+00:00",
"weekDay": "Friday",
"event": "Republic Day",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-02-18 18:30:00+00:00",
"weekDay": "Monday",
"event": "Chatrapati Shivaji Maharaj Jayanti",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"IRD",
"NDM",
"NTRP"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-03-07 18:30:00+00:00",
"weekDay": "Friday",
"event": "Mahashivratri",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-03-24 18:30:00+00:00",
"weekDay": "Monday",
"event": "Holi",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-03-28 18:30:00+00:00",
"weekDay": "Friday",
"event": "Good Friday",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-03-31 18:30:00+00:00",
"weekDay": "Monday",
"event": "Annual Bank Closing",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"IRD",
"NDM",
"NTRP"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-04-08 18:30:00+00:00",
"weekDay": "Tuesday",
"event": "Gudi Padwa",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"IRD",
"NDM",
"NTRP"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-04-10 18:30:00+00:00",
"weekDay": "Thursday",
"event": "Id-Ul-Fitr (Ramadan Id)",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-04-13 18:30:00+00:00",
"weekDay": "Sunday",
"event": "Dr.Baba Saheb Ambedkar Jayanti",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-04-16 18:30:00+00:00",
"weekDay": "Wednesday",
"event": "Shri Ram Navami",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-04-20 18:30:00+00:00",
"weekDay": "Sunday",
"event": "Mahavir Jayanti",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-04-30 18:30:00+00:00",
"weekDay": "Wednesday",
"event": "Maharashtra Day",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-05-19 18:30:00+00:00",
"weekDay": "Monday",
"event": "General Parliamentary Elections",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-05-22 18:30:00+00:00",
"weekDay": "Thursday",
"event": "Buddha Pournima",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"IRD",
"NDM",
"NTRP"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-06-16 18:30:00+00:00",
"weekDay": "Monday",
"event": "Bakri Id",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-07-16 18:30:00+00:00",
"weekDay": "Wednesday",
"event": "Muharram",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-08-14 18:30:00+00:00",
"weekDay": "Thursday",
"event": "Independence Day/Parsi New Year",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-09-06 18:30:00+00:00",
"weekDay": "Saturday",
"event": "Ganesh Chaturthi",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-09-15 18:30:00+00:00",
"weekDay": "Monday",
"event": "Id-E-Milad",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"IRD",
"NDM",
"NTRP"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-10-01 18:30:00+00:00",
"weekDay": "Wednesday",
"event": "Mahatma Gandhi Jayanti",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-10-11 18:30:00+00:00",
"weekDay": "Saturday",
"event": "Dussehra",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-11-01 18:30:00+00:00",
"weekDay": "Saturday",
"event": "Balipratipada",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-11-14 18:30:00+00:00",
"weekDay": "Friday",
"event": "Prakash Gurpurb Sri Guru Nanak Dev",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-11-19 18:30:00+00:00",
"weekDay": "Wednesday",
"event": "Assembly Elections in Maharashtra",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-12-24 18:30:00+00:00",
"weekDay": "Wednesday",
"event": "Christmas",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"CBM",
"CD",
"CM",
"CMOT",
"COM",
"FO",
"IRD",
"MF",
"NDM",
"NTRP",
"SLBS"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2023-12-31 18:30:00+00:00",
"weekDay": "Monday",
"event": "New year",
"morningSession": "Open",
"eveningSession": "Closed",
"holidayType": [
"COM"
]
},
{
"scrapeTs": "2024-12-05T04:19:17.317730+00:00",
"date": "2024-10-31 18:30:00+00:00",
"weekDay": "Friday",
"event": "Diwali Laxmi Pujan",
"morningSession": null,
"eveningSession": null,
"holidayType": [
"MF",
"NDM",
"NTRP"
]
}
]
utils_v2/nse/controllers/calendar/examples/20241205_raw_nse_trading_holiday_calendar.json
-2104
View File
File diff suppressed because it is too large Load Diff
utils_v2/nse/controllers/calendar/holidays.py
-323
View File
@@ -1,323 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 5th Dec., 2024
OBJECTIVE:
To provide a way to retrieve the list of holidays from NSE and to check if today is a holiday.
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
# Data models:
from utils_v2.nse.models.calendar import NSEHoliday
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
# To work with datatypes:
from typing import Any, List
# For asynchronous activities:
import asyncio
# For debugging:
import inspect
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSEHolidaysCalendar(AsyncNSEBase):
def __init__(
self,
http_client: httpx.AsyncClient = None,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (Hol) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/resources/exchange-communication-holidays",
data_url = r"https://www.nseindia.com/api/holiday-master?type=trading",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
async def get_data(
self,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the holiday calendar.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Make the API call:
api_response = await self.get(
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw: api_response.data = await api_response.get_json()
else:
try: api_response.data = self.format_data(
raw_json = await api_response.get_json(),
timestamp = date_time.get_current_utc_date_time(as_string = False),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
@staticmethod
def format_data(
raw_json: dict,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[NSEHoliday] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_json: The raw data as scraped from NSE.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_json is None: return raw_json
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Start by creating an empty dict:
formatted_data = {}
# Process each category in the original data
for category, events in raw_json.items():
for event in events:
trading_date = event["tradingDate"]
# If the tradingDate is not in the dictionary, initialize it
if trading_date not in formatted_data:
formatted_data[trading_date] = {
"scrapeTs": timestamp,
"ts": date_time.to_timezone(
date_time.as_if_timezone(
datetime.datetime.strptime(trading_date, "%d-%b-%Y"),
timezone = date_time.TIMEZONE_IST
),
timezone = date_time.TIMEZONE_UTC
),
"tz": "Asia/Kolkata",
"weekday": event["weekDay"],
"event": event["description"],
"type": []
}
# Add the current category to the "type" list
formatted_data[trading_date]["type"].append(category)
# Now we get rid of the unnecessary keys and make it a list, and model it:
formatted_data = [v for v in formatted_data.values()]
formatted_data = [NSEHoliday(**d) for d in formatted_data]
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
async def is_holiday(
self,
timestamp: datetime.datetime
) -> NSEApiResponse:
"""
Checks if a given date is a trading holiday, or not.
:param timestamp: The datetime instance that you want to check for it being a holiday. Preferably make it an
aware instance. If a naive instance is passed, it will be assumed to be in UTC.
:return: True if it is a holiday, False if it isn't. Can be None if something goes wrong in fetching the data.
The value will be in the 'data' field of the response model.
"""
# We first prepare UTC and IST versions of the incoming datetime:
timestamp_utc = date_time.to_timezone(timestamp, timezone = date_time.TIMEZONE_UTC)
timestamp_ist = date_time.to_timezone(timestamp_utc, timezone = date_time.TIMEZONE_IST)
# If this day is a weekend, it is a holiday by default:
ist_dow = timestamp_ist.weekday()
if ist_dow in [5, 6]: return NSEApiResponse(
action = inspect.stack()[0].function,
url = None,
method = None,
success = True,
data = {
"isHoliday": True,
"event": "Saturday" if ist_dow == 5 else "Sunday"
},
message = "Holiday due to weekend."
)
# First we fetch the data from NSE:
api_response = await self.get_data(return_raw = False)
api_response.action = inspect.stack()[0].function
if not api_response.success: return api_response
# Now we check if our date to check is in the list:
is_holiday = False
event = None
for h in api_response.data:
if h.ts.date() == timestamp_utc.date():
is_holiday = True
event = h.event
break
api_response.data = {
"isHoliday": is_holiday,
"event": event
}
# Done here:
return api_response
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSEHolidaysCalendar()
# Get and show the data:
api_response = await my_nse.get_data(return_raw = False)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("HOLIDAY CALENDAR:", json.to_string(api_response.data, default = str))
ts = api_response.data[0].ts
print(ts, type(ts))
print("\n---\n\n")
# # Check for a holiday:
# date_to_check = date_time.parse_date_time(
# input_value = "2025-12-25 00:00:00",
# date_formats = ["%Y-%m-%d %H:%M:%S"],
# timezone = date_time.TIMEZONE_IST
# )
# api_response = await my_nse.is_holiday(date_to_check)
# print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
# print("DATETIME:", date_to_check)
# print("IS HOLIDAY:", api_response.data)
asyncio.run(main())
utils_v2/nse/controllers/indices/__init__.py
View File
utils_v2/nse/controllers/indices/constituents.py
-378
View File
@@ -1,378 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Monday, 16th Dec., 2024
OBJECTIVE:
This file aims to fetch the current details about the constituents of various indices. This gives you not only
the constituent stocks of the selected index, but also that stock's current activity in the market.
https://www.nseindia.com/market-data/live-equity-market?symbol=NIFTY%2050
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
# To work with datatypes:
from typing import Any, List
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSEIndexConstituents(AsyncNSEBase):
# Broad Market Indices:
INDEX_NIFTY_50 = "NIFTY 50"
INDEX_NIFTY_NEXT_50 = "NIFTY NEXT 50"
INDEX_NIFTY_MIDCAP_50 = "NIFTY MIDCAP 50"
INDEX_NIFTY_MIDCAP_100 = "NIFTY MIDCAP 100"
INDEX_NIFTY_MIDCAP_150 = "NIFTY MIDCAP 150"
INDEX_NIFTY_SMALLCAP_50 = "NIFTY SMALLCAP 50"
INDEX_NIFTY_SMALLCAP_100 = "NIFTY SMALLCAP 100"
INDEX_NIFTY_SMALLCAP_250 = "NIFTY SMALLCAP 250"
INDEX_NIFTY_MIDSMALLCAP_400 = "NIFTY MIDSMALLCAP 400"
INDEX_NIFTY_100 = "NIFTY 100"
INDEX_NIFTY_200 = "NIFTY 200"
INDEX_NIFTY_500_MULTICAP_50_25_25 = "NIFTY500 MULTICAP 50:25:25"
INDEX_NIFTY_LARGEMIDCAP_250 = "NIFTY LARGEMIDCAP 250"
INDEX_NIFTY_MIDCAP_SELECT = "NIFTY MIDCAP SELECT"
INDEX_NIFTY_TOTAL_MARKET = "NIFTY TOTAL MARKET"
INDEX_NIFTY_MICROCAP_250 = "NIFTY MICROCAP 250"
INDEX_NIFTY_500 = "NIFTY 500"
INDEX_NIFTY_500_LARGEMIDSMALL_EQUAL_CAP_WEIGHTED = "NIFTY500 LARGEMIDSMALL EQUAL-CAP WEIGHTED"
# Sectoral Indices:
INDEX_NIFTY_AUTO = "NIFTY AUTO"
INDEX_NIFTY_BANK = "NIFTY BANK"
INDEX_NIFTY_ENERGY = "NIFTY ENERGY"
INDEX_NIFTY_FINANCIAL_SERVICES = "NIFTY FINANCIAL SERVICES"
INDEX_NIFTY_FINANCIAL_SERVICES_25_50 = "NIFTY FINANCIAL SERVICES 25/50"
INDEX_NIFTY_FMCG = "NIFTY FMCG"
INDEX_NIFTY_IT = "NIFTY IT"
INDEX_NIFTY_MEDIA = "NIFTY MEDIA"
INDEX_NIFTY_METAL = "NIFTY METAL"
INDEX_NIFTY_PHARMA = "NIFTY PHARMA"
INDEX_NIFTY_PSU_BANK = "NIFTY PSU BANK"
INDEX_NIFTY_REALTY = "NIFTY REALTY"
INDEX_NIFTY_PRIVATE_BANK = "NIFTY PRIVATE BANK"
INDEX_NIFTY_HEALTHCARE_INDEX = "NIFTY HEALTHCARE INDEX"
INDEX_NIFTY_CONSUMER_DURABLES = "NIFTY CONSUMER DURABLES"
INDEX_NIFTY_OIL_GAS = "NIFTY OIL & GAS"
INDEX_NIFTY_MIDSMALL_HEALTHCARE = "NIFTY MIDSMALL HEALTHCARE"
INDEX_NIFTY_FINANCIAL_SERVICES_EX_BANK = "NIFTY FINANCIAL SERVICES EX-BANK"
INDEX_NIFTY_MIDSMALL_FINANCIAL_SERVICES = "NIFTY MIDSMALL FINANCIAL SERVICES"
INDEX_NIFTY_MIDSMALL_IT_TELECOM = "NIFTY MIDSMALL IT & TELECOM"
# Thematic:
INDEX_NIFTY_COMMODITIES = "NIFTY COMMODITIES"
INDEX_NIFTY_INDIA_CONSUMPTION = "NIFTY INDIA CONSUMPTION"
INDEX_NIFTY_CPSE = "NIFTY CPSE"
INDEX_NIFTY_INFRASTRUCTURE = "NIFTY INFRASTRUCTURE"
INDEX_NIFTY_MNC = "NIFTY MNC"
INDEX_NIFTY_GROWTH_SECTORS_15 = "NIFTY GROWTH SECTORS 15"
INDEX_NIFTY_PSE = "NIFTY PSE"
INDEX_NIFTY_SERVICES_SECTOR = "NIFTY SERVICES SECTOR"
INDEX_NIFTY_100_LIQUID_15 = "NIFTY100 LIQUID 15"
INDEX_NIFTY_MIDCAP_LIQUID_15 = "NIFTY MIDCAP LIQUID 15"
INDEX_NIFTY_INDIA_DIGITAL = "NIFTY INDIA DIGITAL"
INDEX_NIFTY_100_ESG = "NIFTY100 ESG"
INDEX_NIFTY_INDIA_MANUFACTURING = "NIFTY INDIA MANUFACTURING"
INDEX_NIFTY_INDIA_CORPORATE_GROUP_INDEX_TATA_GROUP_25_CAP = "NIFTY INDIA CORPORATE GROUP INDEX - TATA GROUP 25% CAP"
INDEX_NIFTY_500_MULTICAP_INDIA_MANUFACTURING_50_30_20 = "NIFTY500 MULTICAP INDIA MANUFACTURING 50:30:20"
INDEX_NIFTY_500_MULTICAP_INFRASTRUCTURE_50_30_20 = "NIFTY500 MULTICAP INFRASTRUCTURE 50:30:20"
INDEX_NIFTY_INDIA_DEFENCE = "NIFTY INDIA DEFENCE"
INDEX_NIFTY_INDIA_TOURISM = "NIFTY INDIA TOURISM"
INDEX_NIFTY_CAPITAL_MARKETS = "NIFTY CAPITAL MARKETS"
INDEX_NIFTY_EV_NEW_AGE_AUTOMOTIVE = "NIFTY EV & NEW AGE AUTOMOTIVE"
INDEX_NIFTY_INDIA_NEW_AGE_CONSUMPTION = "NIFTY INDIA NEW AGE CONSUMPTION"
INDEX_NIFTY_INDIA_SELECT_5_CORPORATE_GROUPS_MAATR = "NIFTY INDIA SELECT 5 CORPORATE GROUPS (MAATR)"
INDEX_NIFTY_MOBILITY = "NIFTY MOBILITY"
INDEX_NIFTY_100_ENHANCED_ESG = "NIFTY100 ENHANCED ESG"
INDEX_NIFTY_CORE_HOUSING = "NIFTY CORE HOUSING"
INDEX_NIFTY_HOUSING = "NIFTY HOUSING"
INDEX_NIFTY_IPO = "NIFTY IPO"
INDEX_NIFTY_MIDSMALL_INDIA_CONSUMPTION = "NIFTY MIDSMALL INDIA CONSUMPTION"
INDEX_NIFTY_NON_CYCLICAL_CONSUMER = "NIFTY NON-CYCLICAL CONSUMER"
INDEX_NIFTY_RURAL = "NIFTY RURAL"
INDEX_NIFTY_SHARIAH_25 = "NIFTY SHARIAH 25"
INDEX_NIFTY_TRANSPORTATION_LOGISTICS = "NIFTY TRANSPORTATION & LOGISTICS"
INDEX_NIFTY_50_SHARIAH = "NIFTY50 SHARIAH"
INDEX_NIFTY_500_SHARIAH = "NIFTY500 SHARIAH"
# Strategy Indices:
INDEX_NIFTY_DIVIDEND_OPPORTUNITIES_50 = "NIFTY DIVIDEND OPPORTUNITIES 50"
INDEX_NIFTY_50_VALUE_20 = "NIFTY50 VALUE 20"
INDEX_NIFTY_100_QUALITY_30 = "NIFTY100 QUALITY 30"
INDEX_NIFTY_50_EQUAL_WEIGHT = "NIFTY50 EQUAL WEIGHT"
INDEX_NIFTY_100_EQUAL_WEIGHT = "NIFTY100 EQUAL WEIGHT"
INDEX_NIFTY_100_LOW_VOLATILITY_30 = "NIFTY100 LOW VOLATILITY 30"
INDEX_NIFTY_ALPHA_50 = "NIFTY ALPHA 50"
INDEX_NIFTY_200_QUALITY_30 = "NIFTY200 QUALITY 30"
INDEX_NIFTY_ALPHA_LOW_VOLATILITY_30 = "NIFTY ALPHA LOW-VOLATILITY 30"
INDEX_NIFTY_200_MOMENTUM_30 = "NIFTY200 MOMENTUM 30"
INDEX_NIFTY_MIDCAP_150_QUALITY_50 = "NIFTY MIDCAP150 QUALITY 50"
INDEX_NIFTY_200_ALPHA_30 = "NIFTY200 ALPHA 30"
INDEX_NIFTY_MIDCAP_150_MOMENTUM_50 = "NIFTY MIDCAP150 MOMENTUM 50"
INDEX_NIFTY_500_MOMENTUM_50 = "NIFTY500 MOMENTUM 50"
INDEX_NIFTY_MIDSMALLCAP_400_MOMENTUM_QUALITY_100 = "NIFTY MIDSMALLCAP400 MOMENTUM QUALITY 100"
INDEX_NIFTY_SMALLCAP_250_MOMENTUM_QUALITY_100 = "NIFTY SMALLCAP250 MOMENTUM QUALITY 100"
INDEX_NIFTY_TOP_10_EQUAL_WEIGHT = "NIFTY TOP 10 EQUAL WEIGHT"
INDEX_NIFTY_ALPHA_QUALITY_LOW_VOLATILITY_30 = "NIFTY ALPHA QUALITY LOW-VOLATILITY 30"
INDEX_NIFTY_ALPHA_QUALITY_VALUE_LOW_VOLATILITY_30 = "NIFTY ALPHA QUALITY VALUE LOW-VOLATILITY 30"
INDEX_NIFTY_HIGH_BETA_50 = "NIFTY HIGH BETA 50"
INDEX_NIFTY_LOW_VOLATILITY_50 = "NIFTY LOW VOLATILITY 50"
INDEX_NIFTY_QUALITY_LOW_VOLATILITY_30 = "NIFTY QUALITY LOW-VOLATILITY 30"
INDEX_NIFTY_SMALLCAP_250_QUALITY_50 = "NIFTY SMALLCAP250 QUALITY 50"
INDEX_NIFTY_TOP_15_EQUAL_WEIGHT = "NIFTY TOP 15 EQUAL WEIGHT"
INDEX_NIFTY_100_ALPHA_30 = "NIFTY100 ALPHA 30"
INDEX_NIFTY_200_VALUE_30 = "NIFTY200 VALUE 30"
INDEX_NIFTY_500_EQUAL_WEIGHT = "NIFTY500 EQUAL WEIGHT"
INDEX_NIFTY_500_MULTICAP_MOMENTUM_QUALITY_50 = "NIFTY500 MULTICAP MOMENTUM QUALITY 50"
INDEX_NIFTY_500_VALUE_50 = "NIFTY500 VALUE 50"
INDEX_NIFTY_TOP_20_EQUAL_WEIGHT = "NIFTY TOP 20 EQUAL WEIGHT"
# Others:
INDEX_SECURITIES_IN_FNO = "SECURITIES IN F&O"
INDEX_PERMITTED_TO_TRADE = "PERMITTED TO TRADE"
def __init__(
self,
http_client: httpx.AsyncClient = None,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (IdxCons) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/market-data/live-equity-market",
data_url = r"https://www.nseindia.com/api/equity-stockIndices",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
async def get_data(
self,
index_name: str,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the corporate event calendar.
:param index_name: The value held in the 'indexName' field of the formatted output of the Index Master.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Make the API call:
api_response = await self.get(
params = {"index": index_name},
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw: api_response.data = await api_response.get_json()
else:
try: api_response.data = self.format_data(
raw_json = await api_response.get_json(),
index_name = index_name,
timestamp = date_time.get_current_utc_date_time(as_string = False),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
def format_data(
self,
raw_json: dict,
index_name: str,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[dict] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_json: The raw data as scraped from NSE.
:param index_name: The value that you had used to fetch the raw data in the first place.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_json is None: return raw_json
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Format the data:
formatted_data = [
{
"scrapeTs": timestamp,
"ts": self.parse_datetime_string(
dt_str = symbol.get("lastUpdateTime"),
dt_formats = ["%d-%b-%Y %H:%M:%S"],
input_tz = date_time.TIMEZONE_IST,
output_tz = date_time.TIMEZONE_UTC
),
"indexName": index_name,
"symbol": symbol["symbol"],
"name": symbol.get("meta", {}).get("companyName"),
"industry": symbol.get("meta", {}).get("industry"),
"isFNOSec": symbol.get("meta", {}).get("isFNOSec"),
"isSuspended": symbol.get("meta", {}).get("isSuspended"),
"isin": symbol.get("meta", {}).get("isin"),
"open": symbol["open"],
"high": symbol["dayHigh"],
"low": symbol["dayLow"],
"close": symbol["lastPrice"],
"totTradedVol": symbol["totalTradedVolume"],
"totTradedVal": symbol["totalTradedValue"],
"prevClose": symbol["previousClose"],
"chg": symbol["change"],
"pChg": symbol["pChange"],
"yearHigh": symbol["yearHigh"],
"yearLow": symbol["yearLow"],
"pChg30d": symbol.get("perChange30d"),
"pChg365d": symbol.get("perChange365d"),
"ffmc": symbol.get("ffmc")
} for symbol in raw_json["data"] if symbol.get("priority") in [0, None]
]
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSEIndexConstituents()
# Get and show the data:
api_response = await my_nse.get_data(
index_name = NSEIndexConstituents.INDEX_NIFTY_500,
return_raw = False
)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("INDEX CONSTITUENTS:", json.to_string(api_response.data, default = str))
asyncio.run(main())
utils_v2/nse/controllers/indices/examples/20241128_formatted_nse_index_master.json
-2301
View File
File diff suppressed because it is too large Load Diff
utils_v2/nse/controllers/indices/examples/20241128_raw_nse_index_master.json
-2785
View File
File diff suppressed because it is too large Load Diff
utils_v2/nse/controllers/indices/examples/20241216_formatted_nse_index_constituent.json
-1252
View File
File diff suppressed because it is too large Load Diff
utils_v2/nse/controllers/indices/examples/20241216_raw_nse_index_constituent.json
-2963
View File
File diff suppressed because it is too large Load Diff
utils_v2/nse/controllers/indices/master.py
-284
View File
@@ -1,284 +0,0 @@
"""
AUTHOR:
Khushal P Soonderji
DATE:
Thursday, 28th Nov., 2024
OBJECTIVE:
The "Index Master" contains information about just the indices, and not the component symbols of those indices.
This script provides a way to get the data that is available on the screen on the following URL:
https://www.nseindia.com/market-data/live-market-indices
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
more professional data-sources be used when the product starts becoming mature.
REFERENCES:
N/A
DOWNLOADS:
N/A
"""
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
# ***** ****
# *****************************************************************************************************************
# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")
# System-level activities:
import io
# My utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.date_time import date_time
# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse
from utils_v2.nse.models.indices import NSEIndexInfo
# To make REST-ful API calls:
import httpx
# To work with date and time:
import datetime
# To work with datatypes:
from typing import Any, List
# For asynchronous activities:
import asyncio
# *****************************************************************************************************************
# ***** ****
# *** MACROS / ONE-TIME INIT ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** VARIABLES ***
# ***** ****
# *****************************************************************************************************************
# --- Nothing Yet
# *****************************************************************************************************************
# ***** ****
# *** FUNCTIONS ***
# ***** ****
# *****************************************************************************************************************
class NSEIndexMaster(AsyncNSEBase):
# Index Types:
INDEX_TYPE_BROAD_MARKET = "BROAD MARKET INDICES"
INDEX_TYPE_SECTORAL = "SECTORAL INDICES"
INDEX_TYPE_STRATEGY = "STRATEGY INDICES"
INDEX_TYPE_THEMATIC = "THEMATIC INDICES"
INDEX_TYPE_FIXED_INCOME = "FIXED INCOME INDICES"
def __init__(
self,
http_client: httpx.AsyncClient,
cookies_refresh_interval: int | float = 300,
debug = True,
debug_prefix = "NSE (IdxMstr) | ",
debug_only_errors = True
):
# Pass on the initialization to the parent:
super().__init__(
base_url = r"https://www.nseindia.com/market-data/live-market-indices",
data_url = r"https://www.nseindia.com/api/allIndices",
http_client = http_client,
cookies_refresh_interval = cookies_refresh_interval,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
)
async def get_data(
self,
return_raw: bool = False,
refresh_cookies: bool = True,
force_refresh_cookies: bool = False,
retry_count: int = 1,
backoff_seconds: int | float = 0.5,
backoff_multiplier: float = 1.1
) -> NSEApiResponse:
"""
To get the data of the corporate event calendar.
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
successful refresh.
:param retry_count: The no. of times to try to get the data from the API.
:param backoff_seconds: The delay between unsuccessful API calls.
:param backoff_multiplier: The multiplier to add to the delay to change delay.
:return: The raw or formatted event calendar data in the 'data' field of the response model.
"""
# Make the API call:
api_response = await self.get(
refresh_cookies = refresh_cookies,
force_refresh_cookies = force_refresh_cookies,
retry_count = retry_count,
backoff_seconds = backoff_seconds,
backoff_multiplier = backoff_multiplier
)
# If the API call was successful:
if api_response.httpCode in [200]:
api_response.success = True
if return_raw: api_response.data = await api_response.get_json()
else:
try: api_response.data = self.format_data(
raw_json = await api_response.get_json(),
timestamp = date_time.get_current_utc_date_time(as_string = False),
raise_exception = True
)
except Exception as exception:
api_response.exception = exception
api_response.success = False
# Done here:
return api_response
@staticmethod
def parse_float(value: str | float | int) -> float | None:
"""
Parses a value as a floating point figure.
:param value: Any input value.
:return: A floating point value if parsed successfully, else None.
"""
if value is None: return None
if isinstance(value, float): return value
if isinstance(value, int): return float(value)
if isinstance(value, str):
value = regex.find_first(text = value.strip(), pattern = r"[\d,]+\.?[\d,]*")
if value: return float(value.replace(",", ""))
else: return None
def format_data(
self,
raw_json: dict,
timestamp: datetime.datetime = None,
raise_exception: bool = False
) -> List[NSEIndexInfo] | None:
"""
We format the data here to be able to retrieve it properly later.
:param raw_json: The raw data as scraped from NSE.
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
database, later.
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
suppressed internally.
:return: The formatted data if successful, else None.
"""
# Can't do anything if the chain itself is null:
if raw_json is None: return raw_json
# Start by assuming failure:
formatted_data = None
# Ensure that we've got a proper timestamp:
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
try:
# Format the data:
formatted_data = [
NSEIndexInfo(
scrapeTs = timestamp,
kind = idx["key"],
name = idx["index"],
symbol = idx["indexSymbol"],
open = self.parse_float(idx["open"]),
high = self.parse_float(idx["high"]),
low = self.parse_float(idx["low"]),
close = self.parse_float(idx["last"]),
prevClose = self.parse_float(idx["previousClose"]),
pChg = self.parse_float(idx["percentChange"]),
yearHigh = self.parse_float(idx["yearHigh"]),
yearLow = self.parse_float(idx["yearLow"]),
advances = self.parse_float(idx.get("advances")),
declines = self.parse_float(idx.get("declines")),
unchanged = self.parse_float(idx.get("unchanged")),
pChg30d = self.parse_float(idx["perChange30d"]),
pChg365d = self.parse_float(idx["perChange365d"])
) for idx in raw_json["data"]
]
# If something goes wrong:
except Exception as exception:
formatted_data = None
if raise_exception: raise
# Done here:
return formatted_data
# *****************************************************************************************************************
# ***** ****
# *** MAIN PROGRAM ***
# ***** ****
# *****************************************************************************************************************
if __name__ == "__main__":
async def main():
# Create an HTTP client:
test_client = httpx.AsyncClient(
limits = httpx.Limits(
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
),
timeout = httpx.Timeout(
pool = 120.0, # .... Time to wait for a free connection from the pool.
connect = 2.5, # ... Time to wait for establishing a connection to the server.
write = 10.0, # .... Time to wait for sending data.
read = 2.5 # ....... Time to wait for receiving data.
)
)
# Create an instance of the scraper, and refresh its cookies:
my_nse = NSEIndexMaster(http_client = test_client)
# Get and show the data:
api_response = await my_nse.get_data(return_raw = False)
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
if api_response.success: print("INDEX MASTER:", json.to_string(api_response.data, default = str))
asyncio.run(main())
utils_v2/nse/controllers/ipo/__init__.py
View File

Some files were not shown because too many files have changed in this diff Show More