Squashed 'utils_v2/' content from commit 8bb584e4

git-subtree-dir: utils_v2
git-subtree-split: 8bb584e4734606740c0b42d51bc7fd458c39031a

nse/controllers/base.py

@@ -0,0 +1,443 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a base class for common behaviour of NSE's APIs.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+import asyncio
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# For working with datatypes:
+from typing import Literal, List
+
+# For debugging:
+from icecream import IceCreamDebugger
+import inspect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   CLASSES                                                     ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class AsyncNSEBase:
+
+    def __init__(
+            self,
+            base_url: str,
+            data_url: str,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE | ",
+            debug_only_errors = True
+    ):
+
+        """
+        To be used as a base class for all sorts of NSE scraping.
+        :param base_url: The base URL (that you open in your browser) for your targeted module.
+        :param data_url: The internal URL (that NSE's script calls by itself) for your targeted module.
+        :param http_client: An HTTP client to use to make API calls.
+        :param cookies_refresh_interval: The no. of seconds after which the cookies should get refreshed.
+        :param debug: Whether, or not, you want to show debugging messages.
+        :param debug_prefix: The prefix string to use to recognize the module that is printing the debugging text.
+        :param debug_only_errors: Whether you want to show all debugging messages, or just error messages.
+        """
+
+        # Prepare the debugging utility:
+        self._debug_prefix = debug_prefix
+        self._printer = IceCreamDebugger(prefix = debug_prefix, includeContext = True)
+        if not debug: self._printer.disable()
+        self._debug_only_errors = debug_only_errors
+
+        # Accept the input configuration:
+        self._base_url = base_url
+        self._data_url = data_url
+        self._http_client = http_client or httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 5.0,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 60.0  # ...... Time to wait for receiving data.
+            )
+        )
+
+        # Make some provisions for later:
+        self._cookies = None
+        self._cookies_refresh_interval = cookies_refresh_interval
+        self._last_cookies_refresh = datetime.datetime.fromtimestamp(0, tz = date_time.TIMEZONE_UTC)
+
+        # For locking asynchronous tasks:
+        self._exclusive_lock = asyncio.Semaphore(1)
+
+    def enable_debug(self):
+        self._printer.enable()
+
+    def disable_debug(self):
+        self._printer.disable()
+
+    def debug_only_errors(self):
+        self._debug_only_errors = True
+
+    def debug_everything(self):
+        self._debug_only_errors = False
+
+    # ┏┓           •
+    # ┃┃┏┓┏┓┏┓┏┓┏┓╋┓┏┓┏
+    # ┣┛┛ ┗┛┣┛┗ ┛ ┗┗┗ ┛
+    #       ┛
+
+    @property
+    def cookies(self):
+        return self._cookies
+
+    # ┓┏  ┓
+    # ┣┫┏┓┃┏┓┏┓┏┓┏
+    # ┛┗┗ ┗┣┛┗ ┛ ┛
+    #      ┛
+
+    @staticmethod
+    def parse_datetime_string(
+            dt_str: str,
+            dt_formats: str | List[str] = None,
+            input_tz: str = date_time.TIMEZONE_IST,
+            output_tz: str = date_time.TIMEZONE_UTC
+    ) -> datetime.datetime | None:
+
+        """
+        Handles the date-time conversion and normalization for strings received from NSE.
+        :param dt_str: The string value as received from NSE.
+        :param dt_formats: The format(s) in which to attempt to parse the date string.
+        :param input_tz: The timezone in which the input must be assumed to be.
+        :param output_tz: The timezone the output must be converted to.
+        :return: The parsed datetime object, or null if parsing fails.
+        """
+
+        # Basic input formatting:
+        dt_formats = dt_formats or ["%d-%b-%Y"]
+
+        # Try to parse the date-time string:
+        if isinstance(dt_formats, list):
+            dt_obj = date_time.parse_date_time(
+                input_value = dt_str,
+                date_formats = dt_formats
+            )
+        else:
+            try: dt_obj = datetime.datetime.strptime(dt_str, dt_formats)
+            except: dt_obj = None
+
+        # If the parsing failed:
+        if not dt_obj: return None
+
+        # If parsed successfully:
+        return date_time.to_timezone(
+            datetime_object = date_time.as_if_timezone(
+                datetime_object = dt_obj,
+                timezone = input_tz
+            ),
+            timezone = output_tz
+        )
+
+    # ┏┓    ┓ •     ┳┳┓  ┓ •
+    # ┃ ┏┓┏┓┃┏┓┏┓┏  ┃┃┃┏┓┃┏┓┏┓┏┓
+    # ┗┛┗┛┗┛┛┗┗┗ ┛  ┛ ┗┗┻┛┗┗┛┗┗┫
+    #                          ┛
+
+    @property
+    def seconds_since_cookies_refreshed(self):
+        return (date_time.get_current_utc_date_time() - self._last_cookies_refresh).total_seconds()
+
+    async def refresh_cookies(
+            self,
+            force_refresh = True,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> bool:
+
+        """
+        To get cookies for the given base URL. NSE has some sort of strict cookies and origin policy that is beyond my
+        current understanding. But I have noticed that if you furnish the right base URL and then use that, NSE will
+        respond just fine.
+        USAGE: Call this right after you initialize your class and then call it every so often after that.
+        :param force_refresh: Set to True to ignore the time criterion and refresh the cookie even if it is new.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The cookies in the 'data' field of the
+        """
+
+        # Start by assuming failure:
+        success = False
+        http_code = None
+
+        # Ensure that only one cookie-refresh attempt is being made at a time:
+        async with self._exclusive_lock:
+
+            # We refresh the cookies only if the time has elapsed,
+            # or we have been asked to forcefully refresh the cookies:
+            if (
+                    force_refresh or
+                    self._cookies is None or
+                    self.seconds_since_cookies_refreshed > self._cookies_refresh_interval
+            ):
+
+                # Make the API call:
+                api_response = await self.get(
+                    url = self._base_url,
+                    headers = {
+                        "Accept": "*/*",
+                        "Accept-Encoding": "gzip, deflate, br",
+                        "Accept-Language": "en-US,en;q=0.9",
+                        "Upgrade-Insecure-Requests": "1",
+                        "Sec-Ch-Ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"128\"",
+                        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+                    },
+                    refresh_cookies = False,
+                    force_refresh_cookies = False,
+                    retry_count = retry_count,
+                    backoff_seconds = backoff_seconds,
+                    backoff_multiplier = backoff_multiplier
+                )
+                http_code = api_response.httpCode
+
+                # If the API call succeeds:
+                if api_response.httpCode in [200]:
+                    self._last_cookies_refresh = date_time.get_current_utc_date_time(as_string = False)
+                    self._cookies = api_response.response.cookies
+                    success = True
+
+        # Done here:
+        if not success or not self._debug_only_errors: self._printer(success, http_code)
+        return success
+
+    # ┏┓┏┓┳  ┏┓  ┓┓•
+    # ┣┫┃┃┃  ┃ ┏┓┃┃┓┏┓┏┓
+    # ┛┗┣┛┻  ┗┛┗┻┗┗┗┛┗┗┫
+    #                  ┛
+
+    async def _get(
+            self,
+            url: str = None,
+            headers: dict = None,
+            params: dict = None,
+            cookies: httpx.Cookies | dict = None
+    ) -> (httpx.Response | None, Exception | None):
+
+        """
+        To call an API once using the GET method.
+        :param url: A custom URL to hit. If not specified, the data URL will be hit.
+        :param headers: Custom headers to use. If not specified, default values will be used.
+        :param params: The params to send in the query string itself.
+        :param cookies: The cookies to use when calling the API.
+        :return: The raw response from the API call and any exception that occurred.
+        """
+
+        # Start by assuming failure:
+        response = None
+        excp = None
+
+        try:
+
+            # Make the API call:
+            response = await self._http_client.get(
+                url = url or self._data_url,
+                headers = headers or {
+                    "Accept": "*/*",
+                    "Accept-Encoding": "gzip, deflate, br",
+                    "Accept-Language": "en-US,en;q=0.9",
+                    "Host": "www.nseindia.com",
+                    "Referer": self._base_url,
+                    "Sec-Fetch-Dest": "empty",
+                    "Sec-Fetch-Mode": "cors",
+                    "Sec-Fetch-Site": "same-origin",
+                    "Sec-GPC": "1",
+                    "Upgrade-Insecure-Requests": "1",
+                    "TE": "trailers",
+                    "Sec-Ch-Ua": "\"Chromium\";v=\"128\", \"Not;A=Brand\";v=\"24\", \"Google Chrome\";v=\"128\"",
+                    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+                },
+                params = params,
+                cookies = cookies
+            )
+
+        # If something goes wrong:
+        except Exception as exception:
+            excp = exception
+            self._printer(exception, url, params)
+
+        # Done here:
+        return response, excp
+
+    async def get(
+            self,
+            url: str = None,
+            headers: dict = None,
+            params: dict = None,
+            cookies: httpx.Cookies | dict = None,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        Call an API using the GET method.
+        :param url: A custom URL to hit. If not specified, the data URL will be hit.
+        :param headers: Custom headers to use. If not specified, default values will be used.
+        :param params: The params to send in the query string itself.
+        :param cookies: Any custom cookies to use when calling the API. If not given, the class's internal cookies will
+            be used by default. Use this only to override the default one.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: A structured response that includes the raw response, the exception (if any), and so on.
+        """
+
+        # Select and refresh the cookies as needed:
+        if cookies: use_cookies = cookies
+        else:
+            if refresh_cookies: await self.refresh_cookies(force_refresh = force_refresh_cookies)
+            use_cookies = self._cookies
+
+        # Prepare the structure of the response:
+        frame = inspect.stack()[1]
+        caller_locals = frame[0].f_locals
+        calling_class = caller_locals.get("self", None)
+        calling_class = calling_class.__class__.__name__ if calling_class else ""
+        calling_function = inspect.stack()[1].function
+        api_response = NSEApiResponse(
+            action = f"{calling_class}.{calling_function}",
+            url = url or self._data_url,
+            method = "GET",
+            inputs = {"headers": headers, "params": params}
+        )
+
+        # Try as many times as asked:
+        for attempt in range(retry_count):
+
+            # Simple debugging statement:
+            if not self._debug_only_errors:
+                self._printer("API call.", attempt)
+
+            # Make the API call:
+            raw_response, exception = await self._get(
+                url = url,
+                headers = headers,
+                params = params,
+                cookies = use_cookies
+            )
+
+            # Populate the response model:
+            if raw_response is not None:
+                api_response.response = raw_response
+                api_response.httpCode = raw_response.status_code
+                api_response.message = raw_response.reason_phrase
+                if raw_response.status_code in [200]: api_response.success = True
+            if exception is not None:
+                api_response.exception = exception
+                api_response.message = str(exception)
+
+            # If the API call succeeded:
+            if api_response.success: break
+
+            # If the API call failed:
+            await asyncio.sleep(backoff_seconds)
+            backoff_seconds *= backoff_multiplier
+
+        # Done here:
+        return api_response
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    pass

nse/controllers/bhavcopy/equities.py

@@ -0,0 +1,326 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Saturday, 4th Jan., 2025.
+
+    OBJECTIVE:
+
+        To provide a way to fetch the daily BhavCopy from NSE.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+import pytz
+
+# To work with datatypes:
+from typing import Any, List
+
+# To work with tabulated dataL
+import pandas as pd
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEEquitiesBhavCopy(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (Eq. BhavCopy) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/all-reports",
+            data_url = r"https://nsearchives.nseindia.com/products/content/sec_bhavdata_full_03012025.csv",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    @staticmethod
+    def parse_float(value):
+        try: return float(value)
+        except ValueError: return None
+
+    @staticmethod
+    def parse_int(value):
+        try: return int(value)
+        except ValueError: return None
+
+    async def get_data(
+            self,
+            target_date: datetime.datetime,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param target_date: The starting date (inclusive) from which the events must be fetched.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Figure out the file name:
+        date_time.to_timezone(target_date, date_time.TIMEZONE_IST)
+        file_name = f"sec_bhavdata_full_{target_date.strftime('%d%m%Y')}.csv"
+
+        # Make the API call:
+        api_response = await self.get(
+            url = f"https://nsearchives.nseindia.com/products/content/{file_name}",
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+            },
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw:
+                api_response.data = io.BytesIO(await api_response.get_content())
+                api_response.data.seek(0)
+            else:
+                try: api_response.data = self.format_data(
+                    raw_data = await api_response.get_content(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    def format_data(
+            self,
+            raw_data: bytes | io.BytesIO,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_data: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_data is None: return raw_data
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Ensure that we have a BytesIO object,
+            # and that we are pointing at the start:
+            if isinstance(raw_data, bytes):
+                raw_data = io.BytesIO(raw_data)
+            raw_data.seek(0)
+
+            # Read the data into Pandas:
+            bhav_df = pd.read_csv(raw_data)
+            bhav_df.rename(
+                columns = {c: c.strip() for c in bhav_df.columns.to_list()},
+                inplace = True
+            )
+            bhav_df.rename(
+                columns = {
+                    "SYMBOL": "symbol",
+                    "SERIES": "segment",
+                    "DATE1": "date",
+                    "PREV_CLOSE": "prevClose",
+                    "OPEN_PRICE": "open",
+                    "HIGH_PRICE": "high",
+                    "LOW_PRICE": "low",
+                    "CLOSE_PRICE": "close",
+                    "LAST_PRICE": "ltp",
+                    "AVG_PRICE": "vwap",
+                    "TTL_TRD_QNTY": "totVol",
+                    "TURNOVER_LACS": "totCash",
+                    "NO_OF_TRADES": "tradeCount",
+                    "DELIV_QTY": "deliveryVol",
+                    "DELIV_PER": "deliveryPct"
+                },
+                inplace = True
+            )
+
+            # Format the data:
+            for column in ["symbol", "segment", "date"]:
+                bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip())
+            for column in ["prevClose", "open", "high", "low", "close", "ltp", "vwap", "totCash", "deliveryPct"]:
+                bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x))
+            for column in ["totVol", "deliveryVol"]:
+                bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x))
+            bhav_df["totCash"] *= 1_00_000
+
+            # Give the date as a timestamp:
+            bhav_df["scrapeTs"] = timestamp
+            bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string(
+                dt_str = x,
+                dt_formats = ["%d-%b-%Y"],
+                input_tz = date_time.TIMEZONE_IST,
+                output_tz = date_time.TIMEZONE_UTC
+            ))
+            bhav_df["tz"] = "Asia/Kolkata"
+
+            # Keep what is needed, in an order that makes intuitive sense:
+            bhav_df = bhav_df[[
+                "symbol", "segment",
+                "prevClose", "open", "high", "low", "close", "ltp", "vwap",
+                "totVol", "totCash", "deliveryVol", "deliveryPct",
+                "date", "ts", "tz", "scrapeTs"
+            ]]
+
+            # Done here:
+            formatted_data = bhav_df.to_dict(orient = "records")
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEEquitiesBhavCopy(http_client = test_client)
+
+        # Get and show the data:
+        target_date = datetime.datetime.strptime("20250103", "%Y%m%d")
+        api_response = await my_nse.get_data(
+            target_date = target_date,
+            return_raw = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str))
+
+    asyncio.run(main())

nse/controllers/bhavcopy/fno.py

@@ -0,0 +1,325 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 6th Jan., 2025.
+
+    OBJECTIVE:
+
+        To provide a way to fetch the daily BhavCopy from NSE.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+import pytz
+
+# To work with datatypes:
+from typing import Any, List
+
+# To work with tabulated dataL
+import pandas as pd
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEFNOBhavCopy(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (FNO BhavCopy) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/all-reports",
+            data_url = r"https://nsearchives.nseindia.com/content/trdops/FNO_BC02012025.DAT",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    @staticmethod
+    def parse_float(value):
+        try: return float(value)
+        except ValueError: return None
+
+    @staticmethod
+    def parse_int(value):
+        try: return int(value)
+        except ValueError: return None
+
+    async def get_data(
+            self,
+            target_date: datetime.datetime,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param target_date: The starting date (inclusive) from which the events must be fetched.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Figure out the file name:
+        date_time.to_timezone(target_date, date_time.TIMEZONE_IST)
+        file_name = f"FNO_BC{target_date.strftime('%d%m%Y')}.DAT"
+
+        # Make the API call:
+        api_response = await self.get(
+            url = f"https://nsearchives.nseindia.com/content/trdops/{file_name}",
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+            },
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw:
+                api_response.data = io.BytesIO(await api_response.get_content())
+                api_response.data.seek(0)
+            else:
+                try: api_response.data = self.format_data(
+                    raw_data = await api_response.get_content(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    def format_data(
+            self,
+            raw_data: bytes | io.BytesIO,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_data: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_data is None: return raw_data
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Ensure that we have a BytesIO object,
+            # and that we are pointing at the start:
+            if isinstance(raw_data, bytes):
+                raw_data = io.BytesIO(raw_data)
+            raw_data.seek(0)
+
+            # Read the data into Pandas:
+            bhav_df = pd.read_csv(
+                raw_data,
+                header = None,
+                names = [
+                    "symbol", "segment", "underlying", "expiry", "strike",
+                    "type", "G?", "H?", "I?", "prevClose",
+                    "open", "high", "low", "close", "O?",
+                    "totVol", "totCash", "oi", "oiChg", "T?",
+                    "U?", "date", "W?", "X?", "Y?",
+                    "Z?"
+                ]
+            )
+            # Format the data:
+            for column in ["symbol", "segment", "type", "underlying", "expiry", "date"]:
+                bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip())
+            for column in ["strike", "prevClose", "open", "high", "low", "close", "totCash"]:
+                bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x))
+            for column in ["totVol", "oi", "oiChg"]:
+                bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x))
+
+            # Handle segment, type and other normalizations:
+            bhav_df["isIndex"] = bhav_df["segment"].apply(lambda x: True if x.endswith("IDX") else False)
+            bhav_df["segment"] = bhav_df["segment"].apply(lambda x: x[:3])
+            bhav_df["type"] = bhav_df.apply(lambda x: None if x["segment"] == "FUT" else x["type"], axis = 1)
+            bhav_df["strike"] = bhav_df.apply(lambda x: None if x["segment"] == "FUT" else x["strike"], axis = 1)
+
+            # Give the expiry as a timestamp:
+            bhav_df["expiryTs"] = bhav_df["expiry"].apply(lambda x: self.parse_datetime_string(
+                dt_str = f"{int(x):0>8}",
+                dt_formats = ["%d%m%Y"],
+                input_tz = date_time.TIMEZONE_IST,
+                output_tz = date_time.TIMEZONE_UTC
+            ))
+            bhav_df["expiryTz"] = "Asia/Kolkata"
+
+            # Give the data recording date as a timestamp:
+            bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string(
+                dt_str = x,
+                dt_formats = ["%d%m%Y", "%d-%b-%Y", "%d/%b/%Y"],
+                input_tz = date_time.TIMEZONE_IST,
+                output_tz = date_time.TIMEZONE_UTC
+            ))
+            bhav_df["tz"] = "Asia/Kolkata"
+            bhav_df["scrapeTs"] = timestamp
+
+            # Keep what is needed, in an order that makes intuitive sense:
+            bhav_df = bhav_df[[
+                "symbol", "segment", "type", "underlying", "isIndex", "expiry", "expiryTs", "expiryTz", "strike",
+                "prevClose", "open", "high", "low", "close", "totVol", "totCash", "oi", "oiChg",
+                "date", "ts", "tz", "scrapeTs"
+            ]]
+
+            # Done here:
+            formatted_data = bhav_df.to_dict(orient = "records")
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEFNOBhavCopy(http_client = test_client)
+
+        # Get and show the data:
+        target_date = datetime.datetime.strptime("20250103", "%Y%m%d")
+        api_response = await my_nse.get_data(
+            target_date = target_date,
+            return_raw = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str))
+
+    asyncio.run(main())

nse/controllers/calendar/corporate_actions.py

@@ -0,0 +1,282 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates of important events like financial-results, stock-splits, fund-raising, etc.
+        from NSE's portal.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# data models:
+from utils_v2.nse.models.calendar import NSECorporateAction
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+import pytz
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSECorporateActionsCalendar(AsyncNSEBase):
+
+    # The types of corporate actions/entities:
+    KIND_EQUITIES = "equities"
+    KIND_MUTUAL_FUNDS = "mf"
+    KIND_DEBT = "debt"
+    KIND_SME = "sme"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (C.Act.) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/companies-listing/corporate-filings-actions",
+            data_url = r"https://www.nseindia.com/api/corporates-corporateActions",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            kind: str = KIND_EQUITIES,
+            from_date: datetime.datetime = None,
+            to_date: datetime.datetime = None,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param kind: The option between 'equities", "mf", "debt", and "sme".
+        :param from_date: The starting date (inclusive) from which the events must be fetched.
+        :param to_date: The ending date (inclusive) till which the events must be fetched.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Prepare the inputs:
+        now = date_time.get_current_utc_date_time(as_string = False)
+        from_date = date_time.to_timezone(from_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
+        to_date = date_time.to_timezone(to_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
+
+        # Make the API call:
+        api_response = await self.get(
+            params = {
+                "index": kind,
+                "from_date": from_date,
+                "to_date": to_date,
+            },
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    kind = kind,
+                    timestamp = date_time.get_current_utc_date_time(as_string = True),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    def format_data(
+            self,
+            raw_json: List[dict],
+            kind: str = None,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSECorporateAction] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param kind: The kind of data pulled using the 'get_data' method.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = []
+            for action in raw_json:
+                formatted_data.append(NSECorporateAction(**{
+                    "scrapeTs": timestamp,
+                    "kind": kind,
+                    "symbol": action["symbol"],
+                    "name": action["comp"],
+                    "isin": action["isin"],
+                    "series": action["series"],
+                    "ind": action["ind"],
+                    "faceVal": float(action["faceVal"]),
+                    "action": action["subject"],
+                    "exTs": self.parse_datetime_string(action["exDate"], ["%d-%b-%Y"]),
+                    "recTs": self.parse_datetime_string(action["recDate"]),
+                    "bcStartTs": self.parse_datetime_string(action["bcStartDate"]),
+                    "bcEndTs": self.parse_datetime_string(action["bcEndDate"]),
+                    "ndStartTs": self.parse_datetime_string(action["ndStartDate"]),
+                    "ndEndTs": self.parse_datetime_string(action["ndEndDate"]),
+                    "caBroadcastTs": self.parse_datetime_string(action["caBroadcastDate"])
+                }))
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSECorporateActionsCalendar()
+
+        # Get and show the data:
+        now = date_time.get_current_ist_date_time()
+        to_date = now + datetime.timedelta(days = 31)
+        from_date = now - datetime.timedelta(days = 31)
+        print("FROM :", from_date)
+        print("TO   :", to_date)
+        api_response = await my_nse.get_data(
+            kind = NSECorporateActionsCalendar.KIND_EQUITIES,
+            from_date = from_date,
+            to_date = to_date,
+            return_raw = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("CORPORATE ACTIONS:", json.to_string(api_response.data, default = str))
+        else: raise api_response.exception
+
+    asyncio.run(main())

nse/controllers/calendar/corporate_events.py

@@ -0,0 +1,273 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates of important events like financial-results, stock-splits, fund-raising, etc.
+        from NSE's portal.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# Data models:
+from utils_v2.nse.models.calendar import NSECorporateEvent
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSECorporateEventsCalendar(AsyncNSEBase):
+
+    # Index types:
+    KIND_EQUITIES = "equities"
+    KIND_SME = "sme"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (CECal) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/companies-listing/corporate-filings-event-calendar",
+            data_url = r"https://www.nseindia.com/api/event-calendar",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            kind: str = KIND_EQUITIES,
+            from_date: datetime.datetime = None,
+            to_date: datetime.datetime = None,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param kind: The type of index you want. A choice between "equities" and "sme"
+        :param from_date: The starting date (inclusive) from which the events must be fetched.
+        :param to_date: The ending date (inclusive) till which the events must be fetched.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Prepare the inputs:
+        now = date_time.get_current_utc_date_time(as_string = False)
+        from_date = date_time.to_timezone(from_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
+        to_date = date_time.to_timezone(to_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
+
+        # Make the API call:
+        api_response = await self.get(
+            params = {
+                "index": kind,
+                "from_date": from_date,
+                "to_date": to_date
+            },
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    kind = kind,
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: List[dict],
+            kind: str = None,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSECorporateEvent] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param kind: The kind of data pulled using the 'get_data' method.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                {
+                    "scrapeTs": timestamp,
+                    "kind": kind,
+                    "symbol": event.get("symbol", event.get("bm_symbol")),
+                    "name": event.get("company", event.get("sm_name")),
+                    "ts": date_time.to_timezone(
+                        date_time.parse_date_time(
+                            event.get("date", event.get("bm_date")),
+                            timezone = date_time.TIMEZONE_IST
+                        ),
+                        timezone = date_time.TIMEZONE_UTC
+                    ),
+                    "tz": "Asia/Kolkata",
+                    "purpose": event.get("purpose", event.get("bm_purpose")),
+                    "description": event.get("bm_desc"),
+                    "attachment": event.get("attachment")
+                } for event in raw_json
+            ]
+            formatted_data = [NSECorporateEvent(**d) for d in formatted_data]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSECorporateEventsCalendar()
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(
+            kind = NSECorporateEventsCalendar.KIND_EQUITIES,
+            from_date = datetime.datetime(year = 2024, month = 1, day = 1),
+            to_date = datetime.datetime(year = 2025, month = 1, day = 1),
+            return_raw = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("CORPORATE EVENT CALENDAR:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/calendar/examples/20241128_formatted_nse_corporate_event_calendar.json

@@ -0,0 +1,178 @@
+[
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "AAREYDRUGS",
+        "company": "Aarey Drugs & Pharmaceuticals Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Fund Raising/Other business matters",
+        "brief": "To consider Fund Raising and other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "ASTERDM",
+        "company": "Aster DM Healthcare Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters pertaining to Preferential Issue."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "EROSMEDIA",
+        "company": "Eros International Media Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Financial Results/Other business matters",
+        "brief": "To consider and approve the financial results for the quarter and year ended March 31, 2024 and other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "MUFIN",
+        "company": "Mufin Green Finance Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Fund Raising",
+        "brief": "To consider Fund Raising by Issue of Secured Unlisted Non Convertible Debenture."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "ONMOBILE",
+        "company": "OnMobile Global Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Fund Raising",
+        "brief": "To consider Fund Raising"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "PRIVISCL",
+        "company": "Privi Speciality Chemicals Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider and Approve \"Privi Employee Stock Option Scheme - 2024\""
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "SOUTHBANK",
+        "company": "The South Indian Bank Limited",
+        "eventDate": "2024-11-28 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider, decide on exercise of call option of Bank s Non-convertible, fully paid-up, unsecured, perpetual, Basel III Compliant, Tier I Bonds, listed in BSE"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "eventDate": "2024-11-29 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "eventDate": "2024-11-29 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "To consider and approve the financial results for the period ended Jun 30, 2024"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "SIKKO",
+        "company": "Sikko Industries Limited",
+        "eventDate": "2024-11-29 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider discuss and decide terms and conditions such as determination of the rights issue price, rights entitlement ratio, record date and other matters incidental orconnected therewith."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "AGIIL",
+        "company": "Agi Infra Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Stock Split",
+        "brief": "To consider stock split of equity shares of the Company"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "MANAKCOAT",
+        "company": "Manaksia Coated Metals & Industries Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Fund Raising",
+        "brief": "To consider Fund Raising"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider the enclosed business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "PAR",
+        "company": "Par Drugs and Chemicals Limited",
+        "eventDate": "2024-12-01 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "FIBERWEB",
+        "company": "Fiberweb (India) Limited",
+        "eventDate": "2024-12-02 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider and discuss about the expansion plans"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "IITL",
+        "company": "Industrial Investment Trust Limited",
+        "eventDate": "2024-12-02 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "SWIGGY",
+        "company": "Swiggy Limited",
+        "eventDate": "2024-12-02 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "To consider and approve the financial results for the quarter and half year ended September 30, 2024"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "GBGLOBAL",
+        "company": "GB Global Limited",
+        "eventDate": "2024-12-03 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider other business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "MOTOGENFIN",
+        "company": "The Motor & General Finance Limited",
+        "eventDate": "2024-12-03 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "Intimation Regarding Independent Directors Meeting will be held on 04.12.2024."
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "DIL",
+        "company": "Debock Industries Limited",
+        "eventDate": "2024-12-04 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "DIL : 05-Dec-2024 :  The Company has informed the Exchange that a Board meeting to be held on November 27, 2024 has been re-scheduled. Further, the Company has informed the Exchange that the meeting of the Board of Directors of the Company will be held on December 05, 2024,  To consider and approve the financial results for the period ended September 30, 2024"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "eventDate": "2024-12-08 18:07:00+00:00",
+        "eventType": "Other business matters",
+        "brief": "To consider enclosed business matters"
+    },
+    {
+        "ts": "2024-11-28 15:11:22.793824+05:30",
+        "symbol": "EXIDEIND",
+        "company": "Exide Industries Limited",
+        "eventDate": "2025-01-27 18:07:00+00:00",
+        "eventType": "Financial Results",
+        "brief": "To consider and approve the financial results for the period ended December 31, 2024"
+    }
+]

nse/controllers/calendar/examples/20241128_raw_nse_corporate_event_calendar.json

@@ -0,0 +1,156 @@
+[
+    {
+        "symbol": "AAREYDRUGS",
+        "company": "Aarey Drugs & Pharmaceuticals Limited",
+        "purpose": "Fund Raising/Other business matters",
+        "bm_desc": "To consider Fund Raising and other business matters",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "ASTERDM",
+        "company": "Aster DM Healthcare Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters pertaining to Preferential Issue.",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "EROSMEDIA",
+        "company": "Eros International Media Limited",
+        "purpose": "Financial Results/Other business matters",
+        "bm_desc": "To consider and approve the financial results for the quarter and year ended March 31, 2024 and other business matters",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "MUFIN",
+        "company": "Mufin Green Finance Limited",
+        "purpose": "Fund Raising",
+        "bm_desc": "To consider Fund Raising by Issue of Secured Unlisted Non Convertible Debenture.",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "ONMOBILE",
+        "company": "OnMobile Global Limited",
+        "purpose": "Fund Raising",
+        "bm_desc": "To consider Fund Raising",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "PRIVISCL",
+        "company": "Privi Speciality Chemicals Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider and Approve \"Privi Employee Stock Option Scheme - 2024\"",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "SOUTHBANK",
+        "company": "The South Indian Bank Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider, decide on exercise of call option of Bank s Non-convertible, fully paid-up, unsecured, perpetual, Basel III Compliant, Tier I Bonds, listed in BSE",
+        "date": "29-Nov-2024"
+    },
+    {
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "30-Nov-2024"
+    },
+    {
+        "symbol": "BCG",
+        "company": "Brightcom Group Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "To consider and approve the financial results for the period ended Jun 30, 2024",
+        "date": "30-Nov-2024"
+    },
+    {
+        "symbol": "SIKKO",
+        "company": "Sikko Industries Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider discuss and decide terms and conditions such as determination of the rights issue price, rights entitlement ratio, record date and other matters incidental orconnected therewith.",
+        "date": "30-Nov-2024"
+    },
+    {
+        "symbol": "AGIIL",
+        "company": "Agi Infra Limited",
+        "purpose": "Stock Split",
+        "bm_desc": "To consider stock split of equity shares of the Company",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "MANAKCOAT",
+        "company": "Manaksia Coated Metals & Industries Limited",
+        "purpose": "Fund Raising",
+        "bm_desc": "To consider Fund Raising",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider the enclosed business matters",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "PAR",
+        "company": "Par Drugs and Chemicals Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "02-Dec-2024"
+    },
+    {
+        "symbol": "FIBERWEB",
+        "company": "Fiberweb (India) Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider and discuss about the expansion plans",
+        "date": "03-Dec-2024"
+    },
+    {
+        "symbol": "IITL",
+        "company": "Industrial Investment Trust Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "03-Dec-2024"
+    },
+    {
+        "symbol": "SWIGGY",
+        "company": "Swiggy Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "To consider and approve the financial results for the quarter and half year ended September 30, 2024",
+        "date": "03-Dec-2024"
+    },
+    {
+        "symbol": "GBGLOBAL",
+        "company": "GB Global Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider other business matters",
+        "date": "04-Dec-2024"
+    },
+    {
+        "symbol": "MOTOGENFIN",
+        "company": "The Motor & General Finance Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "Intimation Regarding Independent Directors Meeting will be held on 04.12.2024.",
+        "date": "04-Dec-2024"
+    },
+    {
+        "symbol": "DIL",
+        "company": "Debock Industries Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "DIL : 05-Dec-2024 :  The Company has informed the Exchange that a Board meeting to be held on November 27, 2024 has been re-scheduled. Further, the Company has informed the Exchange that the meeting of the Board of Directors of the Company will be held on December 05, 2024,  To consider and approve the financial results for the period ended September 30, 2024",
+        "date": "05-Dec-2024"
+    },
+    {
+        "symbol": "NTL",
+        "company": "Neueon Towers Limited",
+        "purpose": "Other business matters",
+        "bm_desc": "To consider enclosed business matters",
+        "date": "09-Dec-2024"
+    },
+    {
+        "symbol": "EXIDEIND",
+        "company": "Exide Industries Limited",
+        "purpose": "Financial Results",
+        "bm_desc": "To consider and approve the financial results for the period ended December 31, 2024",
+        "date": "28-Jan-2025"
+    }
+]

nse/controllers/calendar/examples/20241205_formatted_nse_trading_holiday_calendar.json

@@ -0,0 +1,542 @@
+[
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-01-21 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Special Holiday",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-01-25 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Republic Day",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-02-18 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Chatrapati Shivaji Maharaj Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-07 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Mahashivratri",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-24 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Holi",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-28 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Good Friday",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-03-31 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Annual Bank Closing",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-08 18:30:00+00:00",
+        "weekDay": "Tuesday",
+        "event": "Gudi Padwa",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-10 18:30:00+00:00",
+        "weekDay": "Thursday",
+        "event": "Id-Ul-Fitr (Ramadan Id)",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-13 18:30:00+00:00",
+        "weekDay": "Sunday",
+        "event": "Dr.Baba Saheb Ambedkar Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-16 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Shri Ram Navami",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-20 18:30:00+00:00",
+        "weekDay": "Sunday",
+        "event": "Mahavir Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-04-30 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Maharashtra Day",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-05-19 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "General Parliamentary Elections",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-05-22 18:30:00+00:00",
+        "weekDay": "Thursday",
+        "event": "Buddha Pournima",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-06-16 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Bakri Id",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-07-16 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Muharram",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-08-14 18:30:00+00:00",
+        "weekDay": "Thursday",
+        "event": "Independence Day/Parsi New Year",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-09-06 18:30:00+00:00",
+        "weekDay": "Saturday",
+        "event": "Ganesh Chaturthi",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-09-15 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "Id-E-Milad",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "IRD",
+            "NDM",
+            "NTRP"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-10-01 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Mahatma Gandhi Jayanti",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-10-11 18:30:00+00:00",
+        "weekDay": "Saturday",
+        "event": "Dussehra",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-11-01 18:30:00+00:00",
+        "weekDay": "Saturday",
+        "event": "Balipratipada",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-11-14 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Prakash Gurpurb Sri Guru Nanak Dev",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-11-19 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Assembly Elections in Maharashtra",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-12-24 18:30:00+00:00",
+        "weekDay": "Wednesday",
+        "event": "Christmas",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "CBM",
+            "CD",
+            "CM",
+            "CMOT",
+            "COM",
+            "FO",
+            "IRD",
+            "MF",
+            "NDM",
+            "NTRP",
+            "SLBS"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2023-12-31 18:30:00+00:00",
+        "weekDay": "Monday",
+        "event": "New year",
+        "morningSession": "Open",
+        "eveningSession": "Closed",
+        "holidayType": [
+            "COM"
+        ]
+    },
+    {
+        "scrapeTs": "2024-12-05T04:19:17.317730+00:00",
+        "date": "2024-10-31 18:30:00+00:00",
+        "weekDay": "Friday",
+        "event": "Diwali Laxmi Pujan",
+        "morningSession": null,
+        "eveningSession": null,
+        "holidayType": [
+            "MF",
+            "NDM",
+            "NTRP"
+        ]
+    }
+]

nse/controllers/calendar/holidays.py

@@ -0,0 +1,323 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 5th Dec., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve the list of holidays from NSE and to check if today is a holiday.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# Data models:
+from utils_v2.nse.models.calendar import NSEHoliday
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+# For debugging:
+import inspect
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEHolidaysCalendar(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (Hol) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/resources/exchange-communication-holidays",
+            data_url = r"https://www.nseindia.com/api/holiday-master?type=trading",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the holiday calendar.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSEHoliday] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Start by creating an empty dict:
+            formatted_data = {}
+
+            # Process each category in the original data
+            for category, events in raw_json.items():
+                for event in events:
+                    trading_date = event["tradingDate"]
+
+                    # If the tradingDate is not in the dictionary, initialize it
+                    if trading_date not in formatted_data:
+                        formatted_data[trading_date] = {
+                            "scrapeTs": timestamp,
+                            "ts": date_time.to_timezone(
+                                date_time.as_if_timezone(
+                                    datetime.datetime.strptime(trading_date, "%d-%b-%Y"),
+                                    timezone = date_time.TIMEZONE_IST
+                                ),
+                                timezone = date_time.TIMEZONE_UTC
+                            ),
+                            "tz": "Asia/Kolkata",
+                            "weekday": event["weekDay"],
+                            "event": event["description"],
+                            "type": []
+                        }
+
+                    # Add the current category to the "type" list
+                    formatted_data[trading_date]["type"].append(category)
+
+            # Now we get rid of the unnecessary keys and make it a list, and model it:
+            formatted_data = [v for v in formatted_data.values()]
+            formatted_data = [NSEHoliday(**d) for d in formatted_data]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+    async def is_holiday(
+            self,
+            timestamp: datetime.datetime
+    ) -> NSEApiResponse:
+
+        """
+        Checks if a given date is a trading holiday, or not.
+        :param timestamp: The datetime instance that you want to check for it being a holiday. Preferably make it an
+            aware instance. If a naive instance is passed, it will be assumed to be in UTC.
+        :return: True if it is a holiday, False if it isn't. Can be None if something goes wrong in fetching the data.
+            The value will be in the 'data' field of the response model.
+        """
+
+        # We first prepare UTC and IST versions of the incoming datetime:
+        timestamp_utc = date_time.to_timezone(timestamp, timezone = date_time.TIMEZONE_UTC)
+        timestamp_ist = date_time.to_timezone(timestamp_utc, timezone = date_time.TIMEZONE_IST)
+
+        # If this day is a weekend, it is a holiday by default:
+        ist_dow = timestamp_ist.weekday()
+        if ist_dow in [5, 6]: return NSEApiResponse(
+            action = inspect.stack()[0].function,
+            url = None,
+            method = None,
+            success = True,
+            data = {
+                "isHoliday": True,
+                "event": "Saturday" if ist_dow == 5 else "Sunday"
+            },
+            message = "Holiday due to weekend."
+        )
+
+        # First we fetch the data from NSE:
+        api_response = await self.get_data(return_raw = False)
+        api_response.action = inspect.stack()[0].function
+        if not api_response.success: return api_response
+
+        # Now we check if our date to check is in the list:
+        is_holiday = False
+        event = None
+        for h in api_response.data:
+            if h.ts.date() == timestamp_utc.date():
+                is_holiday = True
+                event = h.event
+                break
+        api_response.data = {
+            "isHoliday": is_holiday,
+            "event": event
+        }
+
+        # Done here:
+        return api_response
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEHolidaysCalendar()
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("HOLIDAY CALENDAR:", json.to_string(api_response.data, default = str))
+        ts = api_response.data[0].ts
+        print(ts, type(ts))
+        print("\n---\n\n")
+
+        # # Check for a holiday:
+        # date_to_check = date_time.parse_date_time(
+        #     input_value = "2025-12-25 00:00:00",
+        #     date_formats = ["%Y-%m-%d %H:%M:%S"],
+        #     timezone = date_time.TIMEZONE_IST
+        # )
+        # api_response = await my_nse.is_holiday(date_to_check)
+        # print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        # print("DATETIME:", date_to_check)
+        # print("IS HOLIDAY:", api_response.data)
+
+    asyncio.run(main())

nse/controllers/indices/constituents.py

@@ -0,0 +1,394 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Monday, 16th Dec., 2024
+
+    OBJECTIVE:
+
+        This file aims to fetch the current details about the constituents of various indices. This gives you not only
+        the constituent stocks of the selected index, but also that stock's current activity in the market.
+        https://www.nseindia.com/market-data/live-equity-market?symbol=NIFTY%2050
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEIndexConstituents(AsyncNSEBase):
+
+    # Broad Market Indices:
+    INDEX_NIFTY_50 = "NIFTY 50"
+    INDEX_NIFTY_NEXT_50 = "NIFTY NEXT 50"
+    INDEX_NIFTY_MIDCAP_50 = "NIFTY MIDCAP 50"
+    INDEX_NIFTY_MIDCAP_100 = "NIFTY MIDCAP 100"
+    INDEX_NIFTY_MIDCAP_150 = "NIFTY MIDCAP 150"
+    INDEX_NIFTY_SMALLCAP_50 = "NIFTY SMALLCAP 50"
+    INDEX_NIFTY_SMALLCAP_100 = "NIFTY SMALLCAP 100"
+    INDEX_NIFTY_SMALLCAP_250 = "NIFTY SMALLCAP 250"
+    INDEX_NIFTY_MIDSMALLCAP_400 = "NIFTY MIDSMALLCAP 400"
+    INDEX_NIFTY_100 = "NIFTY 100"
+    INDEX_NIFTY_200 = "NIFTY 200"
+    INDEX_NIFTY_500_MULTICAP_50_25_25 = "NIFTY500 MULTICAP 50:25:25"
+    INDEX_NIFTY_LARGEMIDCAP_250 = "NIFTY LARGEMIDCAP 250"
+    INDEX_NIFTY_MIDCAP_SELECT = "NIFTY MIDCAP SELECT"
+    INDEX_NIFTY_TOTAL_MARKET = "NIFTY TOTAL MARKET"
+    INDEX_NIFTY_MICROCAP_250 = "NIFTY MICROCAP 250"
+    INDEX_NIFTY_500 = "NIFTY 500"
+    INDEX_NIFTY_500_LARGEMIDSMALL_EQUAL_CAP_WEIGHTED = "NIFTY500 LARGEMIDSMALL EQUAL-CAP WEIGHTED"
+
+    # Sectoral Indices:
+    INDEX_NIFTY_AUTO = "NIFTY AUTO"
+    INDEX_NIFTY_BANK = "NIFTY BANK"
+    INDEX_NIFTY_ENERGY = "NIFTY ENERGY"
+    INDEX_NIFTY_FINANCIAL_SERVICES = "NIFTY FINANCIAL SERVICES"
+    INDEX_NIFTY_FINANCIAL_SERVICES_25_50 = "NIFTY FINANCIAL SERVICES 25/50"
+    INDEX_NIFTY_FMCG = "NIFTY FMCG"
+    INDEX_NIFTY_IT = "NIFTY IT"
+    INDEX_NIFTY_MEDIA = "NIFTY MEDIA"
+    INDEX_NIFTY_METAL = "NIFTY METAL"
+    INDEX_NIFTY_PHARMA = "NIFTY PHARMA"
+    INDEX_NIFTY_PSU_BANK = "NIFTY PSU BANK"
+    INDEX_NIFTY_REALTY = "NIFTY REALTY"
+    INDEX_NIFTY_PRIVATE_BANK = "NIFTY PRIVATE BANK"
+    INDEX_NIFTY_HEALTHCARE_INDEX = "NIFTY HEALTHCARE INDEX"
+    INDEX_NIFTY_CONSUMER_DURABLES = "NIFTY CONSUMER DURABLES"
+    INDEX_NIFTY_OIL_GAS = "NIFTY OIL & GAS"
+    INDEX_NIFTY_MIDSMALL_HEALTHCARE = "NIFTY MIDSMALL HEALTHCARE"
+    INDEX_NIFTY_FINANCIAL_SERVICES_EX_BANK = "NIFTY FINANCIAL SERVICES EX-BANK"
+    INDEX_NIFTY_MIDSMALL_FINANCIAL_SERVICES = "NIFTY MIDSMALL FINANCIAL SERVICES"
+    INDEX_NIFTY_MIDSMALL_IT_TELECOM = "NIFTY MIDSMALL IT & TELECOM"
+
+    # Thematic:
+    INDEX_NIFTY_COMMODITIES = "NIFTY COMMODITIES"
+    INDEX_NIFTY_INDIA_CONSUMPTION = "NIFTY INDIA CONSUMPTION"
+    INDEX_NIFTY_CPSE = "NIFTY CPSE"
+    INDEX_NIFTY_INFRASTRUCTURE = "NIFTY INFRASTRUCTURE"
+    INDEX_NIFTY_MNC = "NIFTY MNC"
+    INDEX_NIFTY_GROWTH_SECTORS_15 = "NIFTY GROWTH SECTORS 15"
+    INDEX_NIFTY_PSE = "NIFTY PSE"
+    INDEX_NIFTY_SERVICES_SECTOR = "NIFTY SERVICES SECTOR"
+    INDEX_NIFTY_100_LIQUID_15 = "NIFTY100 LIQUID 15"
+    INDEX_NIFTY_MIDCAP_LIQUID_15 = "NIFTY MIDCAP LIQUID 15"
+    INDEX_NIFTY_INDIA_DIGITAL = "NIFTY INDIA DIGITAL"
+    INDEX_NIFTY_100_ESG = "NIFTY100 ESG"
+    INDEX_NIFTY_INDIA_MANUFACTURING = "NIFTY INDIA MANUFACTURING"
+    INDEX_NIFTY_INDIA_CORPORATE_GROUP_INDEX_TATA_GROUP_25_CAP = "NIFTY INDIA CORPORATE GROUP INDEX - TATA GROUP 25% CAP"
+    INDEX_NIFTY_500_MULTICAP_INDIA_MANUFACTURING_50_30_20 = "NIFTY500 MULTICAP INDIA MANUFACTURING 50:30:20"
+    INDEX_NIFTY_500_MULTICAP_INFRASTRUCTURE_50_30_20 = "NIFTY500 MULTICAP INFRASTRUCTURE 50:30:20"
+    INDEX_NIFTY_INDIA_DEFENCE = "NIFTY INDIA DEFENCE"
+    INDEX_NIFTY_INDIA_TOURISM = "NIFTY INDIA TOURISM"
+    INDEX_NIFTY_CAPITAL_MARKETS = "NIFTY CAPITAL MARKETS"
+    INDEX_NIFTY_EV_NEW_AGE_AUTOMOTIVE = "NIFTY EV & NEW AGE AUTOMOTIVE"
+    INDEX_NIFTY_INDIA_NEW_AGE_CONSUMPTION = "NIFTY INDIA NEW AGE CONSUMPTION"
+    INDEX_NIFTY_INDIA_SELECT_5_CORPORATE_GROUPS_MAATR = "NIFTY INDIA SELECT 5 CORPORATE GROUPS (MAATR)"
+    INDEX_NIFTY_MOBILITY = "NIFTY MOBILITY"
+    INDEX_NIFTY_100_ENHANCED_ESG = "NIFTY100 ENHANCED ESG"
+    INDEX_NIFTY_CORE_HOUSING = "NIFTY CORE HOUSING"
+    INDEX_NIFTY_HOUSING = "NIFTY HOUSING"
+    INDEX_NIFTY_IPO = "NIFTY IPO"
+    INDEX_NIFTY_MIDSMALL_INDIA_CONSUMPTION = "NIFTY MIDSMALL INDIA CONSUMPTION"
+    INDEX_NIFTY_NON_CYCLICAL_CONSUMER = "NIFTY NON-CYCLICAL CONSUMER"
+    INDEX_NIFTY_RURAL = "NIFTY RURAL"
+    INDEX_NIFTY_SHARIAH_25 = "NIFTY SHARIAH 25"
+    INDEX_NIFTY_TRANSPORTATION_LOGISTICS = "NIFTY TRANSPORTATION & LOGISTICS"
+    INDEX_NIFTY_50_SHARIAH = "NIFTY50 SHARIAH"
+    INDEX_NIFTY_500_SHARIAH = "NIFTY500 SHARIAH"
+
+    # Strategy Indices:
+    INDEX_NIFTY_DIVIDEND_OPPORTUNITIES_50 = "NIFTY DIVIDEND OPPORTUNITIES 50"
+    INDEX_NIFTY_50_VALUE_20 = "NIFTY50 VALUE 20"
+    INDEX_NIFTY_100_QUALITY_30 = "NIFTY100 QUALITY 30"
+    INDEX_NIFTY_50_EQUAL_WEIGHT = "NIFTY50 EQUAL WEIGHT"
+    INDEX_NIFTY_100_EQUAL_WEIGHT = "NIFTY100 EQUAL WEIGHT"
+    INDEX_NIFTY_100_LOW_VOLATILITY_30 = "NIFTY100 LOW VOLATILITY 30"
+    INDEX_NIFTY_ALPHA_50 = "NIFTY ALPHA 50"
+    INDEX_NIFTY_200_QUALITY_30 = "NIFTY200 QUALITY 30"
+    INDEX_NIFTY_ALPHA_LOW_VOLATILITY_30 = "NIFTY ALPHA LOW-VOLATILITY 30"
+    INDEX_NIFTY_200_MOMENTUM_30 = "NIFTY200 MOMENTUM 30"
+    INDEX_NIFTY_MIDCAP_150_QUALITY_50 = "NIFTY MIDCAP150 QUALITY 50"
+    INDEX_NIFTY_200_ALPHA_30 = "NIFTY200 ALPHA 30"
+    INDEX_NIFTY_MIDCAP_150_MOMENTUM_50 = "NIFTY MIDCAP150 MOMENTUM 50"
+    INDEX_NIFTY_500_MOMENTUM_50 = "NIFTY500 MOMENTUM 50"
+    INDEX_NIFTY_MIDSMALLCAP_400_MOMENTUM_QUALITY_100 = "NIFTY MIDSMALLCAP400 MOMENTUM QUALITY 100"
+    INDEX_NIFTY_SMALLCAP_250_MOMENTUM_QUALITY_100 = "NIFTY SMALLCAP250 MOMENTUM QUALITY 100"
+    INDEX_NIFTY_TOP_10_EQUAL_WEIGHT = "NIFTY TOP 10 EQUAL WEIGHT"
+    INDEX_NIFTY_ALPHA_QUALITY_LOW_VOLATILITY_30 = "NIFTY ALPHA QUALITY LOW-VOLATILITY 30"
+    INDEX_NIFTY_ALPHA_QUALITY_VALUE_LOW_VOLATILITY_30 = "NIFTY ALPHA QUALITY VALUE LOW-VOLATILITY 30"
+    INDEX_NIFTY_HIGH_BETA_50 = "NIFTY HIGH BETA 50"
+    INDEX_NIFTY_LOW_VOLATILITY_50 = "NIFTY LOW VOLATILITY 50"
+    INDEX_NIFTY_QUALITY_LOW_VOLATILITY_30 = "NIFTY QUALITY LOW-VOLATILITY 30"
+    INDEX_NIFTY_SMALLCAP_250_QUALITY_50 = "NIFTY SMALLCAP250 QUALITY 50"
+    INDEX_NIFTY_TOP_15_EQUAL_WEIGHT = "NIFTY TOP 15 EQUAL WEIGHT"
+    INDEX_NIFTY_100_ALPHA_30 = "NIFTY100 ALPHA 30"
+    INDEX_NIFTY_200_VALUE_30 = "NIFTY200 VALUE 30"
+    INDEX_NIFTY_500_EQUAL_WEIGHT = "NIFTY500 EQUAL WEIGHT"
+    INDEX_NIFTY_500_MULTICAP_MOMENTUM_QUALITY_50 = "NIFTY500 MULTICAP MOMENTUM QUALITY 50"
+    INDEX_NIFTY_500_VALUE_50 = "NIFTY500 VALUE 50"
+    INDEX_NIFTY_TOP_20_EQUAL_WEIGHT = "NIFTY TOP 20 EQUAL WEIGHT"
+
+    # Others:
+    INDEX_SECURITIES_IN_FNO = "SECURITIES IN F&O"
+    INDEX_PERMITTED_TO_TRADE = "PERMITTED TO TRADE"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (IdxCons) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/live-equity-market",
+            data_url = r"https://www.nseindia.com/api/equity-stockIndices",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            index_name: str,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param index_name: The value held in the 'indexName' field of the formatted output of the Index Master.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            params = {"index": index_name},
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    index_name = index_name,
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            index_name: str,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param index_name: The value that you had used to fetch the raw data in the first place.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                {
+                    "scrapeTs": timestamp,
+                    "ts": date_time.to_timezone(
+                        date_time.parse_date_time(
+                            input_value = symbol["lastUpdateTime"],
+                            date_formats = ["%d-%b-%Y %H:%M:%S"],
+                            timezone = date_time.TIMEZONE_IST
+                        ),
+                        timezone = date_time.TIMEZONE_UTC
+                    ),
+                    "indexName": index_name,
+                    "symbol": symbol["symbol"],
+                    "name": symbol["meta"]["companyName"],
+                    "industry": symbol["meta"]["industry"],
+                    "isFNOSec": symbol["meta"]["isFNOSec"],
+                    "isSuspended": symbol["meta"]["isSuspended"],
+                    "isin": symbol["meta"]["isin"],
+                    "open": symbol["open"],
+                    "high": symbol["dayHigh"],
+                    "low": symbol["dayLow"],
+                    "close": symbol["lastPrice"],
+                    "totTradedVol": symbol["totalTradedVolume"],
+                    "totTradedVal": symbol["totalTradedValue"],
+                    "prevClose": symbol["previousClose"],
+                    "chg": symbol["change"],
+                    "pChg": symbol["pChange"],
+                    "yearHigh": symbol["yearHigh"],
+                    "yearLow": symbol["yearLow"],
+                    "pChg30d": symbol["perChange30d"],
+                    "pChg365d": symbol["perChange365d"],
+                    "ffmc": symbol["ffmc"]
+                } for symbol in raw_json["data"] if symbol["priority"] == 0
+            ]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEIndexConstituents(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(
+            index_name = NSEIndexConstituents.INDEX_NIFTY_50,
+            return_raw = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("INDEX CONSTITUENTS:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/indices/master.py

@@ -0,0 +1,284 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        The "Index Master" contains information about just the indices, and not the component symbols of those indices.
+        This script provides a way to get the data that is available on the screen on the following URL:
+        https://www.nseindia.com/market-data/live-market-indices
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+from utils_v2.nse.models.indices import NSEIndexInfo
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEIndexMaster(AsyncNSEBase):
+
+    # Index Types:
+    INDEX_TYPE_BROAD_MARKET = "BROAD MARKET INDICES"
+    INDEX_TYPE_SECTORAL = "SECTORAL INDICES"
+    INDEX_TYPE_STRATEGY = "STRATEGY INDICES"
+    INDEX_TYPE_THEMATIC = "THEMATIC INDICES"
+    INDEX_TYPE_FIXED_INCOME = "FIXED INCOME INDICES"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (IdxMstr) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/live-market-indices",
+            data_url = r"https://www.nseindia.com/api/allIndices",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def parse_float(value: str | float | int) -> float | None:
+
+        """
+        Parses a value as a floating point figure.
+        :param value: Any input value.
+        :return: A floating point value if parsed successfully, else None.
+        """
+
+        if value is None: return None
+        if isinstance(value, float): return value
+        if isinstance(value, int): return float(value)
+        if isinstance(value, str):
+            value = regex.find_first(text = value.strip(), pattern = r"[\d,]+\.?[\d,]*")
+            if value: return float(value.replace(",", ""))
+            else: return None
+
+    def format_data(
+            self,
+            raw_json: dict,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSEIndexInfo] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                NSEIndexInfo(
+                    scrapeTs = timestamp,
+                    kind = idx["key"],
+                    name = idx["index"],
+                    symbol = idx["indexSymbol"],
+                    open = self.parse_float(idx["open"]),
+                    high = self.parse_float(idx["high"]),
+                    low = self.parse_float(idx["low"]),
+                    close = self.parse_float(idx["last"]),
+                    prevClose = self.parse_float(idx["previousClose"]),
+                    pChg = self.parse_float(idx["percentChange"]),
+                    yearHigh = self.parse_float(idx["yearHigh"]),
+                    yearLow = self.parse_float(idx["yearLow"]),
+                    advances = self.parse_float(idx.get("advances")),
+                    declines = self.parse_float(idx.get("declines")),
+                    unchanged = self.parse_float(idx.get("unchanged")),
+                    pChg30d = self.parse_float(idx["perChange30d"]),
+                    pChg365d = self.parse_float(idx["perChange365d"])
+                ) for idx in raw_json["data"]
+            ]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEIndexMaster(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("INDEX MASTER:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/ipo/current.py

@@ -0,0 +1,252 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 10th Jan., 2025
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates and summaries of currently running IPO offers.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# Data models:
+from utils_v2.nse.models.ipo import NSEIPO
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSECurrentIPO(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (Curr. IPO) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/all-upcoming-issues-ipo",
+            data_url = r"https://www.nseindia.com/api/ipo-current-issue",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the currently running IPOs.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    def format_data(
+            self,
+            raw_json: List[dict],
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSEIPO] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = [
+                NSEIPO(
+                    scrapeTs = timestamp,
+                    symbol = ipo["symbol"],
+                    name = ipo["companyName"],
+                    issueStartTs = self.parse_datetime_string(
+                        dt_str = ipo["issueStartDate"],
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    ),
+                    issueEndTs = self.parse_datetime_string(
+                        dt_str = ipo["issueEndDate"],
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    ),
+                    status = ipo["status"],
+                    kind = ipo["series"],
+                    offerQty = float((ipo["noOfSharesOffered"])),
+                    bidQty = int(float(ipo["noOfsharesBid"])),
+                    bidFactor = float(ipo["noOfTime"]),
+                ) for ipo in raw_json
+            ]
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an instance of the scraper:
+        my_nse = NSECurrentIPO(debug_only_errors = False)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(
+            return_raw = False,
+            retry_count = 3
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("CURRENT IPO:", json.to_string(api_response.data, default = str))
+        print("COOKIES:", my_nse.cookies)
+
+    asyncio.run(main())

nse/controllers/ipo/past.py

@@ -0,0 +1,311 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 10th Jan., 2025
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates and summaries of IPOs offered in the past.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# Data models:
+from utils_v2.nse.models.ipo import NSEIPO
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEPastIPO(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (Past IPO) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/all-upcoming-issues-ipo",
+            data_url = r"https://www.nseindia.com/api/public-past-issues",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            from_date: datetime.datetime = None,
+            to_date: datetime.datetime = None,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the currently running IPOs.
+        :param from_date: The starting date (inclusive) from which the events must be fetched.
+        :param to_date: The ending date (inclusive) till which the events must be fetched.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Prepare the inputs:
+        now = date_time.get_current_utc_date_time(as_string = False)
+        one_year_ago = now - datetime.timedelta(days = 365)
+        from_date = date_time.to_timezone(from_date or one_year_ago, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
+        to_date = date_time.to_timezone(to_date or now, timezone = date_time.TIMEZONE_IST).strftime("%d-%m-%Y")
+
+        # Make the API call:
+        api_response = await self.get(
+            params = {
+                "from_date": from_date,
+                "to_date": to_date
+            },
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def get_band_prices(price_band: str) -> (float, float):
+
+        # Find all the matches:
+        matches = regex.find(
+            text = str(price_band),
+            pattern = r"[\d,]+\.?[\d,]*"
+        )
+
+        # Remove the commas if any creep in:
+        matches = [regex.replace(text = m, pattern = r"[ ,]*", substitute_text = "") for m in matches]
+
+        # Parse the matches:
+        match len(matches):
+            case 1: upper_band, lower_band = float(matches[0]), None
+            case 2: upper_band, lower_band = float(matches[0]), float(matches[1])
+            case _: upper_band, lower_band = None, None
+
+        # Done here:
+        return upper_band, lower_band
+
+    def format_data(
+            self,
+            raw_json: List[dict],
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSEIPO] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = []
+            for ipo in raw_json:
+                issue_price = regex.find_first(text = str(ipo.get("issuePrice")), pattern = r"[\d,]+\.?[\d,]*")
+                upper_band, lower_band = self.get_band_prices(ipo.get("priceRange"))
+                this_entry = NSEIPO(
+                    scrapeTs = timestamp,
+                    symbol = ipo["symbol"],
+                    name = ipo.get("company", ipo.get("companyName")),
+                    issueStartTs = self.parse_datetime_string(
+                        dt_str = ipo.get("ipoStartDate", ipo.get("issueStartDate")),
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    ),
+                    issueEndTs = self.parse_datetime_string(
+                        dt_str = ipo.get("ipoEndDate", ipo.get("issueEndDate")),
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    ),
+                    kind = ipo.get("securityType"),
+                    issuePrice = float(issue_price) if issue_price else None,
+                    upperBand = upper_band,
+                    lowerBand = lower_band,
+                    listingTs = self.parse_datetime_string(
+                        dt_str = ipo.get("listingDate"),
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    )
+                )
+                formatted_data.append(this_entry)
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an instance of the scraper:
+        my_nse = NSEPastIPO(debug_only_errors = False)
+
+        # for p in [
+        #     "Rs.1000",
+        #     "140",
+        #     "70",
+        #     "Rs.397 to Rs.418",
+        #     "Rs.397.10 to Rs.4,180.25",
+        # ]:
+        #     my_nse.get_band_prices(p)
+
+        # Get and show the data:
+        now = date_time.get_current_utc_date_time(as_string = False)
+        api_response = await my_nse.get_data(
+            from_date = now - datetime.timedelta(days = 365),
+            to_date = now,
+            return_raw = False,
+            retry_count = 3,
+            backoff_seconds = 1.0,
+            refresh_cookies = True,
+            force_refresh_cookies = False
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("PAST IPO:", json.to_string(api_response.data, default = str))
+        print("COOKIES:", my_nse.cookies)
+
+    asyncio.run(main())

nse/controllers/ipo/upcoming.py

@@ -0,0 +1,275 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Friday, 10th Jan., 2025
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates and summaries of upcoming IPO offers.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# Data models:
+from utils_v2.nse.models.ipo import NSEIPO
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEUpcomingIPO(AsyncNSEBase):
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient = None,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (Curr. IPO) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/all-upcoming-issues-ipo",
+            data_url = r"https://www.nseindia.com/api/all-upcoming-issues?category=ipo",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the currently running IPOs.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def get_band_prices(price_band: str) -> (float, float):
+
+        # Find all the matches:
+        matches = regex.find(
+            text = str(price_band),
+            pattern = r"[\d,]+\.?[\d,]*"
+        )
+
+        # Remove the commas if any creep in:
+        matches = [regex.replace(text = m, pattern = r"[ ,]*", substitute_text = "") for m in matches]
+
+        # Parse the matches:
+        match len(matches):
+            case 1: upper_band, lower_band = float(matches[0]), None
+            case 2: upper_band, lower_band = float(matches[0]), float(matches[1])
+            case _: upper_band, lower_band = None, None
+
+        # Done here:
+        return upper_band, lower_band
+
+    def format_data(
+            self,
+            raw_json: List[dict],
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[NSEIPO] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Format the data:
+            formatted_data = []
+            for ipo in raw_json:
+                issue_size = regex.find_first(text = str(ipo.get("issueSize")), pattern = r"[\d,]+\.?[\d,]*")
+                upper_band, lower_band = self.get_band_prices(ipo.get("priceRange"))
+                formatted_data.append(NSEIPO(
+                    scrapeTs = timestamp,
+                    symbol = ipo["symbol"],
+                    name = ipo["companyName"],
+                    issueStartTs = self.parse_datetime_string(
+                        dt_str = ipo["issueStartDate"],
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    ),
+                    issueEndTs = self.parse_datetime_string(
+                        dt_str = ipo["issueEndDate"],
+                        dt_formats = ["%d-%b-%Y"],
+                        input_tz = date_time.TIMEZONE_IST,
+                        output_tz = date_time.TIMEZONE_UTC
+                    ),
+                    status = ipo["status"],
+                    kind = ipo["series"],
+                    offerQty = int(float(issue_size)) if issue_size else None,
+                    upperBand = upper_band,
+                    lowerBand = lower_band,
+                ))
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an instance of the scraper:
+        my_nse = NSEUpcomingIPO(debug_only_errors = False)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(
+            return_raw = False,
+            retry_count = 3
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("UPCOMING IPO:", json.to_string(api_response.data, default = str))
+
+    asyncio.run(main())

nse/controllers/market/pre_market.py

@@ -0,0 +1,305 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 5th Dec., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve pre-market data from NSE. This is typically available by 9:10 AM.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.string import regex
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+from utils_v2.nse.models.pre_market import NSEPreMarketData, NSEPreMarketSymbol
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEPreMarket(AsyncNSEBase):
+
+    # Symbol names:
+    PRE_MARKET_KEY_NIFTY = "NIFTY"
+    PRE_MARKET_KEY_BANK_NIFTY = "BANKNIFTY"
+    PRE_MARKET_KEY_SME = "SME"
+    PRE_MARKET_KEY_FO = "FO"
+    PRE_MARKET_KEY_OTHERS = "OTHERS"
+    PRE_MARKET_KEY_ALL = "ALL"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (CECal) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/market-data/pre-open-market-cm-and-emerge-market",
+            data_url = r"https://www.nseindia.com/api/market-data-pre-open",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            key: str,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the pre-open market trading. Useful for finding gaps and expected unusual activity in the
+        trading hours.
+        :param key: The type of pre-market data that you want. Choose from the class variables.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            params = {"key": key},
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    key = key,
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    @staticmethod
+    def format_data(
+            raw_json: dict,
+            key: str = None,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> NSEPreMarketData | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param key: A choice between "NIFTY", "BANKNIFTY", "SME", "FO", "OTHERS", "ALL".
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_data = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Start by extracting basic data:
+            formatted_data = {
+                "scrapeTs": timestamp,
+                "ts": date_time.to_timezone(
+                    date_time.as_if_timezone(
+                        date_time.parse_date_time(
+                            input_value = raw_json["timestamp"],
+                            date_formats = ["%d-%b-%Y %H:%M:%S"]
+                        ),
+                        timezone = date_time.TIMEZONE_IST
+                    ),
+                    timezone = date_time.TIMEZONE_UTC
+                ),
+                "key": key,
+                "advances": raw_json["advances"],
+                "declines": raw_json["declines"],
+                "unchanged": raw_json["unchanged"],
+                "totalMarketCap": raw_json["totalmarketcap"],
+                "totalTradedValue": raw_json["totalTradedValue"],
+                "totalTradedVolume": raw_json["totalTradedVolume"],
+                "symbols": []
+            }
+
+            # Now we iterate through the symbol-wise data and extract what we need:
+            for raw_symbol_data in raw_json["data"]:
+                raw_symbol_metadata = raw_symbol_data["metadata"]
+                raw_symbol_detail = raw_symbol_data["detail"]["preOpenMarket"]
+                market_cap = regex.find_first(text = str(raw_symbol_metadata["marketCap"]), pattern = r"[\d,]+\.?[\d,]*")
+                formatted_data["symbols"].append({
+                    "symbol": raw_symbol_metadata["symbol"],
+                    "ffmc": float(market_cap) if market_cap else None,
+                    "trigger": raw_symbol_metadata["purpose"],
+                    "yearHigh": raw_symbol_metadata["yearHigh"],
+                    "yearLow": raw_symbol_metadata["yearLow"],
+                    "prevClose": raw_symbol_metadata["previousClose"],
+                    "preMarketPrice": raw_symbol_metadata["iep"],
+                    "chg": raw_symbol_metadata["change"],
+                    "pChg": raw_symbol_metadata["pChange"],
+                    "totalTradedVolume": raw_symbol_detail["totalTradedVolume"],
+                    "totalBuyVolume": raw_symbol_detail["totalBuyQuantity"],
+                    "totalSellVolume": raw_symbol_detail["totalSellQuantity"],
+                })
+
+            # Data sorting (descending order of percent change):
+            formatted_data["symbols"] = sorted(
+                formatted_data["symbols"],
+                key = lambda x: x["pChg"],
+                reverse = True
+            )
+
+            # Model the data:
+            formatted_data = NSEPreMarketData(**formatted_data)
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_data = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_data
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 9.9  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEPreMarket(http_client = test_client)
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(key = my_nse.PRE_MARKET_KEY_FO, return_raw = False)
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("PRE-MARKET DATA:", json.to_string(api_response.data, default = str))
+        if api_response.exception: raise api_response.exception
+        print("COUNT:", len(api_response.data.symbols))
+
+    asyncio.run(main())

nse/controllers/options/index.py

@@ -0,0 +1,315 @@
+"""
+
+    AUTHOR:
+
+        Khushal P Soonderji
+
+    DATE:
+
+        Thursday, 28th Nov., 2024
+
+    OBJECTIVE:
+
+        To provide a way to retrieve dates of important events like financial-results, stock-splits, fund-raising, etc.
+        from NSE's portal.
+
+        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
+        more professional data-sources be used when the product starts becoming mature.
+
+    REFERENCES:
+
+        N/A
+
+    DOWNLOADS:
+
+        N/A
+
+"""
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                   IMPORT                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# To make sibling directories accessible for imports:
+import sys
+sys.path.append(".")
+sys.path.append("..")
+
+# System-level activities:
+import io
+
+# My utils:
+from utils_v2.string import json
+from utils_v2.date_time import date_time
+
+# NSE-related utils:
+from utils_v2.nse.controllers.base import AsyncNSEBase
+from utils_v2.nse.models.api_call import NSEApiResponse
+
+# To make REST-ful API calls:
+import httpx
+
+# To work with date and time:
+import datetime
+
+# To work with datatypes:
+from typing import Any, List
+
+# For asynchronous activities:
+import asyncio
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                           MACROS / ONE-TIME INIT                                              ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                VARIABLES                                                      ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+# --- Nothing Yet
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                                  FUNCTIONS                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+class NSEIndexOptionChain(AsyncNSEBase):
+
+    # Index Names:
+    INDEX_NIFTY = "NIFTY"
+    INDEX_BANKNIFTY = "BANKNIFTY"
+    INDEX_FINNIFTY = "FINNIFTY"
+    INDEX_MIDCPNIFTY = "MIDCPNIFTY"
+    INDEX_NIFTYNXT50 = "NIFTYNXT50"
+
+    def __init__(
+            self,
+            http_client: httpx.AsyncClient,
+            cookies_refresh_interval: int | float = 300,
+            debug = True,
+            debug_prefix = "NSE (IdxOC) | ",
+            debug_only_errors = True
+    ):
+
+        # Pass on the initialization to the parent:
+        super().__init__(
+            base_url = r"https://www.nseindia.com/option-chain",
+            data_url = r"https://www.nseindia.com/api/option-chain-indices",
+            http_client = http_client,
+            cookies_refresh_interval = cookies_refresh_interval,
+            debug = debug,
+            debug_prefix = debug_prefix,
+            debug_only_errors = debug_only_errors
+        )
+
+    async def get_data(
+            self,
+            index_name: str,
+            return_raw: bool = False,
+            refresh_cookies: bool = True,
+            force_refresh_cookies: bool = False,
+            retry_count: int = 1,
+            backoff_seconds: int | float = 0.5,
+            backoff_multiplier: float = 1.1
+    ) -> NSEApiResponse:
+
+        """
+        To get the data of the corporate event calendar.
+        :param index_name: The name of the index whose option chain is needed. Use one of the options created within the
+            scope of this class.
+        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
+        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
+        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
+            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
+            successful refresh.
+        :param retry_count: The no. of times to try to get the data from the API.
+        :param backoff_seconds: The delay between unsuccessful API calls.
+        :param backoff_multiplier: The multiplier to add to the delay to change delay.
+        :return: The raw or formatted event calendar data in the 'data' field of the response model.
+        """
+
+        # Make the API call:
+        api_response = await self.get(
+            params = {"symbol": index_name},
+            refresh_cookies = refresh_cookies,
+            force_refresh_cookies = force_refresh_cookies,
+            retry_count = retry_count,
+            backoff_seconds = backoff_seconds,
+            backoff_multiplier = backoff_multiplier
+        )
+
+        # If the API call was successful:
+        if api_response.httpCode in [200]:
+            api_response.success = True
+            if return_raw: api_response.data = await api_response.get_json()
+            else:
+                try: api_response.data = self.format_data(
+                    raw_json = await api_response.get_json(),
+                    timestamp = date_time.get_current_utc_date_time(as_string = False),
+                    raise_exception = True
+                )
+                except Exception as exception:
+                    api_response.exception = exception
+                    api_response.success = False
+
+        # Done here:
+        return api_response
+
+    def format_data(
+            self,
+            raw_json: dict,
+            timestamp: datetime.datetime = None,
+            raise_exception: bool = False
+    ) -> List[dict] | None:
+
+        """
+        We format the data here to be able to retrieve it properly later.
+        :param raw_json: The raw data as scraped from NSE.
+        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
+            database, later.
+        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
+            suppressed internally.
+        :return: The formatted data if successful, else None.
+        """
+
+        # Can't do anything if the chain itself is null:
+        if raw_json is None: return raw_json
+
+        # Start by assuming failure:
+        formatted_chain = None
+
+        # Ensure that we've got a proper timestamp:
+        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
+
+        try:
+
+            # Start at the expiry-level:
+            expiry_dates = raw_json["records"]["expiryDates"]
+            semi_formatted_chain = {e: None for e in expiry_dates}
+
+            # Iterate through the raw records and start plugging them into the formatted records:
+            for record in raw_json["records"]["data"]:
+
+                # Extract some variables:
+                expiry_date = record["expiryDate"]
+                strike_price = record["strikePrice"]
+
+                # Start building the needed structure:
+                if semi_formatted_chain.get(expiry_date) is None: semi_formatted_chain[expiry_date] = {}
+                if semi_formatted_chain[expiry_date].get(strike_price) is None:
+                    semi_formatted_chain[expiry_date][strike_price] = {
+                        "scrapeTs": timestamp,
+                        "strike": record["strikePrice"],
+                        "expiry": record["expiryDate"],
+                        "expiryTs": self.parse_datetime_string(
+                            dt_str = record["expiryDate"],
+                            dt_formats = ["%d-%b-%Y"],
+                            input_tz = date_time.TIMEZONE_IST,
+                            output_tz = date_time.TIMEZONE_UTC
+                        ),
+                        "expiryTz": "Asia/Kolkata",
+                    }
+
+                # Add the CE/PE data:
+                for right in ["CE", "PE"]:
+                    right_data = record.get(right, {})
+                    if right_data:
+                        semi_formatted_chain[expiry_date][strike_price]["underlying"] = right_data["underlying"]
+                        semi_formatted_chain[expiry_date][strike_price]["underlyingValue"] = right_data["underlyingValue"]
+                    semi_formatted_chain[expiry_date][strike_price][right.lower()] = {}
+                    semi_formatted_chain[expiry_date][strike_price][right.lower()] = {
+                        "id": right_data.get("identifier"),
+                        "oi": right_data.get("openInterest"),
+                        "oiChg": right_data.get("changeinOpenInterest"),
+                        "oiPctChg": right_data.get("pchangeinOpenInterest"),
+                        "totTradedVol": right_data.get("totalTradedVolume"),
+                        "iv": right_data.get("impliedVolatility"),
+                        "ltp": right_data.get("lastPrice"),
+                        "change": right_data.get("change"),
+                        "pChange": right_data.get("pChange"),
+                        "totBuyQty": right_data.get("totalBuyQuantity"),
+                        "totSellQty": right_data.get("totalSellQuantity"),
+                        "bidQty": right_data.get("bidQty"),
+                        "bidPrice": right_data.get("bidprice"),
+                        "askQty": right_data.get("askQty"),
+                        "askPrice": right_data.get("askPrice")
+                    }
+
+                # Final formatting:
+                formatted_chain = []
+                for expiry, _0 in semi_formatted_chain.items():
+                    if isinstance(_0, dict):
+                        for strike_price, _1 in _0.items():
+                            _1["ce"] = _1.pop("ce")
+                            _1["pe"] = _1.pop("pe")
+                            formatted_chain.append(_1)
+
+        # If something goes wrong:
+        except Exception as exception:
+            formatted_chain = None
+            if raise_exception: raise
+
+        # Done here:
+        return formatted_chain
+
+
+#   *****************************************************************************************************************
+#  *****                                                                                                          ****
+# ***                                               MAIN PROGRAM                                                    ***
+#  *****                                                                                                          ****
+#   *****************************************************************************************************************
+
+
+if __name__ == "__main__":
+
+    async def main():
+
+        # Create an HTTP client:
+        test_client = httpx.AsyncClient(
+            limits = httpx.Limits(
+                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
+                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
+            ),
+            timeout = httpx.Timeout(
+                pool = 120.0,  # .... Time to wait for a free connection from the pool.
+                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
+                write = 10.0,  # .... Time to wait for sending data.
+                read = 2.5  # ....... Time to wait for receiving data.
+            )
+        )
+
+        # Create an instance of the scraper, and refresh its cookies:
+        my_nse = NSEIndexOptionChain(
+            http_client = test_client,
+            debug_only_errors = False
+        )
+
+        # Get and show the data:
+        api_response = await my_nse.get_data(
+            index_name = NSEIndexOptionChain.INDEX_NIFTY,
+            return_raw = False,
+            retry_count = 3,
+            backoff_seconds = 1,
+            backoff_multiplier = 2.0
+        )
+        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
+        if api_response.success: print("OPTION CHAIN:", json.to_string(api_response.data[:3], default = str))
+
+    asyncio.run(main())