"""

    AUTHOR:

        Khushal P Soonderji

    DATE:

        Saturday, 4th Jan., 2025.

    OBJECTIVE:

        To provide a way to fetch the daily BhavCopy from NSE.

        NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
        more professional data-sources be used when the product starts becoming mature.

    REFERENCES:

        N/A

    DOWNLOADS:

        N/A

"""


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                                   IMPORT                                                      ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")

# System-level activities:
import io

# My utils:
from utils_v2.string import json
from utils_v2.date_time import date_time

# NSE-related utils:
from utils_v2.nse.controllers.base import AsyncNSEBase
from utils_v2.nse.models.api_call import NSEApiResponse

# To make REST-ful API calls:
import httpx

# To work with date and time:
import datetime
import pytz

# To work with datatypes:
from typing import Any, List

# To work with tabulated dataL
import pandas as pd

# For asynchronous activities:
import asyncio


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                           MACROS / ONE-TIME INIT                                              ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


# --- Nothing Yet


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                                VARIABLES                                                      ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


# --- Nothing Yet


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                                  FUNCTIONS                                                    ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


class NSEEquitiesBhavCopy(AsyncNSEBase):

    def __init__(
            self,
            http_client: httpx.AsyncClient,
            cookies_refresh_interval: int | float = 300,
            debug = True,
            debug_prefix = "NSE (Eq. BhavCopy) | ",
            debug_only_errors = True
    ):

        # Pass on the initialization to the parent:
        super().__init__(
            base_url = r"https://www.nseindia.com/all-reports",
            data_url = r"https://nsearchives.nseindia.com/products/content/sec_bhavdata_full_03012025.csv",
            http_client = http_client,
            cookies_refresh_interval = cookies_refresh_interval,
            debug = debug,
            debug_prefix = debug_prefix,
            debug_only_errors = debug_only_errors
        )

    @staticmethod
    def parse_float(value):
        try: return float(value)
        except ValueError: return None

    @staticmethod
    def parse_int(value):
        try: return int(value)
        except ValueError: return None

    async def get_data(
            self,
            target_date: datetime.datetime,
            return_raw: bool = False,
            refresh_cookies: bool = True,
            force_refresh_cookies: bool = False,
            retry_count: int = 1,
            backoff_seconds: int | float = 0.5,
            backoff_multiplier: float = 1.1
    ) -> NSEApiResponse:

        """
        To get the data of the corporate event calendar.
        :param target_date: The starting date (inclusive) from which the events must be fetched.
        :param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
        :param refresh_cookies: Whether, or not, you would like to refresh the cookies.
        :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
            cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
            successful refresh.
        :param retry_count: The no. of times to try to get the data from the API.
        :param backoff_seconds: The delay between unsuccessful API calls.
        :param backoff_multiplier: The multiplier to add to the delay to change delay.
        :return: The raw or formatted event calendar data in the 'data' field of the response model.
        """

        # Figure out the file name:
        date_time.to_timezone(target_date, date_time.TIMEZONE_IST)
        file_name = f"sec_bhavdata_full_{target_date.strftime('%d%m%Y')}.csv"

        # Make the API call:
        api_response = await self.get(
            url = f"https://nsearchives.nseindia.com/products/content/{file_name}",
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
            },
            refresh_cookies = refresh_cookies,
            force_refresh_cookies = force_refresh_cookies,
            retry_count = retry_count,
            backoff_seconds = backoff_seconds,
            backoff_multiplier = backoff_multiplier
        )

        # If the API call was successful:
        if api_response.httpCode in [200]:
            api_response.success = True
            if return_raw:
                api_response.data = io.BytesIO(await api_response.get_content())
                api_response.data.seek(0)
            else:
                try: api_response.data = self.format_data(
                    raw_data = await api_response.get_content(),
                    timestamp = date_time.get_current_utc_date_time(as_string = False),
                    raise_exception = True
                )
                except Exception as exception:
                    api_response.exception = exception
                    api_response.success = False

        # Done here:
        return api_response

    def format_data(
            self,
            raw_data: bytes | io.BytesIO,
            timestamp: datetime.datetime = None,
            raise_exception: bool = False
    ) -> List[dict] | None:

        """
        We format the data here to be able to retrieve it properly later.
        :param raw_data: The raw data as scraped from NSE.
        :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
            database, later.
        :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
            suppressed internally.
        :return: The formatted data if successful, else None.
        """

        # Can't do anything if the chain itself is null:
        if raw_data is None: return raw_data

        # Start by assuming failure:
        formatted_data = None

        # Ensure that we've got a proper timestamp:
        if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)

        try:

            # Ensure that we have a BytesIO object,
            # and that we are pointing at the start:
            if isinstance(raw_data, bytes):
                raw_data = io.BytesIO(raw_data)
            raw_data.seek(0)

            # Read the data into Pandas:
            bhav_df = pd.read_csv(raw_data)
            bhav_df.rename(
                columns = {c: c.strip() for c in bhav_df.columns.to_list()},
                inplace = True
            )
            bhav_df.rename(
                columns = {
                    "SYMBOL": "symbol",
                    "SERIES": "segment",
                    "DATE1": "date",
                    "PREV_CLOSE": "prevClose",
                    "OPEN_PRICE": "open",
                    "HIGH_PRICE": "high",
                    "LOW_PRICE": "low",
                    "CLOSE_PRICE": "close",
                    "LAST_PRICE": "ltp",
                    "AVG_PRICE": "vwap",
                    "TTL_TRD_QNTY": "totVol",
                    "TURNOVER_LACS": "totCash",
                    "NO_OF_TRADES": "tradeCount",
                    "DELIV_QTY": "deliveryVol",
                    "DELIV_PER": "deliveryPct"
                },
                inplace = True
            )

            # Format the data:
            for column in ["symbol", "segment", "date"]:
                bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip())
            for column in ["prevClose", "open", "high", "low", "close", "ltp", "vwap", "totCash", "deliveryPct"]:
                bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x))
            for column in ["totVol", "deliveryVol"]:
                bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x))
            bhav_df["totCash"] *= 1_00_000

            # Give the date as a timestamp:
            bhav_df["scrapeTs"] = timestamp
            bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string(
                dt_str = x,
                dt_formats = ["%d-%b-%Y"],
                input_tz = date_time.TIMEZONE_IST,
                output_tz = date_time.TIMEZONE_UTC
            ))
            bhav_df["tz"] = "Asia/Kolkata"

            # Keep what is needed, in an order that makes intuitive sense:
            bhav_df = bhav_df[[
                "symbol", "segment",
                "prevClose", "open", "high", "low", "close", "ltp", "vwap",
                "totVol", "totCash", "deliveryVol", "deliveryPct",
                "date", "ts", "tz", "scrapeTs"
            ]]

            # Done here:
            formatted_data = bhav_df.to_dict(orient = "records")

        # If something goes wrong:
        except Exception as exception:
            formatted_data = None
            if raise_exception: raise

        # Done here:
        return formatted_data


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                               MAIN PROGRAM                                                    ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


if __name__ == "__main__":

    async def main():

        # Create an HTTP client:
        test_client = httpx.AsyncClient(
            limits = httpx.Limits(
                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
            ),
            timeout = httpx.Timeout(
                pool = 120.0,  # .... Time to wait for a free connection from the pool.
                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
                write = 10.0,  # .... Time to wait for sending data.
                read = 2.5  # ....... Time to wait for receiving data.
            )
        )

        # Create an instance of the scraper, and refresh its cookies:
        my_nse = NSEEquitiesBhavCopy(http_client = test_client)

        # Get and show the data:
        target_date = datetime.datetime.strptime("20250103", "%Y%m%d")
        api_response = await my_nse.get_data(
            target_date = target_date,
            return_raw = False
        )
        print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
        if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str))

    asyncio.run(main())
