"""

    AUTHOR:

        Khushal P Soonderji

    DATE:

        Wednesday, 1st Jan., 2025.

    OBJECTIVE:

        To capture end-of-day market data and store it in the database.

    REFERENCES:

        N/A

    DOWNLOADS:

        N/A

"""


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                                   IMPORT                                                      ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


# To make sibling directories accessible for imports:
import sys
sys.path.append(".")
sys.path.append("..")

# System-level activities:
import io
import os

# My utils:
from utils_v2.string import json
from utils_v2.string import regex
from utils_v2.system import files
from utils_v2.date_time import date_time
from utils_v2.database.async_mysql_v2 import AsyncMySQL

# To make HTTP calls:
import httpx
import socket

# To work with date and time:
import datetime
import time

# To work with NSE:
from utils_v2.nse.controllers.bhavcopy.equities import NSEEquitiesBhavCopy
from utils_v2.nse.controllers.bhavcopy.fno import NSEFNOBhavCopy

# To work with datatypes:
from typing import List

# For handling NaN values:
import pandas as pd

# For debugging:
from icecream import IceCreamDebugger

# For asynchronous operations:
import asyncio


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                           MACROS / ONE-TIME INIT                                              ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


# To make API calls:
http_client = httpx.AsyncClient(
    limits = httpx.Limits(
        max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
        max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
    ),
    timeout = httpx.Timeout(
        pool = 120.0,  # .... Time to wait for a free connection from the pool.
        connect = 2.5,  # ... Time to wait for establishing a connection to the server.
        write = 10.0,  # .... Time to wait for sending data.
        read = 9.9  # ....... Time to wait for receiving data.
    )
)

# For debugging:
printer = IceCreamDebugger(prefix = "EoD Data | ", includeContext = True)
no_context_printer = IceCreamDebugger(prefix = "EoD Data | ", includeContext = False)


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                                VARIABLES                                                      ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


# This script's config.:
SERVER_HOSTNAME = str(socket.gethostname())
SCRIPT_DATA = {}

# For the database:
sql_writer: AsyncMySQL | None = None

# To construct a report:
failed_to_parse_dates = []
failed_eq_dates = []
failed_fo_dates = []


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                                  FUNCTIONS                                                    ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


async def init(
        script_id: str,
        debug: bool
) -> bool:

    """
    To initialize all credentials, instances, and connectivity for this whole script.
    :param script_id: The id to use to load cred and data from the internal service.
    :param debug: Whether, or not, you would like to print the debug messages.
    :return: True if initialized successfully, else False.
    """

    # Declare the required global variables:
    global SCRIPT_DATA
    global sql_writer

    # Basic stuff:
    if not debug: printer.disable()

    # ┏┓     ┓       ┓  ┳┓
    # ┃ ┏┓┏┓┏┫  ┏┓┏┓┏┫  ┃┃┏┓╋┏┓
    # ┗┛┛ ┗ ┗┻  ┗┻┛┗┗┻  ┻┛┗┻┗┗┻

    # Get the script credentials:
    response = await http_client.get(
        url = r"https://nexcom.ditscentre.in/internal/cred/get",
        headers = {"X-Script-Id": script_id}
    )
    if response.status_code not in [200]:
        print("FATAL: SCRIPT CREDENTIALS LOADING FAILED!")
        return False
    script_cred = response.json().get("data")

    # Get the script data:
    response = await http_client.get(
        url = r"https://nexcom.ditscentre.in/internal/data/get",
        headers = {"X-Script-Id": script_id}
    )
    if response.status_code not in [200]:
        print("FATAL: SCRIPT DATA LOADING FAILED!")
        return False
    SCRIPT_DATA = response.json().get("data")

    # Done with this step:
    printer("Cred and Data loaded.")

    # ┳┳┓    •  ┳┓┳┓
    # ┃┃┃┏┓┏┓┓┏┓┃┃┣┫
    # ┛ ┗┗┻┛ ┗┗┻┻┛┻┛

    sql_writer = AsyncMySQL(
        pool_size = script_cred["mariaDb"]["write"]["poolSize"],
        host = script_cred["mariaDb"]["write"]["host"],
        user = script_cred["mariaDb"]["write"]["user"],
        password = script_cred["mariaDb"]["write"]["password"],
        database = script_cred["mariaDb"]["write"]["database"]
    )
    if not await sql_writer.connect():
        print("FATAL: MARIA-DB CONNECTION FAILED!")
        return False

    printer("MariaDB connected.")

    # ┳┓
    # ┃┃┏┓┏┓┏┓
    # ┻┛┗┛┛┗┗

    # If everything went well, we return with success:
    printer("Initialization done.")
    return True


# ---------------------------------------------------------------------------------------------------------------------


async def capture_eq_eod_data(
        nse_conn: NSEEquitiesBhavCopy,
        target_date: datetime.datetime
) -> bool:

    # Start by assuming failure:
    success = False

    # Get the data:
    no_context_printer("Fetching EQ data.")
    nse_response = await nse_conn.get_data(
        target_date = target_date,
        return_raw = False,
        retry_count = 3,
        backoff_seconds = 0.5,
        backoff_multiplier = 1.1
    )
    if not nse_response.success: return success

    # Build the query and data content:
    query_str = (
        "INSERT INTO `eod_market_data_temp` (exchange, symbol, segment, type, underlying, is_index, expiry, strike, "
        "prev_close, open, high, low, close, ltp, vwap, tot_vol, tot_cash, delivery_vol, delivery_pct, oi, oi_chg, "
        "date, ts, tz, scrape_ts) "
        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
    )
    query_data = []
    for data in nse_response.data:
        one_query_data = (
            "nse",  # ..................................... exchange
            data["symbol"],  # ............................ symbol
            data["segment"],  # ........................... segment
            None,  # ...................................... type
            None,  # ...................................... underlying
            False,  # ..................................... is_index
            None,  # ...................................... expiry
            None,  # ...................................... strike
            data["prevClose"],  # ......................... prev_close
            data["open"],  # .............................. open
            data["high"],  # .............................. high
            data["low"],  # ............................... low
            data["close"],  # ............................. close
            data["ltp"],  # ............................... ltp
            data["vwap"],  # .............................. vwap
            data["totVol"],  # ............................ tot_vol
            data["totCash"],  # ........................... tot_cash
            data["deliveryVol"],  # ....................... delivery_vol
            data["deliveryPct"],  # ....................... delivery_pct
            None,  # ...................................... oi
            None,  # ...................................... oi_chg
            target_date,  # ............................... date
            data["ts"].replace(tzinfo = None),  # ......... ts
            data["tz"],  # ................................ tz
            data["scrapeTs"].replace(tzinfo = None),  # ... scrape_ts
        )
        one_query_data = [None if pd.isna(d) else d for d in one_query_data]
        query_data.append(one_query_data)

    # Run the commands:
    no_context_printer("Saving EQ data to DB.")
    rows_affected, db_response = await sql_writer.execute_many(
        query = query_str,
        data = query_data
    )

    # Done here:
    success = True if rows_affected else False
    no_context_printer(success)
    return success


# ---------------------------------------------------------------------------------------------------------------------


async def capture_fo_eod_data(
        nse_conn: NSEFNOBhavCopy,
        target_date: datetime.datetime
) -> bool:

    # Start by assuming failure:
    success = False

    # Get the data:
    no_context_printer("Fetching FO data.")
    nse_response = await nse_conn.get_data(
        target_date = target_date,
        return_raw = False,
        retry_count = 3,
        backoff_seconds = 0.5,
        backoff_multiplier = 1.1
    )
    if not nse_response.success: return success

    # Build the query and data content:
    query_str = (
        "INSERT INTO `eod_market_data_temp` (exchange, symbol, segment, type, underlying, is_index, expiry, strike, "
        "prev_close, open, high, low, close, ltp, vwap, tot_vol, tot_cash, delivery_vol, delivery_pct, oi, oi_chg, "
        "date, ts, tz, scrape_ts) "
        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
    )
    query_data = []
    for data in nse_response.data:
        expiry = data["expiryTs"]
        if isinstance(expiry, pd._libs.tslibs.timestamps.Timestamp): expiry = expiry.to_pydatetime()
        if isinstance(expiry, datetime.datetime):
            expiry = date_time.to_timezone(
                datetime_object = expiry,
                timezone = date_time.TIMEZONE_UTC
            ).strftime("%Y-%m-%d")
        one_query_data = (
            "nse",  # ..................................... exchange
            data["symbol"],  # ............................ symbol
            data["segment"],  # ........................... segment
            data["type"],  # .............................. type
            data["underlying"],  # ........................ underlying
            data["isIndex"],  # ........................... is_index
            expiry,  # .................................... expiry
            data["strike"],  # ............................ strike
            data["prevClose"],  # ......................... prev_close
            data["open"],  # .............................. open
            data["high"],  # .............................. high
            data["low"],  # ............................... low
            data["close"],  # ............................. close
            None,  # ...................................... ltp
            None,  # ...................................... vwap
            data["totVol"],  # ............................ tot_vol
            data["totCash"],  # ........................... tot_cash
            None,  # ...................................... delivery_vol
            None,  # ...................................... delivery_pct
            data["oi"],  # ................................ oi
            data["oiChg"],  # ............................. oi_chg
            target_date,  # ............................... date
            data["ts"].replace(tzinfo = None),  # ......... ts
            data["tz"],  # ................................ tz
            data["scrapeTs"].replace(tzinfo = None),  # ... scrape_ts
        )
        one_query_data = [None if pd.isna(d) else d for d in one_query_data]
        query_data.append(one_query_data)

    # Run the commands:
    print(query_data[3034][19])
    print(query_data[3035][19])
    print(query_data[3036][19])
    print("---")
    print(query_data[2731][19])
    print(query_data[2732][19])
    print(query_data[2733][19])
    no_context_printer("Saving FO data to DB.")
    rows_affected, db_response = await sql_writer.execute_many(
        query = query_str,
        data = query_data
    )

    # Done here:
    success = True if rows_affected else False
    no_context_printer(success)
    return success


# ---------------------------------------------------------------------------------------------------------------------


async def main(
        target_dates: List[str],
        interval: float = 5.0
):

    # Declare the needed global variables:
    global failed_to_parse_dates
    global failed_eq_dates
    global failed_fo_dates

    # Create the NSE connection instances:
    nse_eq_bhavcopy = NSEEquitiesBhavCopy(http_client = http_client, debug = False)
    nse_fo_bhavcopy = NSEFNOBhavCopy(http_client = http_client, debug = False)

    # Load the data for each date:
    for dt_str in target_dates:

        # Try parsing the date:
        dt = date_time.parse_date_time(
            input_value = dt_str,
            date_formats = ["%Y-%m-%d", "%Y%m%d"]
        )

        # If we failed to parse the date:
        if dt is None:
            no_context_printer("FAILED to parse date.", dt_str)
            failed_to_parse_dates.append(dt_str)
            continue

        no_context_printer("Picked date.", dt_str)

        # # Fetch the equity data:
        # success = await capture_eq_eod_data(
        #     nse_conn = nse_eq_bhavcopy,
        #     target_date = dt
        # )
        # if not success:
        #     no_context_printer("FAILED equity for date.", dt_str)
        #     failed_eq_dates.append(dt_str)

        # Fetch the derivative data:
        success = await capture_fo_eod_data(
            nse_conn = nse_fo_bhavcopy,
            target_date = dt
        )
        if not success:
            no_context_printer("FAILED derivatives for date.", dt_str)
            failed_fo_dates.append(dt_str)

        # Pause for a while to not get rate-limited/blocked:
        await asyncio.sleep(interval)
        # break

    # Show the report:
    printer("Process complete.", failed_to_parse_dates, failed_eq_dates, failed_fo_dates)


#   *****************************************************************************************************************
#  *****                                                                                                          ****
# ***                                               MAIN PROGRAM                                                    ***
#  *****                                                                                                          ****
#   *****************************************************************************************************************


if __name__ == "__main__":

    # To get args. from the terminal:
    import argparse

    # Get the config. from the command-line:
    parser = argparse.ArgumentParser(
        description = (
            "To fetch and store EoD data for one or more dates from NSE's BhavCopy section. "
            "The data will be fetched for both, equities and derivatives."
        )
    )
    parser.add_argument(
        "-s", "--script-id",
        dest = "script_id",
        type = str,
        help = "The id of this script (will affect the loaded config)."
    )
    parser.add_argument(
        "-t", "--dates",
        dest = "dates",
        type = str,
        help = "One or more comma-separated dates in 'YYYY-MM-DD' format for which you would like to get the EoD data.",
        default = None
    )
    parser.add_argument(
        "-i", "--interval",
        dest = "interval",
        type = float,
        help = "The delay, in seconds, between processing two dates.",
        default = 2.5
    )
    parser.add_argument(
        "-d", "--debug",
        dest = "debug",
        action = "store_true",
        help = "Whether, or not, you want to see debugging messages in the terminal.",
        default = False
    )
    args = parser.parse_args()


    async def runner():

        # Prepare the list of target dates:
        if args.dates is None: target_dates = [datetime.datetime.now().strftime("%Y-%m-%d")]
        else: target_dates = [d.strip() for d in args.dates.split(",")]

        # Initialize and run the main code:
        if await init(
            script_id = args.script_id,
            debug = args.debug
        ): await main(target_dates = target_dates, interval = args.interval)

        # Disconnect from the database:
        disconnected = await sql_writer.disconnect()


    asyncio.run(runner())