From e3248b874b5e62611053ce0b35f5637d59e9bd56 Mon Sep 17 00:00:00 2001 From: khushal Date: Sat, 4 Jan 2025 18:45:22 +0530 Subject: [PATCH] (20250104) Day-end push. --- utils_v2/nse/controllers/bhavcopy/__init__.py | 0 utils_v2/nse/controllers/bhavcopy/equities.py | 318 ++++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 utils_v2/nse/controllers/bhavcopy/__init__.py create mode 100644 utils_v2/nse/controllers/bhavcopy/equities.py diff --git a/utils_v2/nse/controllers/bhavcopy/__init__.py b/utils_v2/nse/controllers/bhavcopy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils_v2/nse/controllers/bhavcopy/equities.py b/utils_v2/nse/controllers/bhavcopy/equities.py new file mode 100644 index 0000000..e7920ec --- /dev/null +++ b/utils_v2/nse/controllers/bhavcopy/equities.py @@ -0,0 +1,318 @@ +""" + + AUTHOR: + + Khushal P Soonderji + + DATE: + + Saturday, 4th Jan., 2025. + + OBJECTIVE: + + To provide a way to fetch the daily BhavCopy from NSE. + + NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that + more professional data-sources be used when the product starts becoming mature. + + REFERENCES: + + N/A + + DOWNLOADS: + + N/A + +""" + + +# ***************************************************************************************************************** +# ***** **** +# *** IMPORT *** +# ***** **** +# ***************************************************************************************************************** + + +# To make sibling directories accessible for imports: +import sys +sys.path.append(".") +sys.path.append("..") + +# System-level activities: +import io + +# My utils: +from utils_v2.string import json +from utils_v2.date_time import date_time + +# NSE-related utils: +from utils_v2.nse.controllers.base import AsyncNSEBase +from utils_v2.nse.models.api_call import NSEApiResponse + +# To make REST-ful API calls: +import httpx + +# To work with date and time: +import datetime +import pytz + +# To work with datatypes: +from typing import Any, List + +# To work with tabulated dataL +import pandas as pd + +# For asynchronous activities: +import asyncio + + +# ***************************************************************************************************************** +# ***** **** +# *** MACROS / ONE-TIME INIT *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** VARIABLES *** +# ***** **** +# ***************************************************************************************************************** + + +# --- Nothing Yet + + +# ***************************************************************************************************************** +# ***** **** +# *** FUNCTIONS *** +# ***** **** +# ***************************************************************************************************************** + + +class NSEEquitiesBhavCopy(AsyncNSEBase): + + def __init__( + self, + http_client: httpx.AsyncClient, + cookies_refresh_interval: int | float = 300, + debug = True, + debug_prefix = "NSE (Eq. BhavCopy) | ", + debug_only_errors = True + ): + + # Pass on the initialization to the parent: + super().__init__( + base_url = r"https://www.nseindia.com/all-reports", + data_url = r"https://nsearchives.nseindia.com/products/content/sec_bhavdata_full_03012025.csv", + http_client = http_client, + cookies_refresh_interval = cookies_refresh_interval, + debug = debug, + debug_prefix = debug_prefix, + debug_only_errors = debug_only_errors + ) + + @staticmethod + def parse_float(value): + try: return float(value) + except ValueError: return None + + @staticmethod + def parse_int(value): + try: return int(value) + except ValueError: return None + + async def get_data( + self, + target_date: datetime.datetime, + return_raw: bool = False, + refresh_cookies: bool = True, + force_refresh_cookies: bool = False, + retry_count: int = 1, + backoff_seconds: int | float = 0.5, + backoff_multiplier: float = 1.1 + ) -> NSEApiResponse: + + """ + To get the data of the corporate event calendar. + :param target_date: The starting date (inclusive) from which the events must be fetched. + :param return_raw: Whether you want the raw JSON from NSE or you want it formatted. + :param refresh_cookies: Whether, or not, you would like to refresh the cookies. + :param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False, + cookies will be refreshed only when the interval passed to the constructor has elapsed since the last + successful refresh. + :param retry_count: The no. of times to try to get the data from the API. + :param backoff_seconds: The delay between unsuccessful API calls. + :param backoff_multiplier: The multiplier to add to the delay to change delay. + :return: The raw or formatted event calendar data in the 'data' field of the response model. + """ + + # Figure out the file name: + date_time.to_timezone(target_date, date_time.TIMEZONE_IST) + file_name = f"sec_bhavdata_full_{target_date.strftime('%d%m%Y')}.csv" + + # Make the API call: + api_response = await self.get( + url = f"https://nsearchives.nseindia.com/products/content/{file_name}", + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + }, + refresh_cookies = refresh_cookies, + force_refresh_cookies = force_refresh_cookies, + retry_count = retry_count, + backoff_seconds = backoff_seconds, + backoff_multiplier = backoff_multiplier + ) + + # If the API call was successful: + if api_response.httpCode in [200]: + api_response.success = True + if return_raw: + api_response.data = io.BytesIO(await api_response.get_content()) + api_response.data.seek(0) + else: + try: api_response.data = self.format_data( + raw_data = await api_response.get_content(), + timestamp = date_time.get_current_utc_date_time(as_string = True), + raise_exception = True + ) + except Exception as exception: + api_response.exception = exception + api_response.success = False + + # Done here: + return api_response + + def format_data( + self, + raw_data: bytes | io.BytesIO, + timestamp: datetime.datetime = None, + raise_exception: bool = False + ) -> List[dict] | None: + + """ + We format the data here to be able to retrieve it properly later. + :param raw_data: The raw data as scraped from NSE. + :param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the + database, later. + :param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be + suppressed internally. + :return: The formatted data if successful, else None. + """ + + # Can't do anything if the chain itself is null: + if raw_data is None: return raw_data + + # Start by assuming failure: + formatted_data = None + + # Ensure that we've got a proper timestamp: + if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False) + + try: + + # Ensure that we have a BytesIO object, + # and that we are pointing at the start: + if isinstance(raw_data, bytes): + raw_data = io.BytesIO(raw_data) + raw_data.seek(0) + + # Read the data into Pandas: + bhav_df = pd.read_csv(raw_data) + bhav_df.rename( + columns = {c: c.strip() for c in bhav_df.columns.to_list()}, + inplace = True + ) + bhav_df.rename( + columns = { + "SYMBOL": "symbol", + "SERIES": "segment", + "DATE1": "date", + "PREV_CLOSE": "prevClose", + "OPEN_PRICE": "open", + "HIGH_PRICE": "high", + "LOW_PRICE": "low", + "CLOSE_PRICE": "close", + "LAST_PRICE": "ltp", + "AVG_PRICE": "vwap", + "TTL_TRD_QNTY": "totVol", + "TURNOVER_LACS": "totCash", + "NO_OF_TRADES": "tradeCount", + "DELIV_QTY": "deliveryVol", + "DELIV_PER": "deliveryPct" + }, + inplace = True + ) + + # Format the data: + for column in ["symbol", "segment", "date"]: + bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip()) + for column in ["prevClose", "open", "high", "low", "close", "ltp", "vwap", "totCash", "deliveryPct"]: + bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x)) + for column in ["totVol", "deliveryVol"]: + bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x)) + bhav_df["totCash"] *= 1_00_000 + + # Give the date as a timestamp: + bhav_df["scrapeTs"] = timestamp + bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string( + dt_str = x, + dt_formats = ["%d-%b-%Y"], + input_tz = date_time.TIMEZONE_IST, + output_tz = date_time.TIMEZONE_UTC + )) + bhav_df["tz"] = "Asia/Kolkata" + + # Done here: + formatted_data = bhav_df.to_dict(orient = "records") + + # If something goes wrong: + except Exception as exception: + formatted_data = None + if raise_exception: raise + + # Done here: + return formatted_data + + +# ***************************************************************************************************************** +# ***** **** +# *** MAIN PROGRAM *** +# ***** **** +# ***************************************************************************************************************** + + +if __name__ == "__main__": + + async def main(): + + # Create an HTTP client: + test_client = httpx.AsyncClient( + limits = httpx.Limits( + max_connections = 100, # ............ Maximum number of connections allowed in the pool. + max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive. + ), + timeout = httpx.Timeout( + pool = 120.0, # .... Time to wait for a free connection from the pool. + connect = 2.5, # ... Time to wait for establishing a connection to the server. + write = 10.0, # .... Time to wait for sending data. + read = 2.5 # ....... Time to wait for receiving data. + ) + ) + + # Create an instance of the scraper, and refresh its cookies: + my_nse = NSEEquitiesBhavCopy(http_client = test_client) + + # Get and show the data: + target_date = datetime.datetime.strptime("20250103", "%Y%m%d") + api_response = await my_nse.get_data( + target_date = target_date, + return_raw = False + ) + print("SUMMARY:", api_response.to_markdown(), "\n---\n\n") + if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str)) + + asyncio.run(main())