Merge commit '3a64ea7efec7f8dd3e7956411b2ef35a9ebb720b' as 'utils_v2'
This commit is contained in:
@@ -0,0 +1,325 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Monday, 6th Jan., 2025.
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To provide a way to fetch the daily BhavCopy from NSE.
|
||||
|
||||
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
|
||||
more professional data-sources be used when the product starts becoming mature.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# NSE-related utils:
|
||||
from utils_v2.nse.controllers.base import AsyncNSEBase
|
||||
from utils_v2.nse.models.api_call import NSEApiResponse
|
||||
|
||||
# To make REST-ful API calls:
|
||||
import httpx
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, List
|
||||
|
||||
# To work with tabulated dataL
|
||||
import pandas as pd
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class NSEFNOBhavCopy(AsyncNSEBase):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
http_client: httpx.AsyncClient,
|
||||
cookies_refresh_interval: int | float = 300,
|
||||
debug = True,
|
||||
debug_prefix = "NSE (FNO BhavCopy) | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
# Pass on the initialization to the parent:
|
||||
super().__init__(
|
||||
base_url = r"https://www.nseindia.com/all-reports",
|
||||
data_url = r"https://nsearchives.nseindia.com/content/trdops/FNO_BC02012025.DAT",
|
||||
http_client = http_client,
|
||||
cookies_refresh_interval = cookies_refresh_interval,
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix,
|
||||
debug_only_errors = debug_only_errors
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def parse_float(value):
|
||||
try: return float(value)
|
||||
except ValueError: return None
|
||||
|
||||
@staticmethod
|
||||
def parse_int(value):
|
||||
try: return int(value)
|
||||
except ValueError: return None
|
||||
|
||||
async def get_data(
|
||||
self,
|
||||
target_date: datetime.datetime,
|
||||
return_raw: bool = False,
|
||||
refresh_cookies: bool = True,
|
||||
force_refresh_cookies: bool = False,
|
||||
retry_count: int = 1,
|
||||
backoff_seconds: int | float = 0.5,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> NSEApiResponse:
|
||||
|
||||
"""
|
||||
To get the data of the corporate event calendar.
|
||||
:param target_date: The starting date (inclusive) from which the events must be fetched.
|
||||
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
|
||||
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
|
||||
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
|
||||
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
|
||||
successful refresh.
|
||||
:param retry_count: The no. of times to try to get the data from the API.
|
||||
:param backoff_seconds: The delay between unsuccessful API calls.
|
||||
:param backoff_multiplier: The multiplier to add to the delay to change delay.
|
||||
:return: The raw or formatted event calendar data in the 'data' field of the response model.
|
||||
"""
|
||||
|
||||
# Figure out the file name:
|
||||
date_time.to_timezone(target_date, date_time.TIMEZONE_IST)
|
||||
file_name = f"FNO_BC{target_date.strftime('%d%m%Y')}.DAT"
|
||||
|
||||
# Make the API call:
|
||||
api_response = await self.get(
|
||||
url = f"https://nsearchives.nseindia.com/content/trdops/{file_name}",
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
},
|
||||
refresh_cookies = refresh_cookies,
|
||||
force_refresh_cookies = force_refresh_cookies,
|
||||
retry_count = retry_count,
|
||||
backoff_seconds = backoff_seconds,
|
||||
backoff_multiplier = backoff_multiplier
|
||||
)
|
||||
|
||||
# If the API call was successful:
|
||||
if api_response.httpCode in [200]:
|
||||
api_response.success = True
|
||||
if return_raw:
|
||||
api_response.data = io.BytesIO(await api_response.get_content())
|
||||
api_response.data.seek(0)
|
||||
else:
|
||||
try: api_response.data = self.format_data(
|
||||
raw_data = await api_response.get_content(),
|
||||
timestamp = date_time.get_current_utc_date_time(as_string = False),
|
||||
raise_exception = True
|
||||
)
|
||||
except Exception as exception:
|
||||
api_response.exception = exception
|
||||
api_response.success = False
|
||||
|
||||
# Done here:
|
||||
return api_response
|
||||
|
||||
def format_data(
|
||||
self,
|
||||
raw_data: bytes | io.BytesIO,
|
||||
timestamp: datetime.datetime = None,
|
||||
raise_exception: bool = False
|
||||
) -> List[dict] | None:
|
||||
|
||||
"""
|
||||
We format the data here to be able to retrieve it properly later.
|
||||
:param raw_data: The raw data as scraped from NSE.
|
||||
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
|
||||
database, later.
|
||||
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
|
||||
suppressed internally.
|
||||
:return: The formatted data if successful, else None.
|
||||
"""
|
||||
|
||||
# Can't do anything if the chain itself is null:
|
||||
if raw_data is None: return raw_data
|
||||
|
||||
# Start by assuming failure:
|
||||
formatted_data = None
|
||||
|
||||
# Ensure that we've got a proper timestamp:
|
||||
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
try:
|
||||
|
||||
# Ensure that we have a BytesIO object,
|
||||
# and that we are pointing at the start:
|
||||
if isinstance(raw_data, bytes):
|
||||
raw_data = io.BytesIO(raw_data)
|
||||
raw_data.seek(0)
|
||||
|
||||
# Read the data into Pandas:
|
||||
bhav_df = pd.read_csv(
|
||||
raw_data,
|
||||
header = None,
|
||||
names = [
|
||||
"symbol", "segment", "underlying", "expiry", "strike",
|
||||
"type", "G?", "H?", "I?", "prevClose",
|
||||
"open", "high", "low", "close", "O?",
|
||||
"totVol", "totCash", "oi", "oiChg", "T?",
|
||||
"U?", "date", "W?", "X?", "Y?",
|
||||
"Z?"
|
||||
]
|
||||
)
|
||||
# Format the data:
|
||||
for column in ["symbol", "segment", "type", "underlying", "expiry", "date"]:
|
||||
bhav_df[column] = bhav_df[column].apply(lambda x: str(x).strip())
|
||||
for column in ["strike", "prevClose", "open", "high", "low", "close", "totCash"]:
|
||||
bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_float(x))
|
||||
for column in ["totVol", "oi", "oiChg"]:
|
||||
bhav_df[column] = bhav_df[column].apply(lambda x: self.parse_int(x))
|
||||
|
||||
# Handle segment, type and other normalizations:
|
||||
bhav_df["isIndex"] = bhav_df["segment"].apply(lambda x: True if x.endswith("IDX") else False)
|
||||
bhav_df["segment"] = bhav_df["segment"].apply(lambda x: x[:3])
|
||||
bhav_df["type"] = bhav_df.apply(lambda x: None if x["segment"] == "FUT" else x["type"], axis = 1)
|
||||
bhav_df["strike"] = bhav_df.apply(lambda x: None if x["segment"] == "FUT" else x["strike"], axis = 1)
|
||||
|
||||
# Give the expiry as a timestamp:
|
||||
bhav_df["expiryTs"] = bhav_df["expiry"].apply(lambda x: self.parse_datetime_string(
|
||||
dt_str = f"{int(x):0>8}",
|
||||
dt_formats = ["%d%m%Y"],
|
||||
input_tz = date_time.TIMEZONE_IST,
|
||||
output_tz = date_time.TIMEZONE_UTC
|
||||
))
|
||||
bhav_df["expiryTz"] = "Asia/Kolkata"
|
||||
|
||||
# Give the data recording date as a timestamp:
|
||||
bhav_df["ts"] = bhav_df["date"].apply(lambda x: self.parse_datetime_string(
|
||||
dt_str = x,
|
||||
dt_formats = ["%d%m%Y", "%d-%b-%Y", "%d/%b/%Y"],
|
||||
input_tz = date_time.TIMEZONE_IST,
|
||||
output_tz = date_time.TIMEZONE_UTC
|
||||
))
|
||||
bhav_df["tz"] = "Asia/Kolkata"
|
||||
bhav_df["scrapeTs"] = timestamp
|
||||
|
||||
# Keep what is needed, in an order that makes intuitive sense:
|
||||
bhav_df = bhav_df[[
|
||||
"symbol", "segment", "type", "underlying", "isIndex", "expiry", "expiryTs", "expiryTz", "strike",
|
||||
"prevClose", "open", "high", "low", "close", "totVol", "totCash", "oi", "oiChg",
|
||||
"date", "ts", "tz", "scrapeTs"
|
||||
]]
|
||||
|
||||
# Done here:
|
||||
formatted_data = bhav_df.to_dict(orient = "records")
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
formatted_data = None
|
||||
if raise_exception: raise
|
||||
|
||||
# Done here:
|
||||
return formatted_data
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
async def main():
|
||||
|
||||
# Create an HTTP client:
|
||||
test_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 2.5 # ....... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# Create an instance of the scraper, and refresh its cookies:
|
||||
my_nse = NSEFNOBhavCopy(http_client = test_client)
|
||||
|
||||
# Get and show the data:
|
||||
target_date = datetime.datetime.strptime("20250103", "%Y%m%d")
|
||||
api_response = await my_nse.get_data(
|
||||
target_date = target_date,
|
||||
return_raw = False
|
||||
)
|
||||
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
|
||||
if api_response.success: print("BHAV-COPY:", json.to_string(api_response.data[-10:], default = str))
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user