Merge commit 'adb86c891b17171889c2c30380fe8d0b9054e449' as 'utils_v2'
This commit is contained in:
@@ -0,0 +1,394 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Monday, 16th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
This file aims to fetch the current details about the constituents of various indices. This gives you not only
|
||||
the constituent stocks of the selected index, but also that stock's current activity in the market.
|
||||
https://www.nseindia.com/market-data/live-equity-market?symbol=NIFTY%2050
|
||||
|
||||
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
|
||||
more professional data-sources be used when the product starts becoming mature.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# NSE-related utils:
|
||||
from utils_v2.nse.controllers.base import AsyncNSEBase
|
||||
from utils_v2.nse.models.api_call import NSEApiResponse
|
||||
|
||||
# To make REST-ful API calls:
|
||||
import httpx
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, List
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class NSEIndexConstituents(AsyncNSEBase):
|
||||
|
||||
# Broad Market Indices:
|
||||
INDEX_NIFTY_50 = "NIFTY 50"
|
||||
INDEX_NIFTY_NEXT_50 = "NIFTY NEXT 50"
|
||||
INDEX_NIFTY_MIDCAP_50 = "NIFTY MIDCAP 50"
|
||||
INDEX_NIFTY_MIDCAP_100 = "NIFTY MIDCAP 100"
|
||||
INDEX_NIFTY_MIDCAP_150 = "NIFTY MIDCAP 150"
|
||||
INDEX_NIFTY_SMALLCAP_50 = "NIFTY SMALLCAP 50"
|
||||
INDEX_NIFTY_SMALLCAP_100 = "NIFTY SMALLCAP 100"
|
||||
INDEX_NIFTY_SMALLCAP_250 = "NIFTY SMALLCAP 250"
|
||||
INDEX_NIFTY_MIDSMALLCAP_400 = "NIFTY MIDSMALLCAP 400"
|
||||
INDEX_NIFTY_100 = "NIFTY 100"
|
||||
INDEX_NIFTY_200 = "NIFTY 200"
|
||||
INDEX_NIFTY_500_MULTICAP_50_25_25 = "NIFTY500 MULTICAP 50:25:25"
|
||||
INDEX_NIFTY_LARGEMIDCAP_250 = "NIFTY LARGEMIDCAP 250"
|
||||
INDEX_NIFTY_MIDCAP_SELECT = "NIFTY MIDCAP SELECT"
|
||||
INDEX_NIFTY_TOTAL_MARKET = "NIFTY TOTAL MARKET"
|
||||
INDEX_NIFTY_MICROCAP_250 = "NIFTY MICROCAP 250"
|
||||
INDEX_NIFTY_500 = "NIFTY 500"
|
||||
INDEX_NIFTY_500_LARGEMIDSMALL_EQUAL_CAP_WEIGHTED = "NIFTY500 LARGEMIDSMALL EQUAL-CAP WEIGHTED"
|
||||
|
||||
# Sectoral Indices:
|
||||
INDEX_NIFTY_AUTO = "NIFTY AUTO"
|
||||
INDEX_NIFTY_BANK = "NIFTY BANK"
|
||||
INDEX_NIFTY_ENERGY = "NIFTY ENERGY"
|
||||
INDEX_NIFTY_FINANCIAL_SERVICES = "NIFTY FINANCIAL SERVICES"
|
||||
INDEX_NIFTY_FINANCIAL_SERVICES_25_50 = "NIFTY FINANCIAL SERVICES 25/50"
|
||||
INDEX_NIFTY_FMCG = "NIFTY FMCG"
|
||||
INDEX_NIFTY_IT = "NIFTY IT"
|
||||
INDEX_NIFTY_MEDIA = "NIFTY MEDIA"
|
||||
INDEX_NIFTY_METAL = "NIFTY METAL"
|
||||
INDEX_NIFTY_PHARMA = "NIFTY PHARMA"
|
||||
INDEX_NIFTY_PSU_BANK = "NIFTY PSU BANK"
|
||||
INDEX_NIFTY_REALTY = "NIFTY REALTY"
|
||||
INDEX_NIFTY_PRIVATE_BANK = "NIFTY PRIVATE BANK"
|
||||
INDEX_NIFTY_HEALTHCARE_INDEX = "NIFTY HEALTHCARE INDEX"
|
||||
INDEX_NIFTY_CONSUMER_DURABLES = "NIFTY CONSUMER DURABLES"
|
||||
INDEX_NIFTY_OIL_GAS = "NIFTY OIL & GAS"
|
||||
INDEX_NIFTY_MIDSMALL_HEALTHCARE = "NIFTY MIDSMALL HEALTHCARE"
|
||||
INDEX_NIFTY_FINANCIAL_SERVICES_EX_BANK = "NIFTY FINANCIAL SERVICES EX-BANK"
|
||||
INDEX_NIFTY_MIDSMALL_FINANCIAL_SERVICES = "NIFTY MIDSMALL FINANCIAL SERVICES"
|
||||
INDEX_NIFTY_MIDSMALL_IT_TELECOM = "NIFTY MIDSMALL IT & TELECOM"
|
||||
|
||||
# Thematic:
|
||||
INDEX_NIFTY_COMMODITIES = "NIFTY COMMODITIES"
|
||||
INDEX_NIFTY_INDIA_CONSUMPTION = "NIFTY INDIA CONSUMPTION"
|
||||
INDEX_NIFTY_CPSE = "NIFTY CPSE"
|
||||
INDEX_NIFTY_INFRASTRUCTURE = "NIFTY INFRASTRUCTURE"
|
||||
INDEX_NIFTY_MNC = "NIFTY MNC"
|
||||
INDEX_NIFTY_GROWTH_SECTORS_15 = "NIFTY GROWTH SECTORS 15"
|
||||
INDEX_NIFTY_PSE = "NIFTY PSE"
|
||||
INDEX_NIFTY_SERVICES_SECTOR = "NIFTY SERVICES SECTOR"
|
||||
INDEX_NIFTY_100_LIQUID_15 = "NIFTY100 LIQUID 15"
|
||||
INDEX_NIFTY_MIDCAP_LIQUID_15 = "NIFTY MIDCAP LIQUID 15"
|
||||
INDEX_NIFTY_INDIA_DIGITAL = "NIFTY INDIA DIGITAL"
|
||||
INDEX_NIFTY_100_ESG = "NIFTY100 ESG"
|
||||
INDEX_NIFTY_INDIA_MANUFACTURING = "NIFTY INDIA MANUFACTURING"
|
||||
INDEX_NIFTY_INDIA_CORPORATE_GROUP_INDEX_TATA_GROUP_25_CAP = "NIFTY INDIA CORPORATE GROUP INDEX - TATA GROUP 25% CAP"
|
||||
INDEX_NIFTY_500_MULTICAP_INDIA_MANUFACTURING_50_30_20 = "NIFTY500 MULTICAP INDIA MANUFACTURING 50:30:20"
|
||||
INDEX_NIFTY_500_MULTICAP_INFRASTRUCTURE_50_30_20 = "NIFTY500 MULTICAP INFRASTRUCTURE 50:30:20"
|
||||
INDEX_NIFTY_INDIA_DEFENCE = "NIFTY INDIA DEFENCE"
|
||||
INDEX_NIFTY_INDIA_TOURISM = "NIFTY INDIA TOURISM"
|
||||
INDEX_NIFTY_CAPITAL_MARKETS = "NIFTY CAPITAL MARKETS"
|
||||
INDEX_NIFTY_EV_NEW_AGE_AUTOMOTIVE = "NIFTY EV & NEW AGE AUTOMOTIVE"
|
||||
INDEX_NIFTY_INDIA_NEW_AGE_CONSUMPTION = "NIFTY INDIA NEW AGE CONSUMPTION"
|
||||
INDEX_NIFTY_INDIA_SELECT_5_CORPORATE_GROUPS_MAATR = "NIFTY INDIA SELECT 5 CORPORATE GROUPS (MAATR)"
|
||||
INDEX_NIFTY_MOBILITY = "NIFTY MOBILITY"
|
||||
INDEX_NIFTY_100_ENHANCED_ESG = "NIFTY100 ENHANCED ESG"
|
||||
INDEX_NIFTY_CORE_HOUSING = "NIFTY CORE HOUSING"
|
||||
INDEX_NIFTY_HOUSING = "NIFTY HOUSING"
|
||||
INDEX_NIFTY_IPO = "NIFTY IPO"
|
||||
INDEX_NIFTY_MIDSMALL_INDIA_CONSUMPTION = "NIFTY MIDSMALL INDIA CONSUMPTION"
|
||||
INDEX_NIFTY_NON_CYCLICAL_CONSUMER = "NIFTY NON-CYCLICAL CONSUMER"
|
||||
INDEX_NIFTY_RURAL = "NIFTY RURAL"
|
||||
INDEX_NIFTY_SHARIAH_25 = "NIFTY SHARIAH 25"
|
||||
INDEX_NIFTY_TRANSPORTATION_LOGISTICS = "NIFTY TRANSPORTATION & LOGISTICS"
|
||||
INDEX_NIFTY_50_SHARIAH = "NIFTY50 SHARIAH"
|
||||
INDEX_NIFTY_500_SHARIAH = "NIFTY500 SHARIAH"
|
||||
|
||||
# Strategy Indices:
|
||||
INDEX_NIFTY_DIVIDEND_OPPORTUNITIES_50 = "NIFTY DIVIDEND OPPORTUNITIES 50"
|
||||
INDEX_NIFTY_50_VALUE_20 = "NIFTY50 VALUE 20"
|
||||
INDEX_NIFTY_100_QUALITY_30 = "NIFTY100 QUALITY 30"
|
||||
INDEX_NIFTY_50_EQUAL_WEIGHT = "NIFTY50 EQUAL WEIGHT"
|
||||
INDEX_NIFTY_100_EQUAL_WEIGHT = "NIFTY100 EQUAL WEIGHT"
|
||||
INDEX_NIFTY_100_LOW_VOLATILITY_30 = "NIFTY100 LOW VOLATILITY 30"
|
||||
INDEX_NIFTY_ALPHA_50 = "NIFTY ALPHA 50"
|
||||
INDEX_NIFTY_200_QUALITY_30 = "NIFTY200 QUALITY 30"
|
||||
INDEX_NIFTY_ALPHA_LOW_VOLATILITY_30 = "NIFTY ALPHA LOW-VOLATILITY 30"
|
||||
INDEX_NIFTY_200_MOMENTUM_30 = "NIFTY200 MOMENTUM 30"
|
||||
INDEX_NIFTY_MIDCAP_150_QUALITY_50 = "NIFTY MIDCAP150 QUALITY 50"
|
||||
INDEX_NIFTY_200_ALPHA_30 = "NIFTY200 ALPHA 30"
|
||||
INDEX_NIFTY_MIDCAP_150_MOMENTUM_50 = "NIFTY MIDCAP150 MOMENTUM 50"
|
||||
INDEX_NIFTY_500_MOMENTUM_50 = "NIFTY500 MOMENTUM 50"
|
||||
INDEX_NIFTY_MIDSMALLCAP_400_MOMENTUM_QUALITY_100 = "NIFTY MIDSMALLCAP400 MOMENTUM QUALITY 100"
|
||||
INDEX_NIFTY_SMALLCAP_250_MOMENTUM_QUALITY_100 = "NIFTY SMALLCAP250 MOMENTUM QUALITY 100"
|
||||
INDEX_NIFTY_TOP_10_EQUAL_WEIGHT = "NIFTY TOP 10 EQUAL WEIGHT"
|
||||
INDEX_NIFTY_ALPHA_QUALITY_LOW_VOLATILITY_30 = "NIFTY ALPHA QUALITY LOW-VOLATILITY 30"
|
||||
INDEX_NIFTY_ALPHA_QUALITY_VALUE_LOW_VOLATILITY_30 = "NIFTY ALPHA QUALITY VALUE LOW-VOLATILITY 30"
|
||||
INDEX_NIFTY_HIGH_BETA_50 = "NIFTY HIGH BETA 50"
|
||||
INDEX_NIFTY_LOW_VOLATILITY_50 = "NIFTY LOW VOLATILITY 50"
|
||||
INDEX_NIFTY_QUALITY_LOW_VOLATILITY_30 = "NIFTY QUALITY LOW-VOLATILITY 30"
|
||||
INDEX_NIFTY_SMALLCAP_250_QUALITY_50 = "NIFTY SMALLCAP250 QUALITY 50"
|
||||
INDEX_NIFTY_TOP_15_EQUAL_WEIGHT = "NIFTY TOP 15 EQUAL WEIGHT"
|
||||
INDEX_NIFTY_100_ALPHA_30 = "NIFTY100 ALPHA 30"
|
||||
INDEX_NIFTY_200_VALUE_30 = "NIFTY200 VALUE 30"
|
||||
INDEX_NIFTY_500_EQUAL_WEIGHT = "NIFTY500 EQUAL WEIGHT"
|
||||
INDEX_NIFTY_500_MULTICAP_MOMENTUM_QUALITY_50 = "NIFTY500 MULTICAP MOMENTUM QUALITY 50"
|
||||
INDEX_NIFTY_500_VALUE_50 = "NIFTY500 VALUE 50"
|
||||
INDEX_NIFTY_TOP_20_EQUAL_WEIGHT = "NIFTY TOP 20 EQUAL WEIGHT"
|
||||
|
||||
# Others:
|
||||
INDEX_SECURITIES_IN_FNO = "SECURITIES IN F&O"
|
||||
INDEX_PERMITTED_TO_TRADE = "PERMITTED TO TRADE"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
http_client: httpx.AsyncClient,
|
||||
cookies_refresh_interval: int | float = 300,
|
||||
debug = True,
|
||||
debug_prefix = "NSE (IdxCons) | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
# Pass on the initialization to the parent:
|
||||
super().__init__(
|
||||
base_url = r"https://www.nseindia.com/market-data/live-equity-market",
|
||||
data_url = r"https://www.nseindia.com/api/equity-stockIndices",
|
||||
http_client = http_client,
|
||||
cookies_refresh_interval = cookies_refresh_interval,
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix,
|
||||
debug_only_errors = debug_only_errors
|
||||
)
|
||||
|
||||
async def get_data(
|
||||
self,
|
||||
index_name: str,
|
||||
return_raw: bool = False,
|
||||
refresh_cookies: bool = True,
|
||||
force_refresh_cookies: bool = False,
|
||||
retry_count: int = 1,
|
||||
backoff_seconds: int | float = 0.5,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> NSEApiResponse:
|
||||
|
||||
"""
|
||||
To get the data of the corporate event calendar.
|
||||
:param index_name: The value held in the 'indexName' field of the formatted output of the Index Master.
|
||||
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
|
||||
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
|
||||
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
|
||||
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
|
||||
successful refresh.
|
||||
:param retry_count: The no. of times to try to get the data from the API.
|
||||
:param backoff_seconds: The delay between unsuccessful API calls.
|
||||
:param backoff_multiplier: The multiplier to add to the delay to change delay.
|
||||
:return: The raw or formatted event calendar data in the 'data' field of the response model.
|
||||
"""
|
||||
|
||||
# Make the API call:
|
||||
api_response = await self.get(
|
||||
params = {"index": index_name},
|
||||
refresh_cookies = refresh_cookies,
|
||||
force_refresh_cookies = force_refresh_cookies,
|
||||
retry_count = retry_count,
|
||||
backoff_seconds = backoff_seconds,
|
||||
backoff_multiplier = backoff_multiplier
|
||||
)
|
||||
|
||||
# If the API call was successful:
|
||||
if api_response.httpCode in [200]:
|
||||
api_response.success = True
|
||||
if return_raw: api_response.data = await api_response.get_json()
|
||||
else:
|
||||
try: api_response.data = self.format_data(
|
||||
raw_json = await api_response.get_json(),
|
||||
index_name = index_name,
|
||||
timestamp = date_time.get_current_utc_date_time(as_string = False),
|
||||
raise_exception = True
|
||||
)
|
||||
except Exception as exception:
|
||||
api_response.exception = exception
|
||||
api_response.success = False
|
||||
|
||||
# Done here:
|
||||
return api_response
|
||||
|
||||
@staticmethod
|
||||
def format_data(
|
||||
raw_json: dict,
|
||||
index_name: str,
|
||||
timestamp: datetime.datetime = None,
|
||||
raise_exception: bool = False
|
||||
) -> List[dict] | None:
|
||||
|
||||
"""
|
||||
We format the data here to be able to retrieve it properly later.
|
||||
:param raw_json: The raw data as scraped from NSE.
|
||||
:param index_name: The value that you had used to fetch the raw data in the first place.
|
||||
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
|
||||
database, later.
|
||||
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
|
||||
suppressed internally.
|
||||
:return: The formatted data if successful, else None.
|
||||
"""
|
||||
|
||||
# Can't do anything if the chain itself is null:
|
||||
if raw_json is None: return raw_json
|
||||
|
||||
# Start by assuming failure:
|
||||
formatted_data = None
|
||||
|
||||
# Ensure that we've got a proper timestamp:
|
||||
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
try:
|
||||
|
||||
# Format the data:
|
||||
formatted_data = [
|
||||
{
|
||||
"scrapeTs": timestamp,
|
||||
"ts": date_time.to_timezone(
|
||||
date_time.parse_date_time(
|
||||
input_value = symbol["lastUpdateTime"],
|
||||
date_formats = ["%d-%b-%Y %H:%M:%S"],
|
||||
timezone = date_time.TIMEZONE_IST
|
||||
),
|
||||
timezone = date_time.TIMEZONE_UTC
|
||||
),
|
||||
"indexName": index_name,
|
||||
"symbol": symbol["symbol"],
|
||||
"name": symbol["meta"]["companyName"],
|
||||
"industry": symbol["meta"]["industry"],
|
||||
"isFNOSec": symbol["meta"]["isFNOSec"],
|
||||
"isSuspended": symbol["meta"]["isSuspended"],
|
||||
"isin": symbol["meta"]["isin"],
|
||||
"open": symbol["open"],
|
||||
"high": symbol["dayHigh"],
|
||||
"low": symbol["dayLow"],
|
||||
"close": symbol["lastPrice"],
|
||||
"totTradedVol": symbol["totalTradedVolume"],
|
||||
"totTradedVal": symbol["totalTradedValue"],
|
||||
"prevClose": symbol["previousClose"],
|
||||
"chg": symbol["change"],
|
||||
"pChg": symbol["pChange"],
|
||||
"yearHigh": symbol["yearHigh"],
|
||||
"yearLow": symbol["yearLow"],
|
||||
"pChg30d": symbol["perChange30d"],
|
||||
"pChg365d": symbol["perChange365d"],
|
||||
"ffmc": symbol["ffmc"]
|
||||
} for symbol in raw_json["data"] if symbol["priority"] == 0
|
||||
]
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
formatted_data = None
|
||||
if raise_exception: raise
|
||||
|
||||
# Done here:
|
||||
return formatted_data
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
async def main():
|
||||
|
||||
# Create an HTTP client:
|
||||
test_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 2.5 # ....... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# Create an instance of the scraper, and refresh its cookies:
|
||||
my_nse = NSEIndexConstituents(http_client = test_client)
|
||||
|
||||
# Get and show the data:
|
||||
api_response = await my_nse.get_data(
|
||||
index_name = NSEIndexConstituents.INDEX_NIFTY_50,
|
||||
return_raw = False
|
||||
)
|
||||
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
|
||||
if api_response.success: print("INDEX CONSTITUENTS:", json.to_string(api_response.data, default = str))
|
||||
|
||||
asyncio.run(main())
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+1252
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,265 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Thursday, 28th Nov., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
The "Index Master" contains information about just the indices, and not the component symbols of those indices.
|
||||
This script provides a way to get the data that is available on the screen on the following URL:
|
||||
https://www.nseindia.com/market-data/live-market-indices
|
||||
|
||||
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
|
||||
more professional data-sources be used when the product starts becoming mature.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# NSE-related utils:
|
||||
from utils_v2.nse.controllers.base import AsyncNSEBase
|
||||
from utils_v2.nse.models.api_call import NSEApiResponse
|
||||
|
||||
# To make REST-ful API calls:
|
||||
import httpx
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, List
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class NSEIndexMaster(AsyncNSEBase):
|
||||
|
||||
# Index Types:
|
||||
INDEX_TYPE_BROAD_MARKET = "BROAD MARKET INDICES"
|
||||
INDEX_TYPE_SECTORAL = "SECTORAL INDICES"
|
||||
INDEX_TYPE_STRATEGY = "STRATEGY INDICES"
|
||||
INDEX_TYPE_THEMATIC = "THEMATIC INDICES"
|
||||
INDEX_TYPE_FIXED_INCOME = "FIXED INCOME INDICES"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
http_client: httpx.AsyncClient,
|
||||
cookies_refresh_interval: int | float = 300,
|
||||
debug = True,
|
||||
debug_prefix = "NSE (IdxMstr) | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
# Pass on the initialization to the parent:
|
||||
super().__init__(
|
||||
base_url = r"https://www.nseindia.com/market-data/live-market-indices",
|
||||
data_url = r"https://www.nseindia.com/api/allIndices",
|
||||
http_client = http_client,
|
||||
cookies_refresh_interval = cookies_refresh_interval,
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix,
|
||||
debug_only_errors = debug_only_errors
|
||||
)
|
||||
|
||||
async def get_data(
|
||||
self,
|
||||
return_raw: bool = False,
|
||||
refresh_cookies: bool = True,
|
||||
force_refresh_cookies: bool = False,
|
||||
retry_count: int = 1,
|
||||
backoff_seconds: int | float = 0.5,
|
||||
backoff_multiplier: float = 1.1
|
||||
) -> NSEApiResponse:
|
||||
|
||||
"""
|
||||
To get the data of the corporate event calendar.
|
||||
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
|
||||
:param refresh_cookies: Whether, or not, you would like to refresh the cookies.
|
||||
:param force_refresh_cookies: If set to True, cookies will be refreshed even if not timed out. If set to False,
|
||||
cookies will be refreshed only when the interval passed to the constructor has elapsed since the last
|
||||
successful refresh.
|
||||
:param retry_count: The no. of times to try to get the data from the API.
|
||||
:param backoff_seconds: The delay between unsuccessful API calls.
|
||||
:param backoff_multiplier: The multiplier to add to the delay to change delay.
|
||||
:return: The raw or formatted event calendar data in the 'data' field of the response model.
|
||||
"""
|
||||
|
||||
# Make the API call:
|
||||
api_response = await self.get(
|
||||
refresh_cookies = refresh_cookies,
|
||||
force_refresh_cookies = force_refresh_cookies,
|
||||
retry_count = retry_count,
|
||||
backoff_seconds = backoff_seconds,
|
||||
backoff_multiplier = backoff_multiplier
|
||||
)
|
||||
|
||||
# If the API call was successful:
|
||||
if api_response.httpCode in [200]:
|
||||
api_response.success = True
|
||||
if return_raw: api_response.data = await api_response.get_json()
|
||||
else:
|
||||
try: api_response.data = self.format_data(
|
||||
raw_json = await api_response.get_json(),
|
||||
timestamp = date_time.get_current_utc_date_time(as_string = False),
|
||||
raise_exception = True
|
||||
)
|
||||
except Exception as exception:
|
||||
api_response.exception = exception
|
||||
api_response.success = False
|
||||
|
||||
# Done here:
|
||||
return api_response
|
||||
|
||||
@staticmethod
|
||||
def format_data(
|
||||
raw_json: dict,
|
||||
timestamp: datetime.datetime = None,
|
||||
raise_exception: bool = False
|
||||
) -> List[dict] | None:
|
||||
|
||||
"""
|
||||
We format the data here to be able to retrieve it properly later.
|
||||
:param raw_json: The raw data as scraped from NSE.
|
||||
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
|
||||
database, later.
|
||||
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
|
||||
suppressed internally.
|
||||
:return: The formatted data if successful, else None.
|
||||
"""
|
||||
|
||||
# Can't do anything if the chain itself is null:
|
||||
if raw_json is None: return raw_json
|
||||
|
||||
# Start by assuming failure:
|
||||
formatted_data = None
|
||||
|
||||
# Ensure that we've got a proper timestamp:
|
||||
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
try:
|
||||
|
||||
# Format the data:
|
||||
formatted_data = [
|
||||
{
|
||||
"scrapeTs": timestamp,
|
||||
"indexType": idx["key"],
|
||||
"indexName": idx["index"],
|
||||
"indexSymbol": idx["indexSymbol"],
|
||||
"open": idx["open"],
|
||||
"high": idx["high"],
|
||||
"low": idx["low"],
|
||||
"close": idx["last"],
|
||||
"prevClose": idx["previousClose"],
|
||||
"pChg": idx["percentChange"],
|
||||
"yearHigh": idx["yearHigh"],
|
||||
"yearLow": idx["yearLow"],
|
||||
"advances": idx.get("advances"),
|
||||
"declines": idx.get("declines"),
|
||||
"unchanged": idx.get("unchanged"),
|
||||
"pChg30d": idx["perChange30d"],
|
||||
"pChg365d": idx["perChange365d"]
|
||||
} for idx in raw_json["data"]
|
||||
]
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
formatted_data = None
|
||||
if raise_exception: raise
|
||||
|
||||
# Done here:
|
||||
return formatted_data
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
async def main():
|
||||
|
||||
# Create an HTTP client:
|
||||
test_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 2.5 # ....... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# Create an instance of the scraper, and refresh its cookies:
|
||||
my_nse = NSEIndexMaster(http_client = test_client)
|
||||
|
||||
# Get and show the data:
|
||||
api_response = await my_nse.get_data(return_raw = False)
|
||||
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
|
||||
if api_response.success: print("INDEX MASTER:", json.to_string(api_response.data, default = str))
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user