Squashed 'utils_v2/' content from commit af73d53e
git-subtree-dir: utils_v2 git-subtree-split: af73d53e43729f79a736602775d61ef5b1b0d9cf
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Monday, 16th Dec., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
This file aims to fetch the current details about the constituents of various indices. This gives you not only
|
||||
the constituent stocks of the selected index, but also that stock's current activity in the market.
|
||||
https://www.nseindia.com/market-data/live-equity-market?symbol=NIFTY%2050
|
||||
|
||||
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
|
||||
more professional data-sources be used when the product starts becoming mature.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# NSE-related utils:
|
||||
from utils_v2.nse.controllers.base import AsyncNSEBase
|
||||
from utils_v2.nse.models.api_call import NSEApiResponse
|
||||
|
||||
# To make REST-ful API calls:
|
||||
import httpx
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, List
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class NSEIndexConstituents(AsyncNSEBase):
|
||||
|
||||
# Broad Market Indices:
|
||||
INDEX_NIFTY_50 = "NIFTY 50"
|
||||
INDEX_NIFTY_NEXT_50 = "NIFTY NEXT 50"
|
||||
INDEX_NIFTY_MIDCAP_50 = "NIFTY MIDCAP 50"
|
||||
INDEX_NIFTY_MIDCAP_100 = "NIFTY MIDCAP 100"
|
||||
INDEX_NIFTY_MIDCAP_150 = "NIFTY MIDCAP 150"
|
||||
INDEX_NIFTY_SMALLCAP_50 = "NIFTY SMALLCAP 50"
|
||||
INDEX_NIFTY_SMALLCAP_100 = "NIFTY SMALLCAP 100"
|
||||
INDEX_NIFTY_SMALLCAP_250 = "NIFTY SMALLCAP 250"
|
||||
INDEX_NIFTY_MIDSMALLCAP_400 = "NIFTY MIDSMALLCAP 400"
|
||||
INDEX_NIFTY_100 = "NIFTY 100"
|
||||
INDEX_NIFTY_200 = "NIFTY 200"
|
||||
INDEX_NIFTY_500_MULTICAP_50_25_25 = "NIFTY500 MULTICAP 50:25:25"
|
||||
INDEX_NIFTY_LARGEMIDCAP_250 = "NIFTY LARGEMIDCAP 250"
|
||||
INDEX_NIFTY_MIDCAP_SELECT = "NIFTY MIDCAP SELECT"
|
||||
INDEX_NIFTY_TOTAL_MARKET = "NIFTY TOTAL MARKET"
|
||||
INDEX_NIFTY_MICROCAP_250 = "NIFTY MICROCAP 250"
|
||||
INDEX_NIFTY_500 = "NIFTY 500"
|
||||
INDEX_NIFTY_500_LARGEMIDSMALL_EQUAL_CAP_WEIGHTED = "NIFTY500 LARGEMIDSMALL EQUAL-CAP WEIGHTED"
|
||||
|
||||
# Sectoral Indices:
|
||||
INDEX_NIFTY_AUTO = "NIFTY AUTO"
|
||||
INDEX_NIFTY_BANK = "NIFTY BANK"
|
||||
INDEX_NIFTY_ENERGY = "NIFTY ENERGY"
|
||||
INDEX_NIFTY_FINANCIAL_SERVICES = "NIFTY FINANCIAL SERVICES"
|
||||
INDEX_NIFTY_FINANCIAL_SERVICES_25_50 = "NIFTY FINANCIAL SERVICES 25/50"
|
||||
INDEX_NIFTY_FMCG = "NIFTY FMCG"
|
||||
INDEX_NIFTY_IT = "NIFTY IT"
|
||||
INDEX_NIFTY_MEDIA = "NIFTY MEDIA"
|
||||
INDEX_NIFTY_METAL = "NIFTY METAL"
|
||||
INDEX_NIFTY_PHARMA = "NIFTY PHARMA"
|
||||
INDEX_NIFTY_PSU_BANK = "NIFTY PSU BANK"
|
||||
INDEX_NIFTY_REALTY = "NIFTY REALTY"
|
||||
INDEX_NIFTY_PRIVATE_BANK = "NIFTY PRIVATE BANK"
|
||||
INDEX_NIFTY_HEALTHCARE_INDEX = "NIFTY HEALTHCARE INDEX"
|
||||
INDEX_NIFTY_CONSUMER_DURABLES = "NIFTY CONSUMER DURABLES"
|
||||
INDEX_NIFTY_OIL_GAS = "NIFTY OIL & GAS"
|
||||
INDEX_NIFTY_MIDSMALL_HEALTHCARE = "NIFTY MIDSMALL HEALTHCARE"
|
||||
INDEX_NIFTY_FINANCIAL_SERVICES_EX_BANK = "NIFTY FINANCIAL SERVICES EX-BANK"
|
||||
INDEX_NIFTY_MIDSMALL_FINANCIAL_SERVICES = "NIFTY MIDSMALL FINANCIAL SERVICES"
|
||||
INDEX_NIFTY_MIDSMALL_IT_TELECOM = "NIFTY MIDSMALL IT & TELECOM"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
http_client: httpx.AsyncClient,
|
||||
cookies_refresh_interval: int | float = 300,
|
||||
debug = True,
|
||||
debug_prefix = "NSE (IdxCons) | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
# Pass on the initialization to the parent:
|
||||
super().__init__(
|
||||
base_url = r"https://www.nseindia.com/market-data/live-equity-market",
|
||||
data_url = r"https://www.nseindia.com/api/equity-stockIndices",
|
||||
http_client = http_client,
|
||||
cookies_refresh_interval = cookies_refresh_interval,
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix,
|
||||
debug_only_errors = debug_only_errors
|
||||
)
|
||||
|
||||
async def get_data(
|
||||
self,
|
||||
index_name: str,
|
||||
return_raw: bool = False,
|
||||
) -> NSEApiResponse:
|
||||
|
||||
"""
|
||||
To get the data of the corporate event calendar.
|
||||
:param index_name: The value held in the 'indexName' field of the formatted output of the Index Master.
|
||||
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
|
||||
:return: The raw or formatted event calendar data in the 'data' field of the response model.
|
||||
"""
|
||||
|
||||
# Make the API call:
|
||||
api_response = await self.get(params = {"index": index_name})
|
||||
|
||||
# If the API call was successful:
|
||||
if api_response.httpCode in [200]:
|
||||
api_response.success = True
|
||||
if return_raw: api_response.data = await api_response.get_json()
|
||||
else:
|
||||
try: api_response.data = self.format_data(
|
||||
raw_json = await api_response.get_json(),
|
||||
index_name = index_name,
|
||||
timestamp = date_time.get_current_utc_date_time(as_string = True),
|
||||
raise_exception = True
|
||||
)
|
||||
except Exception as exception:
|
||||
api_response.exception = exception
|
||||
api_response.success = False
|
||||
|
||||
# Done here:
|
||||
return api_response
|
||||
|
||||
@staticmethod
|
||||
def format_data(
|
||||
raw_json: dict,
|
||||
index_name: str,
|
||||
timestamp: datetime.datetime = None,
|
||||
raise_exception: bool = False
|
||||
) -> List[dict] | None:
|
||||
|
||||
"""
|
||||
We format the data here to be able to retrieve it properly later.
|
||||
:param raw_json: The raw data as scraped from NSE.
|
||||
:param index_name: The value that you had used to fetch the raw data in the first place.
|
||||
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
|
||||
database, later.
|
||||
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
|
||||
suppressed internally.
|
||||
:return: The formatted data if successful, else None.
|
||||
"""
|
||||
|
||||
# Can't do anything if the chain itself is null:
|
||||
if raw_json is None: return raw_json
|
||||
|
||||
# Start by assuming failure:
|
||||
formatted_data = None
|
||||
|
||||
# Ensure that we've got a proper timestamp:
|
||||
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
try:
|
||||
|
||||
# Format the data:
|
||||
formatted_data = [
|
||||
{
|
||||
"scrapeTs": timestamp,
|
||||
"ts": date_time.to_timezone(
|
||||
date_time.parse_date_time(
|
||||
input_value = symbol["lastUpdateTime"],
|
||||
date_formats = ["%d-%b-%Y %H:%M:%S"],
|
||||
timezone = date_time.TIMEZONE_IST
|
||||
),
|
||||
timezone = date_time.TIMEZONE_UTC
|
||||
),
|
||||
"indexName": index_name,
|
||||
"symbol": symbol["symbol"],
|
||||
"name": symbol["meta"]["companyName"],
|
||||
"industry": symbol["meta"]["industry"],
|
||||
"isFNOSec": symbol["meta"]["isFNOSec"],
|
||||
"isSuspended": symbol["meta"]["isSuspended"],
|
||||
"isin": symbol["meta"]["isin"],
|
||||
"open": symbol["open"],
|
||||
"high": symbol["dayHigh"],
|
||||
"low": symbol["dayLow"],
|
||||
"close": symbol["lastPrice"],
|
||||
"totTradedVol": symbol["totalTradedVolume"],
|
||||
"totTradedVal": symbol["totalTradedValue"],
|
||||
"prevClose": symbol["previousClose"],
|
||||
"chg": symbol["change"],
|
||||
"pChg": symbol["pChange"],
|
||||
"yearHigh": symbol["yearHigh"],
|
||||
"yearLow": symbol["yearLow"],
|
||||
"pChg30d": symbol["perChange30d"],
|
||||
"pChg365d": symbol["perChange365d"],
|
||||
"ffmc": symbol["ffmc"]
|
||||
} for symbol in raw_json["data"] if symbol["priority"] == 0
|
||||
]
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
formatted_data = None
|
||||
if raise_exception: raise
|
||||
|
||||
# Done here:
|
||||
return formatted_data
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
async def main():
|
||||
|
||||
# Create an HTTP client:
|
||||
test_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 2.5 # ....... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# Create an instance of the scraper, and refresh its cookies:
|
||||
my_nse = NSEIndexConstituents(http_client = test_client)
|
||||
|
||||
# Get and show the data:
|
||||
api_response = await my_nse.get_data(
|
||||
index_name = NSEIndexConstituents.INDEX_NIFTY_50,
|
||||
return_raw = False
|
||||
)
|
||||
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
|
||||
if api_response.success: print("INDEX CONSTITUENTS:", json.to_string(api_response.data, default = str))
|
||||
|
||||
asyncio.run(main())
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,240 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Thursday, 28th Nov., 2024
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
The "Index Master" contains information about just the indices, and not the component symbols of those indices.
|
||||
This script provides a way to get the data that is available on the screen on the following URL:
|
||||
https://www.nseindia.com/market-data/live-market-indices
|
||||
|
||||
NOTE: This method involves web scraping. It is good for proof-of-concept development, but it is recommended that
|
||||
more professional data-sources be used when the product starts becoming mature.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.date_time import date_time
|
||||
|
||||
# NSE-related utils:
|
||||
from utils_v2.nse.controllers.base import AsyncNSEBase
|
||||
from utils_v2.nse.models.api_call import NSEApiResponse
|
||||
|
||||
# To make REST-ful API calls:
|
||||
import httpx
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import Any, List
|
||||
|
||||
# For asynchronous activities:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# --- Nothing Yet
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
class NSEIndexMaster(AsyncNSEBase):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
http_client: httpx.AsyncClient,
|
||||
cookies_refresh_interval: int | float = 300,
|
||||
debug = True,
|
||||
debug_prefix = "NSE (IdxMstr) | ",
|
||||
debug_only_errors = True
|
||||
):
|
||||
|
||||
# Pass on the initialization to the parent:
|
||||
super().__init__(
|
||||
base_url = r"https://www.nseindia.com/market-data/live-market-indices",
|
||||
data_url = r"https://www.nseindia.com/api/allIndices",
|
||||
http_client = http_client,
|
||||
cookies_refresh_interval = cookies_refresh_interval,
|
||||
debug = debug,
|
||||
debug_prefix = debug_prefix,
|
||||
debug_only_errors = debug_only_errors
|
||||
)
|
||||
|
||||
async def get_data(
|
||||
self,
|
||||
return_raw: bool = False,
|
||||
) -> NSEApiResponse:
|
||||
|
||||
"""
|
||||
To get the data of the corporate event calendar.
|
||||
:param return_raw: Whether you want the raw JSON from NSE or you want it formatted.
|
||||
:return: The raw or formatted event calendar data in the 'data' field of the response model.
|
||||
"""
|
||||
|
||||
# Make the API call:
|
||||
api_response = await self.get()
|
||||
|
||||
# If the API call was successful:
|
||||
if api_response.httpCode in [200]:
|
||||
api_response.success = True
|
||||
if return_raw: api_response.data = await api_response.get_json()
|
||||
else:
|
||||
try: api_response.data = self.format_data(
|
||||
raw_json = await api_response.get_json(),
|
||||
timestamp = date_time.get_current_utc_date_time(as_string = True),
|
||||
raise_exception = True
|
||||
)
|
||||
except Exception as exception:
|
||||
api_response.exception = exception
|
||||
api_response.success = False
|
||||
|
||||
# Done here:
|
||||
return api_response
|
||||
|
||||
@staticmethod
|
||||
def format_data(
|
||||
raw_json: dict,
|
||||
timestamp: datetime.datetime = None,
|
||||
raise_exception: bool = False
|
||||
) -> List[dict] | None:
|
||||
|
||||
"""
|
||||
We format the data here to be able to retrieve it properly later.
|
||||
:param raw_json: The raw data as scraped from NSE.
|
||||
:param timestamp: The timestamp at which the data was scraped. This shall be useful for data retrieval from the
|
||||
database, later.
|
||||
:param raise_exception: If set to True, any exception will be propagated. If set to False, any exception will be
|
||||
suppressed internally.
|
||||
:return: The formatted data if successful, else None.
|
||||
"""
|
||||
|
||||
# Can't do anything if the chain itself is null:
|
||||
if raw_json is None: return raw_json
|
||||
|
||||
# Start by assuming failure:
|
||||
formatted_data = None
|
||||
|
||||
# Ensure that we've got a proper timestamp:
|
||||
if timestamp is None: timestamp = date_time.get_current_utc_date_time(as_string = False)
|
||||
|
||||
try:
|
||||
|
||||
# Format the data:
|
||||
formatted_data = [
|
||||
{
|
||||
"scrapeTs": timestamp,
|
||||
"indexType": idx["key"],
|
||||
"indexName": idx["index"],
|
||||
"indexSymbol": idx["indexSymbol"],
|
||||
"open": idx["open"],
|
||||
"high": idx["high"],
|
||||
"low": idx["low"],
|
||||
"close": idx["last"],
|
||||
"prevClose": idx["previousClose"],
|
||||
"pChg": idx["percentChange"],
|
||||
"yearHigh": idx["yearHigh"],
|
||||
"yearLow": idx["yearLow"],
|
||||
"advances": idx.get("advances"),
|
||||
"declines": idx.get("declines"),
|
||||
"unchanged": idx.get("unchanged"),
|
||||
"pChg30d": idx["perChange30d"],
|
||||
"pChg365d": idx["perChange365d"]
|
||||
} for idx in raw_json["data"]
|
||||
]
|
||||
|
||||
# If something goes wrong:
|
||||
except Exception as exception:
|
||||
formatted_data = None
|
||||
if raise_exception: raise
|
||||
|
||||
# Done here:
|
||||
return formatted_data
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
async def main():
|
||||
|
||||
# Create an HTTP client:
|
||||
test_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 2.5 # ....... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# Create an instance of the scraper, and refresh its cookies:
|
||||
my_nse = NSEIndexMaster(http_client = test_client)
|
||||
|
||||
# Get and show the data:
|
||||
api_response = await my_nse.get_data(return_raw = False)
|
||||
print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")
|
||||
if api_response.success: print("INDEX MASTER:", json.to_string(api_response.data, default = str))
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user