Resetting utils subtree.
This commit is contained in:
@@ -0,0 +1,419 @@
|
||||
"""
|
||||
|
||||
AUTHOR:
|
||||
|
||||
Khushal P Soonderji
|
||||
|
||||
DATE:
|
||||
|
||||
Wednesday, 1st Jan., 2025.
|
||||
|
||||
OBJECTIVE:
|
||||
|
||||
To capture end-of-day market data and store it in the database.
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N/A
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
N/A
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** IMPORT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make sibling directories accessible for imports:
|
||||
import sys
|
||||
sys.path.append(".")
|
||||
sys.path.append("..")
|
||||
|
||||
# System-level activities:
|
||||
import io
|
||||
import os
|
||||
|
||||
# My utils:
|
||||
from utils_v2.string import json
|
||||
from utils_v2.string import regex
|
||||
from utils_v2.system import files
|
||||
from utils_v2.date_time import date_time
|
||||
from utils_v2.database.async_mysql_v2 import AsyncMySQL
|
||||
|
||||
# To make HTTP calls:
|
||||
import httpx
|
||||
import socket
|
||||
|
||||
# To work with date and time:
|
||||
import datetime
|
||||
import time
|
||||
|
||||
# To work with NSE:
|
||||
from utils_v2.nse.controllers.bhavcopy.equities import NSEEquitiesBhavCopy
|
||||
from utils_v2.nse.controllers.bhavcopy.fno import NSEFNOBhavCopy
|
||||
|
||||
# To work with datatypes:
|
||||
from typing import List
|
||||
|
||||
# For handling NaN values:
|
||||
import pandas as pd
|
||||
|
||||
# For debugging:
|
||||
from icecream import IceCreamDebugger
|
||||
|
||||
# For asynchronous operations:
|
||||
import asyncio
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MACROS / ONE-TIME INIT ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# To make API calls:
|
||||
http_client = httpx.AsyncClient(
|
||||
limits = httpx.Limits(
|
||||
max_connections = 100, # ............ Maximum number of connections allowed in the pool.
|
||||
max_keepalive_connections = 50, # ... Maximum number of connections that can be kept alive.
|
||||
),
|
||||
timeout = httpx.Timeout(
|
||||
pool = 120.0, # .... Time to wait for a free connection from the pool.
|
||||
connect = 2.5, # ... Time to wait for establishing a connection to the server.
|
||||
write = 10.0, # .... Time to wait for sending data.
|
||||
read = 9.9 # ....... Time to wait for receiving data.
|
||||
)
|
||||
)
|
||||
|
||||
# For debugging:
|
||||
printer = IceCreamDebugger(prefix = "EoD Data | ", includeContext = True)
|
||||
no_context_printer = IceCreamDebugger(prefix = "EoD Data | ", includeContext = False)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** VARIABLES ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
# This script's config.:
|
||||
SERVER_HOSTNAME = str(socket.gethostname())
|
||||
SCRIPT_DATA = {}
|
||||
|
||||
# For the database:
|
||||
sql_writer: AsyncMySQL | None = None
|
||||
|
||||
# To construct a report:
|
||||
failed_to_parse_dates = []
|
||||
failed_eq_dates = []
|
||||
failed_fo_dates = []
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** FUNCTIONS ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
async def init(
|
||||
script_id: str,
|
||||
debug: bool
|
||||
) -> bool:
|
||||
|
||||
"""
|
||||
To initialize all credentials, instances, and connectivity for this whole script.
|
||||
:param script_id: The id to use to load cred and data from the internal service.
|
||||
:param debug: Whether, or not, you would like to print the debug messages.
|
||||
:return: True if initialized successfully, else False.
|
||||
"""
|
||||
|
||||
# Declare the required global variables:
|
||||
global SCRIPT_DATA
|
||||
global sql_writer
|
||||
|
||||
# Basic stuff:
|
||||
if not debug: printer.disable()
|
||||
|
||||
# ┏┓ ┓ ┓ ┳┓
|
||||
# ┃ ┏┓┏┓┏┫ ┏┓┏┓┏┫ ┃┃┏┓╋┏┓
|
||||
# ┗┛┛ ┗ ┗┻ ┗┻┛┗┗┻ ┻┛┗┻┗┗┻
|
||||
|
||||
# Get the script credentials:
|
||||
response = await http_client.get(
|
||||
url = r"https://nexcom.ditscentre.in/internal/cred/get",
|
||||
headers = {"X-Script-Id": script_id}
|
||||
)
|
||||
if response.status_code not in [200]:
|
||||
print("FATAL: SCRIPT CREDENTIALS LOADING FAILED!")
|
||||
return False
|
||||
script_cred = response.json().get("data")
|
||||
|
||||
# Get the script data:
|
||||
response = await http_client.get(
|
||||
url = r"https://nexcom.ditscentre.in/internal/data/get",
|
||||
headers = {"X-Script-Id": script_id}
|
||||
)
|
||||
if response.status_code not in [200]:
|
||||
print("FATAL: SCRIPT DATA LOADING FAILED!")
|
||||
return False
|
||||
SCRIPT_DATA = response.json().get("data")
|
||||
|
||||
# Done with this step:
|
||||
printer("Cred and Data loaded.")
|
||||
|
||||
# ┳┳┓ • ┳┓┳┓
|
||||
# ┃┃┃┏┓┏┓┓┏┓┃┃┣┫
|
||||
# ┛ ┗┗┻┛ ┗┗┻┻┛┻┛
|
||||
|
||||
sql_writer = AsyncMySQL(
|
||||
pool_size = script_cred["mariaDb"]["write"]["poolSize"],
|
||||
host = script_cred["mariaDb"]["write"]["host"],
|
||||
user = script_cred["mariaDb"]["write"]["user"],
|
||||
password = script_cred["mariaDb"]["write"]["password"],
|
||||
database = script_cred["mariaDb"]["write"]["database"]
|
||||
)
|
||||
if not await sql_writer.connect():
|
||||
print("FATAL: MARIA-DB CONNECTION FAILED!")
|
||||
return False
|
||||
|
||||
printer("MariaDB connected.")
|
||||
|
||||
# ┳┓
|
||||
# ┃┃┏┓┏┓┏┓
|
||||
# ┻┛┗┛┛┗┗
|
||||
|
||||
# If everything went well, we return with success:
|
||||
printer("Initialization done.")
|
||||
return True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def capture_eq_eod_data(
|
||||
nse_conn: NSEEquitiesBhavCopy,
|
||||
target_date: datetime.datetime
|
||||
) -> bool:
|
||||
|
||||
# Start by assuming failure:
|
||||
success = False
|
||||
|
||||
# Get the data:
|
||||
no_context_printer("Fetching EQ data.")
|
||||
nse_response = await nse_conn.get_data(
|
||||
target_date = target_date,
|
||||
return_raw = False,
|
||||
retry_count = 3,
|
||||
backoff_seconds = 0.5,
|
||||
backoff_multiplier = 1.1
|
||||
)
|
||||
if not nse_response.success: return success
|
||||
print(nse_response.data[-1])
|
||||
|
||||
# Build the query and data content:
|
||||
query_str = (
|
||||
"INSERT INTO `eod_market_data_temp` (exchange, symbol, segment, type, underlying, is_index, expiry, strike, "
|
||||
"prev_close, open, high, low, close, ltp, vwap, tot_vol, tot_cash, delivery_vol, delivery_pct, oi, oi_chg, "
|
||||
"date, ts, tz, scrape_ts) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
|
||||
)
|
||||
# query_data = []
|
||||
# for data in nse_response.data:
|
||||
# one_query_data = (
|
||||
# "nse", # ... exchange
|
||||
# data["symbol"], # ... symbol
|
||||
# data["segment"], # ... segment
|
||||
# None, # ... type
|
||||
# None, # ... underlying
|
||||
# False, # ... is_index
|
||||
# None, # ... expiry
|
||||
# None, # ... strike
|
||||
# data["prevClose"], # ... prev_close
|
||||
# data["open"], # ... open
|
||||
# data["high"], # ... high
|
||||
# data["low"], # ... low
|
||||
# data["close"], # ... close
|
||||
# data["ltp"], # ... ltp
|
||||
# data["vwap"], # ... vwap
|
||||
# data["totVol"], # ... tot_vol
|
||||
# data["totCash"], # ... tot_cash
|
||||
# data["deliveryVol"], # ... delivery_vol
|
||||
# data["deliveryPct"], # ... delivery_pct
|
||||
# None, # ... oi
|
||||
# None, # ... oi_chg
|
||||
# target_date, # ... date
|
||||
# data["ts"], # ... ts
|
||||
# data["tz"], # ... tz
|
||||
# data["scrapeTs"], # ... scrape_ts
|
||||
# )
|
||||
# one_query_data = [None if pd.isna(d) else d for d in one_query_data]
|
||||
# query_data.append(one_query_data)
|
||||
|
||||
query_data = [
|
||||
(
|
||||
"nse", # ... exchange
|
||||
data["symbol"], # ... symbol
|
||||
data["segment"], # ... segment
|
||||
None, # ... type
|
||||
None, # ... underlying
|
||||
False, # ... is_index
|
||||
None, # ... expiry
|
||||
None, # ... strike
|
||||
data["prevClose"] or None, # ... prev_close
|
||||
data["open"] or None, # ... open
|
||||
data["high"] or None, # ... high
|
||||
data["low"] or None, # ... low
|
||||
data["close"] or None, # ... close
|
||||
data["ltp"] or None, # ... ltp
|
||||
data["vwap"] or None, # ... vwap
|
||||
data["totVol"] or None, # ... tot_vol
|
||||
data["totCash"] or None, # ... tot_cash
|
||||
data["deliveryVol"] or None, # ... delivery_vol
|
||||
data["deliveryPct"] or None, # ... delivery_pct
|
||||
None, # ... oi
|
||||
None, # ... oi_chg
|
||||
target_date, # ... date
|
||||
data["ts"].replace(tzinfo = None), # ... ts
|
||||
data["tz"], # ... tz
|
||||
data["scrapeTs"].replace(tzinfo = None), # ... scrape_ts
|
||||
) for data in nse_response.data
|
||||
]
|
||||
|
||||
# Run the commands:
|
||||
no_context_printer("Saving EQ data to DB.")
|
||||
rows_affected, db_response = await sql_writer.execute_many(
|
||||
query = query_str,
|
||||
data = query_data
|
||||
)
|
||||
|
||||
# Done here:
|
||||
success = True if rows_affected else False
|
||||
no_context_printer(success)
|
||||
return success
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def main(
|
||||
target_dates: List[str],
|
||||
interval: float = 5.0
|
||||
):
|
||||
|
||||
# Declare the needed global variables:
|
||||
global failed_to_parse_dates
|
||||
global failed_eq_dates
|
||||
global failed_fo_dates
|
||||
|
||||
# Create the NSE connection instances:
|
||||
nse_eq_bhavcopy = NSEEquitiesBhavCopy(http_client = http_client, debug = False)
|
||||
nse_fo_bhavcopy = NSEFNOBhavCopy(http_client = http_client, debug = False)
|
||||
|
||||
# Load the data for each date:
|
||||
for dt_str in target_dates:
|
||||
|
||||
# Try parsing the date:
|
||||
dt = date_time.parse_date_time(
|
||||
input_value = dt_str,
|
||||
date_formats = ["%Y-%m-%d", "%Y%m%d"]
|
||||
)
|
||||
|
||||
# If we failed to parse the date:
|
||||
if dt is None:
|
||||
no_context_printer("FAILED to parse date.", dt_str)
|
||||
failed_to_parse_dates.append(dt_str)
|
||||
continue
|
||||
|
||||
no_context_printer("Picked date.", dt_str)
|
||||
|
||||
# Fetch the equity data:
|
||||
success = await capture_eq_eod_data(
|
||||
nse_conn = nse_eq_bhavcopy,
|
||||
target_date = dt
|
||||
)
|
||||
if not success:
|
||||
no_context_printer("FAILED equity for date.", dt_str)
|
||||
failed_eq_dates.append(dt_str)
|
||||
|
||||
# Fetch the derivative data:
|
||||
pass
|
||||
|
||||
# Pause for a while to not get rate-limited/blocked:
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
# Show the report:
|
||||
printer("Process complete.", failed_to_parse_dates, failed_eq_dates, failed_fo_dates)
|
||||
|
||||
|
||||
# *****************************************************************************************************************
|
||||
# ***** ****
|
||||
# *** MAIN PROGRAM ***
|
||||
# ***** ****
|
||||
# *****************************************************************************************************************
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
# To get args. from the terminal:
|
||||
import argparse
|
||||
|
||||
# Get the config. from the command-line:
|
||||
parser = argparse.ArgumentParser(
|
||||
description = (
|
||||
"To fetch and store EoD data for a given data from NSE's BhavCopy section. "
|
||||
"The data will be fetched for both, equities and derivatives."
|
||||
)
|
||||
)
|
||||
parser.add_argument(
|
||||
"-s", "--script-id",
|
||||
dest = "script_id",
|
||||
type = str,
|
||||
help = "The id of this script (will affect the loaded config)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t", "--dates",
|
||||
dest = "dates",
|
||||
type = str,
|
||||
help = "One or more comma-separated dates in 'YYYY-MM-DD' format for which you would like to get the EoD data.",
|
||||
default = None
|
||||
)
|
||||
parser.add_argument(
|
||||
"-i", "--interval",
|
||||
dest = "interval",
|
||||
type = float,
|
||||
help = "The delay, in seconds, between processing two dates.",
|
||||
default = 2.5
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d", "--debug",
|
||||
dest = "debug",
|
||||
action = "store_true",
|
||||
help = "Whether, or not, you want to see debugging messages in the terminal.",
|
||||
default = False
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
async def runner():
|
||||
|
||||
# Prepare the list of target dates:
|
||||
if args.dates is None: target_dates = [datetime.datetime.now().strftime("%Y-%m-%d")]
|
||||
else: target_dates = [d.strip() for d in args.dates.split(",")]
|
||||
|
||||
# Initialize and run the main code:
|
||||
if await init(
|
||||
script_id = args.script_id,
|
||||
debug = args.debug
|
||||
): await main(target_dates = target_dates, interval = args.interval)
|
||||
|
||||
|
||||
asyncio.run(runner())
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
REFERENCES:
|
||||
|
||||
N01. YouTube Webinar: https://www.youtube.com/watch?v=9vzd289Eedk
|
||||
01. YouTube Webinar: https://www.youtube.com/watch?v=9vzd289Eedk
|
||||
02. Official Example (GitHub): https://github.com/zerodha/pykiteconnect/blob/master/examples/threaded_ticker.py
|
||||
|
||||
DOWNLOADS:
|
||||
|
||||
Reference in New Issue
Block a user