khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resetting utils subtree.

Browse Source
This commit is contained in:
khushal
2025-02-28 13:35:45 +05:30
parent 07800bd403
commit d07ed14854
212 changed files with 171 additions and 148158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cron/finstitutions/trading/latest_ohlc_from_ticks.py
+114 -115
View File
@@ -10,7 +10,7 @@
OBJECTIVE:
To infer the day's latest EoD
To infer the day's latest OHLC values from the ticks stored in the database.
REFERENCES:
@@ -93,8 +93,8 @@ http_client = httpx.AsyncClient(
)
# For debugging:
printer = IceCreamDebugger(prefix = "EoD (Ticks) | ", includeContext = True)
no_context_printer = IceCreamDebugger(prefix = "EoD (Ticks) | ", includeContext = False)
printer = IceCreamDebugger(prefix = "1D OHLC (Ticks) | ", includeContext = True)
no_context_printer = IceCreamDebugger(prefix = "1D OHLC (Ticks) | ", includeContext = False)
# *****************************************************************************************************************
@@ -170,7 +170,7 @@ async def init(
SCRIPT_DATA = response.json().get("data")
# Done with this step:
printer("Cred and Data loaded.")
no_context_printer("Cred and Data loaded.")
# ┳┳┓ • ┳┓┳┓
# ┃┃┃┏┓┏┓┓┏┓┃┃┣┫
@@ -187,7 +187,7 @@ async def init(
print("FATAL: MARIA-DB CONNECTION FAILED!")
return False
printer("MariaDB connected.")
no_context_printer("MariaDB connected.")
# ┳┳┓
# ┃┃┃┏┓┏┓┏┓┏┓
@@ -211,7 +211,7 @@ async def init(
# ┻┛┗┛┛┗┗
# If everything went well, we return with success:
printer("Initialization done.")
no_context_printer("Initialization done.")
return True
@@ -259,21 +259,21 @@ async def send_telegram(
# ---------------------------------------------------------------------------------------------------------------------
async def get_latest_eod_data(target_date: datetime.datetime = None) -> dict:
async def get_latest_ohlc_data(target_date: datetime.datetime = None) -> dict:
"""
To fetch the latest EoD (daily candle) data from the ticks database.
To fetch the latest 1D OHLC (daily candle) data from the ticks database.
:param target_date: The date (UTC) whose EoD ticks are desired.
:return: The inferred daily candle data.
"""
no_context_printer("Getting EoD data from ticks.")
no_context_printer("Getting 1D OHLC data from ticks.")
# Prepare the inputs needed for the aggregation:
if not isinstance(target_date, datetime.datetime): target_date = date_time.get_current_utc_date_time()
else: target_date = date_time.to_timezone(target_date, date_time.TIMEZONE_UTC)
start_ts = target_date.replace(hour = 3, minute = 45, second = 0, microsecond = 0)
end_ts = target_date.replace(hour = 10, minute = 0, second = 0, microsecond = 0)
start_ts = target_date.replace(hour = 0, minute = 0, second = 0, microsecond = 0)
end_ts = target_date.replace(hour = 23, minute = 59, second = 59, microsecond = 999)
# Construct the aggregation pipeline:
# Consider only the target date's ticks:
@@ -289,77 +289,39 @@ async def get_latest_eod_data(target_date: datetime.datetime = None) -> dict:
# Add a field that has the rounded timestamp.
# We round it to one day for EoD data:
stage_1 = {
"$addFields": {
"roundTs": {
"$dateTrunc": {
"date": "$tradeTs",
"unit": "day",
"binSize": 1
}
}
"$sort": {
"tradeTs": -1
}
}
# Now we convert tick to candlesticks:
stage_2 = {
"$group": {
"_id": {
"roundTs": "$roundTs",
"symbol": "$symbol"
},
"roundTs": {"$last": "$roundTs"},
"symbol": {"$last": "$symbol"},
"name": {"$last": "$name"},
"exchange": {"$last": "$exchange"},
"segment": {"$last": "$segment"},
"type": {"$last": "$type"},
"expiry": {"$last": "$expiry"},
"strike": {"$last": "$strike"},
"open": {"$first": "$ltp"},
"high": {"$max": "$ltp"},
"low": {"$min": "$ltp"},
"close": {"$last": "$ltp"},
"vwap": {"$last": "$vwap"},
"chg": {"$last": "$chg"},
"pChg": {"$last": "$pChg"},
"volume": {"$last": "$totVol"},
"dayHigh": {"$last": "$h"},
"dayLow": {"$last": "$l"},
"ticks": {"$sum": 1},
"broker": {"$last": "$broker"},
"brokerToken": {"$last": "$brokerToken"},
}
}
# Finally we organize and present the data:
stage_3 = {
"$sort": {
"symbol": 1,
"roundTs": 1
}
}
stage_4 = {
"$project": {
"_id": False
"_id": "$symbol",
"latestTick": {"$first": "$$ROOT"}
}
}
# Now we run the aggregation:
eod_data = await data_mongo.aggregate(
agg_ohlc_data = await data_mongo.aggregate(
collection = "__hot_zerodhaTicks",
pipeline = [
stage_0,
stage_1,
stage_2,
stage_3,
stage_4
stage_2
],
limit = None,
raise_exception = False
)
# Format the aggregation now:
ohlc_data = {
item["_id"]: item["latestTick"]
for item in agg_ohlc_data or []
}
# Done here:
return eod_data
return ohlc_data
# ---------------------------------------------------------------------------------------------------------------------
@@ -369,7 +331,7 @@ async def save_latest_eod_data(eod_data: dict) -> bool:
"""
To save the loaded data to the SQL database.
:param eod_data: The dict of the EoD data received from the ticks database.
:param eod_data: The dict of the latest 1D OHLC data received from the ticks database.
:return: True if successful, else False
"""
@@ -384,10 +346,10 @@ async def save_latest_eod_data(eod_data: dict) -> bool:
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
)
query_data = []
for data in eod_data:
total_cash = data["vwap"] * data["volume"]
target_date = data["roundTs"].strftime("%Y-%m-%d")
curr_close = data["close"]
for symbol, data in eod_data.items():
total_cash = data["vwap"] * data["totVol"]
target_date = data["tradeTs"].strftime("%Y-%m-%d")
curr_close = data["ltp"]
prev_close = curr_close - data["chg"]
one_query_data = (
data["exchange"], # ......................... exchange
@@ -399,28 +361,40 @@ async def save_latest_eod_data(eod_data: dict) -> bool:
data["expiry"], # ........................... expiry
data["strike"], # ........................... strike
prev_close, # ............................... prev_close
data["open"], # ............................. open
data["high"], # ............................. high
data["low"], # .............................. low
data["o"], # ................................ open
data["h"], # ................................ high
data["l"], # ................................ low
curr_close, # ............................... close
curr_close, # ............................... ltp
data["vwap"], # ............................. vwap
data["volume"], # ........................... tot_vol
data["totVol"], # ........................... tot_vol
total_cash, # ............................... tot_cash
None, # ..................................... delivery_vol
None, # ..................................... delivery_pct
None, # ..................................... oi
None, # ..................................... oi_chg
target_date, # .............................. date
data["roundTs"].replace(tzinfo = None), # ... ts
data["tradeTs"].replace(tzinfo = None), # ... ts
"Asia/Kolkata", # ........................... tz
scrape_ts.replace(tzinfo = None), # ......... scrape_ts
)
one_query_data = [None if pd.isna(d) else d for d in one_query_data]
query_data.append(one_query_data)
# If the query data is empty:
if not query_data:
await send_telegram(
message = (
f"*1D OHLC From Ticks:*\n\n"
"Message: `Did NOT find any 1D OHLC data from ticks.`\n\n"
),
message_type = "warning"
)
return False
# Run the commands:
no_context_printer("Saving data to SQL DB.")
no_context_printer(len(query_data))
rows_affected, db_response, db_exception = await sql_writer.execute_many(
query = query_str,
data = query_data,
@@ -428,7 +402,7 @@ async def save_latest_eod_data(eod_data: dict) -> bool:
)
if db_exception: await send_telegram(
message = (
f"*EoD From Ticks:*\n\n"
f"*1D OHLC From Ticks:*\n\n"
"Message: `SQL database threw an exception.`\n\n"
f"Exception: `{db_exception}`"
),
@@ -452,38 +426,39 @@ async def run_once() -> bool:
"""
# Get the data from the ticks database:
eod_data = await get_latest_eod_data()
if not eod_data:
await send_telegram(
message = (
f"*EoD From Ticks:*\n\n"
"Message: `Failed to get EoD candles from ticks.`"
),
message_type = "error"
)
return False
ohlc_data = await get_latest_ohlc_data()
# If there is no data to save:
if not eod_data:
if not ohlc_data:
# Send out an alert:
await send_telegram(
message = (
f"*EoD From Ticks:*\n\n"
"Message: `No EoD data to save to SQL.`"
f"*1D OHLC From Ticks:*\n\n"
"Message: `Failed to get 1D OHLC candles from ticks.`"
),
message_type = "error"
)
# Return with failure:
return False
# Save the data to the SQL database:
success = await save_latest_eod_data(eod_data)
success = await save_latest_eod_data(ohlc_data)
# If the data was not saved:
if not success:
# Send out an alert:
await send_telegram(
message = (
f"*EoD From Ticks:*\n\n"
f"*1D OHLC From Ticks:*\n\n"
"Message: `Failed to save EoD data to SQL.`"
),
message_type = "error"
)
# Return with failure:
return False
# If both th steps succeeded, we are good to go:
@@ -493,39 +468,60 @@ async def run_once() -> bool:
# ---------------------------------------------------------------------------------------------------------------------
async def main(
start_time: datetime.datetime,
end_time: datetime.datetime,
interval_seconds: int = 300,
):
async def main(script_args) -> None:
"""
The main scheduler that manages jobs.
:param start_time: The time of the day at which messages can start going out.
:param end_time: The time of the day after which new messages should not go out.
:param interval_seconds: The time (in seconds) between two reminder jobs.
:param script_args: The args received from the command line.
:return: None.
"""
# ┏┳ ┓ ┏┳┓•
# ┃┏┓┣┓ ┃ ┓┏┳┓┏┓┏╋┏┓┏┳┓┏┓┏
# ┗┛┗┛┗┛ ┻ ┗┛┗┗┗ ┛┗┗┻┛┗┗┣┛┛
# ┛
# Figure out the system's timezone so that cron activities can run as per it:
system_tz = date_time.get_system_timezone(as_string = False)
# Start configuring the scheduler:
printer("Configuring the schedule-manager.")
no_context_printer("Configuring the schedule-manager.")
schedule_manager = Scheduler()
# Create all the timestamps at which the job must be done:
all_job_ts = []
# Get the current date-time and parse the open and close time values as UTC.
# The input string does NOT have the date value. By default, python will take a date from way back in the past.
# When you translate from UTC to the local machine's timezone, if the date value is from way back in the past, an
# accidental historical timezone may get applied. Fo example, there was a time before the adoption of IST when
# "Asia/Kolkata" meant an offset of +05:53. Ensure to replace the date values to today's values to avoid ending up
# with old offsets:
now = date_time.get_current_utc_date_time(as_string = False)
now = date_time.to_timezone(now, timezone = system_tz)
# ---
jobs_start_time = datetime.datetime.strptime(script_args.start_time, "%H:%M:%S")
jobs_start_time = jobs_start_time.replace(year = now.year, month = now.month, day = now.day)
jobs_start_time = date_time.as_if_timezone(jobs_start_time, timezone = date_time.TIMEZONE_UTC)
jobs_start_time = date_time.to_timezone(jobs_start_time, timezone = system_tz)
# ---
jobs_end_time = datetime.datetime.strptime(script_args.end_time, "%H:%M:%S")
jobs_end_time = jobs_end_time.replace(year = now.year, month = now.month, day = now.day)
jobs_end_time = date_time.as_if_timezone(jobs_end_time, timezone = date_time.TIMEZONE_UTC)
jobs_end_time = date_time.to_timezone(jobs_end_time, timezone = system_tz)
# Create all the timestamps at which the job(s) must be done:
all_jobs_ts = []
offset_seconds = 0
while True:
ts = start_time + datetime.timedelta(seconds = offset_seconds)
if ts > end_time: break
all_job_ts.append(ts.time())
offset_seconds += interval_seconds
ts = jobs_start_time + datetime.timedelta(seconds = offset_seconds)
if ts > jobs_end_time: break
all_jobs_ts.append(date_time.to_timezone(ts, timezone = system_tz))
offset_seconds += script_args.interval
# Add the jobs:
for ts in all_job_ts: schedule_manager.daily(ts, run_once)
printer(len(all_job_ts))
for ts in all_jobs_ts: schedule_manager.daily(ts.time(), run_once)
no_context_printer(len(all_jobs_ts))
# Infinite loop to keep doing the tasks:
printer("Schedule-manager ready.")
no_context_printer("Schedule-manager ready.")
while True: await asyncio.sleep(3_600)
@@ -544,7 +540,8 @@ if __name__ == "__main__":
# Get the config. from the command-line:
parser = argparse.ArgumentParser(
description = (
"To periodically infer EoD data from tick-by-tick data and feed it into the SQL database."
"To periodically infer the current day's latest 1D OHLC data from tick-by-tick data and feed it into the "
"SQL database."
)
)
parser.add_argument(
@@ -555,13 +552,15 @@ if __name__ == "__main__":
)
parser.add_argument(
"--start-time",
dest = "start_time",
type = str,
help = "The 24-hr time of the day (in 'HH:MM:SS' format) from which the data can be refreshed."
help = "The UTC 24-hr time of the day (in 'HH:MM:SS' format) from which the data can be refreshed."
)
parser.add_argument(
"--end-time",
dest = "end_time",
type = str,
help = "The 24-hr time of the day (in 'HH:MM:SS' format) till which the data must be refreshed."
help = "The UTC 24-hr time of the day (in 'HH:MM:SS' format) till which the data must be refreshed."
)
parser.add_argument(
"--interval",
@@ -588,14 +587,14 @@ if __name__ == "__main__":
# Initialize and run the main code:
if await init(
script_id = args.script_id,
debug = args.debug
): await main(
start_time = start_time,
end_time = end_time,
interval_seconds = args.interval,
debug = False
): await main(args)
else: await send_telegram(
message = "Failed to initialize DB connectivity for 1D OHLC from Ticks!",
message_type = "error"
)
# Disconnect from the database:
# Disconnect from the database(s):
disconnected = await sql_writer.disconnect()
# disconnected = await data_mongo.disconnect()