khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resetting utils subtree.

Browse Source
This commit is contained in:
khushal
2025-01-07 18:51:02 +05:30
parent b20d4850c2
commit 9a9be7d4dc
187 changed files with 140 additions and 141306 deletions
Download Patch File Download Diff File Expand all files Collapse all files
background/finstitutions/trading/eod_capture.py
+136 -69
View File
@@ -217,7 +217,6 @@ async def capture_eq_eod_data(
backoff_multiplier = 1.1
)
if not nse_response.success: return success
print(nse_response.data[-1])
# Build the query and data content:
query_str = (
@@ -226,67 +225,37 @@ async def capture_eq_eod_data(
"date, ts, tz, scrape_ts) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
)
# query_data = []
# for data in nse_response.data:
# one_query_data = (
# "nse", # ... exchange
# data["symbol"], # ... symbol
# data["segment"], # ... segment
# None, # ... type
# None, # ... underlying
# False, # ... is_index
# None, # ... expiry
# None, # ... strike
# data["prevClose"], # ... prev_close
# data["open"], # ... open
# data["high"], # ... high
# data["low"], # ... low
# data["close"], # ... close
# data["ltp"], # ... ltp
# data["vwap"], # ... vwap
# data["totVol"], # ... tot_vol
# data["totCash"], # ... tot_cash
# data["deliveryVol"], # ... delivery_vol
# data["deliveryPct"], # ... delivery_pct
# None, # ... oi
# None, # ... oi_chg
# target_date, # ... date
# data["ts"], # ... ts
# data["tz"], # ... tz
# data["scrapeTs"], # ... scrape_ts
# )
# one_query_data = [None if pd.isna(d) else d for d in one_query_data]
# query_data.append(one_query_data)
query_data = [
(
"nse", # ... exchange
data["symbol"], # ... symbol
data["segment"], # ... segment
None, # ... type
None, # ... underlying
False, # ... is_index
None, # ... expiry
None, # ... strike
data["prevClose"] or None, # ... prev_close
data["open"] or None, # ... open
data["high"] or None, # ... high
data["low"] or None, # ... low
data["close"] or None, # ... close
data["ltp"] or None, # ... ltp
data["vwap"] or None, # ... vwap
data["totVol"] or None, # ... tot_vol
data["totCash"] or None, # ... tot_cash
data["deliveryVol"] or None, # ... delivery_vol
data["deliveryPct"] or None, # ... delivery_pct
None, # ... oi
None, # ... oi_chg
target_date, # ... date
data["ts"].replace(tzinfo = None), # ... ts
data["tz"], # ... tz
query_data = []
for data in nse_response.data:
one_query_data = (
"nse", # ..................................... exchange
data["symbol"], # ............................ symbol
data["segment"], # ........................... segment
None, # ...................................... type
None, # ...................................... underlying
False, # ..................................... is_index
None, # ...................................... expiry
None, # ...................................... strike
data["prevClose"], # ......................... prev_close
data["open"], # .............................. open
data["high"], # .............................. high
data["low"], # ............................... low
data["close"], # ............................. close
data["ltp"], # ............................... ltp
data["vwap"], # .............................. vwap
data["totVol"], # ............................ tot_vol
data["totCash"], # ........................... tot_cash
data["deliveryVol"], # ....................... delivery_vol
data["deliveryPct"], # ....................... delivery_pct
None, # ...................................... oi
None, # ...................................... oi_chg
target_date, # ............................... date
data["ts"].replace(tzinfo = None), # ......... ts
data["tz"], # ................................ tz
data["scrapeTs"].replace(tzinfo = None), # ... scrape_ts
) for data in nse_response.data
]
)
one_query_data = [None if pd.isna(d) else d for d in one_query_data]
query_data.append(one_query_data)
# Run the commands:
no_context_printer("Saving EQ data to DB.")
@@ -304,6 +273,94 @@ async def capture_eq_eod_data(
# ---------------------------------------------------------------------------------------------------------------------
async def capture_fo_eod_data(
nse_conn: NSEFNOBhavCopy,
target_date: datetime.datetime
) -> bool:
# Start by assuming failure:
success = False
# Get the data:
no_context_printer("Fetching FO data.")
nse_response = await nse_conn.get_data(
target_date = target_date,
return_raw = False,
retry_count = 3,
backoff_seconds = 0.5,
backoff_multiplier = 1.1
)
if not nse_response.success: return success
# Build the query and data content:
query_str = (
"INSERT INTO `eod_market_data_temp` (exchange, symbol, segment, type, underlying, is_index, expiry, strike, "
"prev_close, open, high, low, close, ltp, vwap, tot_vol, tot_cash, delivery_vol, delivery_pct, oi, oi_chg, "
"date, ts, tz, scrape_ts) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
)
query_data = []
for data in nse_response.data:
expiry = data["expiryTs"]
if isinstance(expiry, pd._libs.tslibs.timestamps.Timestamp): expiry = expiry.to_pydatetime()
if isinstance(expiry, datetime.datetime):
expiry = date_time.to_timezone(
datetime_object = expiry,
timezone = date_time.TIMEZONE_UTC
).strftime("%Y-%m-%d")
one_query_data = (
"nse", # ..................................... exchange
data["symbol"], # ............................ symbol
data["segment"], # ........................... segment
data["type"], # .............................. type
data["underlying"], # ........................ underlying
data["isIndex"], # ........................... is_index
expiry, # .................................... expiry
data["strike"], # ............................ strike
data["prevClose"], # ......................... prev_close
data["open"], # .............................. open
data["high"], # .............................. high
data["low"], # ............................... low
data["close"], # ............................. close
None, # ...................................... ltp
None, # ...................................... vwap
data["totVol"], # ............................ tot_vol
data["totCash"], # ........................... tot_cash
None, # ...................................... delivery_vol
None, # ...................................... delivery_pct
data["oi"], # ................................ oi
data["oiChg"], # ............................. oi_chg
target_date, # ............................... date
data["ts"].replace(tzinfo = None), # ......... ts
data["tz"], # ................................ tz
data["scrapeTs"].replace(tzinfo = None), # ... scrape_ts
)
one_query_data = [None if pd.isna(d) else d for d in one_query_data]
query_data.append(one_query_data)
# Run the commands:
print(query_data[3034][19])
print(query_data[3035][19])
print(query_data[3036][19])
print("---")
print(query_data[2731][19])
print(query_data[2732][19])
print(query_data[2733][19])
no_context_printer("Saving FO data to DB.")
rows_affected, db_response = await sql_writer.execute_many(
query = query_str,
data = query_data
)
# Done here:
success = True if rows_affected else False
no_context_printer(success)
return success
# ---------------------------------------------------------------------------------------------------------------------
async def main(
target_dates: List[str],
interval: float = 5.0
@@ -335,20 +392,27 @@ async def main(
no_context_printer("Picked date.", dt_str)
# Fetch the equity data:
success = await capture_eq_eod_data(
nse_conn = nse_eq_bhavcopy,
# # Fetch the equity data:
# success = await capture_eq_eod_data(
# nse_conn = nse_eq_bhavcopy,
# target_date = dt
# )
# if not success:
# no_context_printer("FAILED equity for date.", dt_str)
# failed_eq_dates.append(dt_str)
# Fetch the derivative data:
success = await capture_fo_eod_data(
nse_conn = nse_fo_bhavcopy,
target_date = dt
)
if not success:
no_context_printer("FAILED equity for date.", dt_str)
failed_eq_dates.append(dt_str)
# Fetch the derivative data:
pass
no_context_printer("FAILED derivatives for date.", dt_str)
failed_fo_dates.append(dt_str)
# Pause for a while to not get rate-limited/blocked:
await asyncio.sleep(interval)
# break
# Show the report:
printer("Process complete.", failed_to_parse_dates, failed_eq_dates, failed_fo_dates)
@@ -415,5 +479,8 @@ if __name__ == "__main__":
debug = args.debug
): await main(target_dates = target_dates, interval = args.interval)
# Disconnect from the database:
disconnected = await sql_writer.disconnect()
asyncio.run(runner())
background/finstitutions/trading/tick_in_stateful.py
+4 -1
View File
@@ -425,7 +425,10 @@ def init(
ca_file = producer_creds["config"].get("caFile"),
cert_file = producer_creds["config"].get("certFile"),
key_file = producer_creds["config"].get("keyFile"),
client_id = f"{SERVER_HOSTNAME}_{TOKEN_KEY}"
client_id = f"{SERVER_HOSTNAME}_{TOKEN_KEY}",
acks = producer_creds["config"].get("acks", 1),
retries = producer_creds["config"].get("retries", 1),
linger_ms = producer_creds["config"].get("lingerMs", 0)
),
topic = producer_creds["topic"],
serializer = JSONSerializer(),