Resetting utils subtree.
This commit is contained in:
@@ -217,7 +217,6 @@ async def capture_eq_eod_data(
|
||||
backoff_multiplier = 1.1
|
||||
)
|
||||
if not nse_response.success: return success
|
||||
print(nse_response.data[-1])
|
||||
|
||||
# Build the query and data content:
|
||||
query_str = (
|
||||
@@ -226,67 +225,37 @@ async def capture_eq_eod_data(
|
||||
"date, ts, tz, scrape_ts) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
|
||||
)
|
||||
# query_data = []
|
||||
# for data in nse_response.data:
|
||||
# one_query_data = (
|
||||
# "nse", # ... exchange
|
||||
# data["symbol"], # ... symbol
|
||||
# data["segment"], # ... segment
|
||||
# None, # ... type
|
||||
# None, # ... underlying
|
||||
# False, # ... is_index
|
||||
# None, # ... expiry
|
||||
# None, # ... strike
|
||||
# data["prevClose"], # ... prev_close
|
||||
# data["open"], # ... open
|
||||
# data["high"], # ... high
|
||||
# data["low"], # ... low
|
||||
# data["close"], # ... close
|
||||
# data["ltp"], # ... ltp
|
||||
# data["vwap"], # ... vwap
|
||||
# data["totVol"], # ... tot_vol
|
||||
# data["totCash"], # ... tot_cash
|
||||
# data["deliveryVol"], # ... delivery_vol
|
||||
# data["deliveryPct"], # ... delivery_pct
|
||||
# None, # ... oi
|
||||
# None, # ... oi_chg
|
||||
# target_date, # ... date
|
||||
# data["ts"], # ... ts
|
||||
# data["tz"], # ... tz
|
||||
# data["scrapeTs"], # ... scrape_ts
|
||||
# )
|
||||
# one_query_data = [None if pd.isna(d) else d for d in one_query_data]
|
||||
# query_data.append(one_query_data)
|
||||
|
||||
query_data = [
|
||||
(
|
||||
"nse", # ... exchange
|
||||
data["symbol"], # ... symbol
|
||||
data["segment"], # ... segment
|
||||
None, # ... type
|
||||
None, # ... underlying
|
||||
False, # ... is_index
|
||||
None, # ... expiry
|
||||
None, # ... strike
|
||||
data["prevClose"] or None, # ... prev_close
|
||||
data["open"] or None, # ... open
|
||||
data["high"] or None, # ... high
|
||||
data["low"] or None, # ... low
|
||||
data["close"] or None, # ... close
|
||||
data["ltp"] or None, # ... ltp
|
||||
data["vwap"] or None, # ... vwap
|
||||
data["totVol"] or None, # ... tot_vol
|
||||
data["totCash"] or None, # ... tot_cash
|
||||
data["deliveryVol"] or None, # ... delivery_vol
|
||||
data["deliveryPct"] or None, # ... delivery_pct
|
||||
None, # ... oi
|
||||
None, # ... oi_chg
|
||||
target_date, # ... date
|
||||
data["ts"].replace(tzinfo = None), # ... ts
|
||||
data["tz"], # ... tz
|
||||
query_data = []
|
||||
for data in nse_response.data:
|
||||
one_query_data = (
|
||||
"nse", # ..................................... exchange
|
||||
data["symbol"], # ............................ symbol
|
||||
data["segment"], # ........................... segment
|
||||
None, # ...................................... type
|
||||
None, # ...................................... underlying
|
||||
False, # ..................................... is_index
|
||||
None, # ...................................... expiry
|
||||
None, # ...................................... strike
|
||||
data["prevClose"], # ......................... prev_close
|
||||
data["open"], # .............................. open
|
||||
data["high"], # .............................. high
|
||||
data["low"], # ............................... low
|
||||
data["close"], # ............................. close
|
||||
data["ltp"], # ............................... ltp
|
||||
data["vwap"], # .............................. vwap
|
||||
data["totVol"], # ............................ tot_vol
|
||||
data["totCash"], # ........................... tot_cash
|
||||
data["deliveryVol"], # ....................... delivery_vol
|
||||
data["deliveryPct"], # ....................... delivery_pct
|
||||
None, # ...................................... oi
|
||||
None, # ...................................... oi_chg
|
||||
target_date, # ............................... date
|
||||
data["ts"].replace(tzinfo = None), # ......... ts
|
||||
data["tz"], # ................................ tz
|
||||
data["scrapeTs"].replace(tzinfo = None), # ... scrape_ts
|
||||
) for data in nse_response.data
|
||||
]
|
||||
)
|
||||
one_query_data = [None if pd.isna(d) else d for d in one_query_data]
|
||||
query_data.append(one_query_data)
|
||||
|
||||
# Run the commands:
|
||||
no_context_printer("Saving EQ data to DB.")
|
||||
@@ -304,6 +273,94 @@ async def capture_eq_eod_data(
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def capture_fo_eod_data(
|
||||
nse_conn: NSEFNOBhavCopy,
|
||||
target_date: datetime.datetime
|
||||
) -> bool:
|
||||
|
||||
# Start by assuming failure:
|
||||
success = False
|
||||
|
||||
# Get the data:
|
||||
no_context_printer("Fetching FO data.")
|
||||
nse_response = await nse_conn.get_data(
|
||||
target_date = target_date,
|
||||
return_raw = False,
|
||||
retry_count = 3,
|
||||
backoff_seconds = 0.5,
|
||||
backoff_multiplier = 1.1
|
||||
)
|
||||
if not nse_response.success: return success
|
||||
|
||||
# Build the query and data content:
|
||||
query_str = (
|
||||
"INSERT INTO `eod_market_data_temp` (exchange, symbol, segment, type, underlying, is_index, expiry, strike, "
|
||||
"prev_close, open, high, low, close, ltp, vwap, tot_vol, tot_cash, delivery_vol, delivery_pct, oi, oi_chg, "
|
||||
"date, ts, tz, scrape_ts) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);"
|
||||
)
|
||||
query_data = []
|
||||
for data in nse_response.data:
|
||||
expiry = data["expiryTs"]
|
||||
if isinstance(expiry, pd._libs.tslibs.timestamps.Timestamp): expiry = expiry.to_pydatetime()
|
||||
if isinstance(expiry, datetime.datetime):
|
||||
expiry = date_time.to_timezone(
|
||||
datetime_object = expiry,
|
||||
timezone = date_time.TIMEZONE_UTC
|
||||
).strftime("%Y-%m-%d")
|
||||
one_query_data = (
|
||||
"nse", # ..................................... exchange
|
||||
data["symbol"], # ............................ symbol
|
||||
data["segment"], # ........................... segment
|
||||
data["type"], # .............................. type
|
||||
data["underlying"], # ........................ underlying
|
||||
data["isIndex"], # ........................... is_index
|
||||
expiry, # .................................... expiry
|
||||
data["strike"], # ............................ strike
|
||||
data["prevClose"], # ......................... prev_close
|
||||
data["open"], # .............................. open
|
||||
data["high"], # .............................. high
|
||||
data["low"], # ............................... low
|
||||
data["close"], # ............................. close
|
||||
None, # ...................................... ltp
|
||||
None, # ...................................... vwap
|
||||
data["totVol"], # ............................ tot_vol
|
||||
data["totCash"], # ........................... tot_cash
|
||||
None, # ...................................... delivery_vol
|
||||
None, # ...................................... delivery_pct
|
||||
data["oi"], # ................................ oi
|
||||
data["oiChg"], # ............................. oi_chg
|
||||
target_date, # ............................... date
|
||||
data["ts"].replace(tzinfo = None), # ......... ts
|
||||
data["tz"], # ................................ tz
|
||||
data["scrapeTs"].replace(tzinfo = None), # ... scrape_ts
|
||||
)
|
||||
one_query_data = [None if pd.isna(d) else d for d in one_query_data]
|
||||
query_data.append(one_query_data)
|
||||
|
||||
# Run the commands:
|
||||
print(query_data[3034][19])
|
||||
print(query_data[3035][19])
|
||||
print(query_data[3036][19])
|
||||
print("---")
|
||||
print(query_data[2731][19])
|
||||
print(query_data[2732][19])
|
||||
print(query_data[2733][19])
|
||||
no_context_printer("Saving FO data to DB.")
|
||||
rows_affected, db_response = await sql_writer.execute_many(
|
||||
query = query_str,
|
||||
data = query_data
|
||||
)
|
||||
|
||||
# Done here:
|
||||
success = True if rows_affected else False
|
||||
no_context_printer(success)
|
||||
return success
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def main(
|
||||
target_dates: List[str],
|
||||
interval: float = 5.0
|
||||
@@ -335,20 +392,27 @@ async def main(
|
||||
|
||||
no_context_printer("Picked date.", dt_str)
|
||||
|
||||
# Fetch the equity data:
|
||||
success = await capture_eq_eod_data(
|
||||
nse_conn = nse_eq_bhavcopy,
|
||||
# # Fetch the equity data:
|
||||
# success = await capture_eq_eod_data(
|
||||
# nse_conn = nse_eq_bhavcopy,
|
||||
# target_date = dt
|
||||
# )
|
||||
# if not success:
|
||||
# no_context_printer("FAILED equity for date.", dt_str)
|
||||
# failed_eq_dates.append(dt_str)
|
||||
|
||||
# Fetch the derivative data:
|
||||
success = await capture_fo_eod_data(
|
||||
nse_conn = nse_fo_bhavcopy,
|
||||
target_date = dt
|
||||
)
|
||||
if not success:
|
||||
no_context_printer("FAILED equity for date.", dt_str)
|
||||
failed_eq_dates.append(dt_str)
|
||||
|
||||
# Fetch the derivative data:
|
||||
pass
|
||||
no_context_printer("FAILED derivatives for date.", dt_str)
|
||||
failed_fo_dates.append(dt_str)
|
||||
|
||||
# Pause for a while to not get rate-limited/blocked:
|
||||
await asyncio.sleep(interval)
|
||||
# break
|
||||
|
||||
# Show the report:
|
||||
printer("Process complete.", failed_to_parse_dates, failed_eq_dates, failed_fo_dates)
|
||||
@@ -415,5 +479,8 @@ if __name__ == "__main__":
|
||||
debug = args.debug
|
||||
): await main(target_dates = target_dates, interval = args.interval)
|
||||
|
||||
# Disconnect from the database:
|
||||
disconnected = await sql_writer.disconnect()
|
||||
|
||||
|
||||
asyncio.run(runner())
|
||||
|
||||
Reference in New Issue
Block a user