(20250117) Minor changes in NSE scraper for getting constituents of an index.

api/blueprints/message/mail/tags/update_v2.py

@@ -6,7 +6,7 @@
 
     DATE:
 
-        Friday, 13th Dec., 2024
+        Friday, 17th Jan., 2025.
 
     OBJECTIVE:
 
@@ -188,7 +188,7 @@ async def update_mail_tags(
 
     # We check if the token that was used to fetch the mail is owned by this user:
     if not await token_check.is_authorized(
-            mongo_conn=current_app.data_mongo,
+            mongo_data_conn = current_app.data_mongo,
             user_info = CoreUserInfoModel(**kwargs["session_info"]),
             token_ids = [message.tokenId]
     ): return ResponseModel(
@@ -203,7 +203,7 @@ async def update_mail_tags(
     #   ┛
 
     # Update the mail:
-    success = await current_app.mail_controller.update_tags(
+    success = await current_app.mail_controller.update_mail_tags(
         mongo_conn = current_app.data_mongo,
         message_id = inbound_data.messageId,
         unset_tags = inbound_data.unsetTags,

utils_v2/goog/controllers/base.py

@@ -161,6 +161,10 @@ class AsyncGoogleBase:
     def client_secret(self):
         return self._client_secret
 
+    @property
+    def http_client(self):
+        return self._http_client
+
     # ┏┓┏┓   ┓   ┏┓ ┏┓
     # ┃┃┣┫┓┏╋┣┓  ┏┛ ┃┫
     # ┗┛┛┗┗┻┗┛┗  ┗━•┗┛

utils_v2/nse/controllers/indices/constituents.py

@@ -209,7 +209,7 @@ class NSEIndexConstituents(AsyncNSEBase):
 
     def __init__(
             self,
-            http_client: httpx.AsyncClient,
+            http_client: httpx.AsyncClient = None,
             cookies_refresh_interval: int | float = 300,
             debug = True,
             debug_prefix = "NSE (IdxCons) | ",
@@ -280,8 +280,8 @@ class NSEIndexConstituents(AsyncNSEBase):
         # Done here:
         return api_response
 
-    @staticmethod
     def format_data(
+            self,
             raw_json: dict,
             index_name: str,
             timestamp: datetime.datetime = None,
@@ -314,21 +314,19 @@ class NSEIndexConstituents(AsyncNSEBase):
             formatted_data = [
                 {
                     "scrapeTs": timestamp,
-                    "ts": date_time.to_timezone(
+                    "ts": self.parse_datetime_string(
-                        date_time.parse_date_time(
+                        dt_str = symbol.get("lastUpdateTime"),
-                            input_value = symbol["lastUpdateTime"],
+                        dt_formats = ["%d-%b-%Y %H:%M:%S"],
-                            date_formats = ["%d-%b-%Y %H:%M:%S"],
+                        input_tz = date_time.TIMEZONE_IST,
-                            timezone = date_time.TIMEZONE_IST
+                        output_tz = date_time.TIMEZONE_UTC
-                        ),
-                        timezone = date_time.TIMEZONE_UTC
                     ),
                     "indexName": index_name,
                     "symbol": symbol["symbol"],
-                    "name": symbol["meta"]["companyName"],
+                    "name": symbol.get("meta", {}).get("companyName"),
-                    "industry": symbol["meta"]["industry"],
+                    "industry": symbol.get("meta", {}).get("industry"),
-                    "isFNOSec": symbol["meta"]["isFNOSec"],
+                    "isFNOSec": symbol.get("meta", {}).get("isFNOSec"),
-                    "isSuspended": symbol["meta"]["isSuspended"],
+                    "isSuspended": symbol.get("meta", {}).get("isSuspended"),
-                    "isin": symbol["meta"]["isin"],
+                    "isin": symbol.get("meta", {}).get("isin"),
                     "open": symbol["open"],
                     "high": symbol["dayHigh"],
                     "low": symbol["dayLow"],
@@ -340,10 +338,10 @@ class NSEIndexConstituents(AsyncNSEBase):
                     "pChg": symbol["pChange"],
                     "yearHigh": symbol["yearHigh"],
                     "yearLow": symbol["yearLow"],
-                    "pChg30d": symbol["perChange30d"],
+                    "pChg30d": symbol.get("perChange30d"),
-                    "pChg365d": symbol["perChange365d"],
+                    "pChg365d": symbol.get("perChange365d"),
-                    "ffmc": symbol["ffmc"]
+                    "ffmc": symbol.get("ffmc")
-                } for symbol in raw_json["data"] if symbol["priority"] == 0
+                } for symbol in raw_json["data"] if symbol.get("priority") in [0, None]
             ]
 
         # If something goes wrong:
@@ -366,26 +364,12 @@ if __name__ == "__main__":
 
     async def main():
 
-        # Create an HTTP client:
-        test_client = httpx.AsyncClient(
-            limits = httpx.Limits(
-                max_connections = 100,  # ............ Maximum number of connections allowed in the pool.
-                max_keepalive_connections = 50,  # ... Maximum number of connections that can be kept alive.
-            ),
-            timeout = httpx.Timeout(
-                pool = 120.0,  # .... Time to wait for a free connection from the pool.
-                connect = 2.5,  # ... Time to wait for establishing a connection to the server.
-                write = 10.0,  # .... Time to wait for sending data.
-                read = 2.5  # ....... Time to wait for receiving data.
-            )
-        )
-
         # Create an instance of the scraper, and refresh its cookies:
-        my_nse = NSEIndexConstituents(http_client = test_client)
+        my_nse = NSEIndexConstituents()
 
         # Get and show the data:
         api_response = await my_nse.get_data(
-            index_name = NSEIndexConstituents.INDEX_NIFTY_50,
+            index_name = NSEIndexConstituents.INDEX_NIFTY_500,
             return_raw = False
         )
         print("SUMMARY:", api_response.to_markdown(), "\n---\n\n")