khushalps/api_utils_converse_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(20250117) Day-end push.

Browse Source
This commit is contained in:
khushal
2025-01-17 18:57:46 +05:30
parent 8ab63d036d
commit 4b2cdbee76
7 changed files with 356 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controllers_v2/core/auth_token.py
+113 -1
View File
@@ -21,7 +21,7 @@
N/A
"""
import datetime
# *****************************************************************************************************************
# ***** ****
# *** IMPORT ***
@@ -55,6 +55,7 @@ import httpx
# To work with MongoDB:
from bson import ObjectId
from pymongo import UpdateOne
# *****************************************************************************************************************
@@ -571,6 +572,117 @@ class CoreAuthTokenController(CoreBaseModel):
# Done here:
return [CoreAuthTokenModel(**token) for token in tokens]
async def get_batches_to_sync(
self,
mongo_data_conn: AsyncMongo,
limit: int = 25,
batch_timeout_seconds: int | float = 120,
additional_filter: dict = None
) -> List[CoreAuthTokenModel]:
"""
Use this when some kind of account that has been integrated needs to perform data synchronization at periodic
intervals. One simple example is mails. The aim is to pick batches of auth-tokens
:param mongo_data_conn: The database connection (MongoDB) to use to perform the action.
:param limit: The max. no. of records to pick.
:param batch_timeout_seconds: The no. of seconds to not pick an element that has been marked as a member of
another batch.
:param additional_filter: Any addition filters to use.
:return: The retrieved record that has the token, and information about the service and client if found, else
None when there is no matching record.
"""
# Prepare the filter:
now = date_time.get_current_utc_date_time(as_string = False)
batch_timeout_ts = now - datetime.timedelta(seconds = int(batch_timeout_seconds))
filter_json = {
"$and": [
{
"$or": [
{"syncAfterTs": None}, # ............ The time after which sync'ing is allowed is null.
{"syncAfterTs": {"$lte": now}}, # ... The time after which sync'ing is allowed has passed.
]
},
{
"$or": [
{"batchId": None}, # ....................... Not currently a member of a batch.
{"batchTs": {"$lt": batch_timeout_ts}} # ... Is a member of a batch, but it has timed-out.
]
}
]
}
if self._base_filter:
for k, v in self._base_filter.items(): filter_json[k] = v
if additional_filter:
for k, v in additional_filter.items(): filter_json[k] = v
# Generate a batch-id:
batch_id = ObjectId()
# fetch the credentials that match the conditions:
tokens = await mongo_data_conn.find_many(
collection = self.AUTH_COLLECTION,
filter = filter_json,
sort = {"lastSyncTs": 1},
limit = limit,
projection = {"_id": True}
)
token_ids = [t["_id"] for t in tokens]
# Mark 'n' no. of auth-tokens as a member of your batch:
await mongo_data_conn.update_many(
collection = self.AUTH_COLLECTION,
filter = {"_id": {"$in": token_ids}},
update = {"$set": {"batchId": batch_id, "batchTs": now}},
upsert = False
)
# Fetch the credentials to use for this batch:
tokens = await mongo_data_conn.find_many(
collection = self.AUTH_COLLECTION,
filter = {"batchId": batch_id},
limit = len(token_ids)
)
# Done here:
return [CoreAuthTokenModel(**token) for token in tokens]
async def release_token_from_batch_by_id(
self,
mongo_data_conn: AsyncMongo,
token_id: ObjectId | str,
sync_after_ts: datetime.datetime | None,
last_sync_ts: datetime.datetime | None = None
) -> bool:
"""
To release a particular auth-tokn from a sync'ing batch. This helps in gracefully completing the cycle.
:param mongo_data_conn: The database connection (MongoDB) to use to perform the action.
:param token_id: The ObjectId of the document in MongoDb that holds the auth-token.
:param sync_after_ts: The time (UTC) after which this same token can be picked again for sync'ing.
:param last_sync_ts: The time at which this latest sync'ing was done. Leave it to null for the current time,
else pass a custom timestamp.
:return: True if successful, else False.
"""
# Process the inputs:
if isinstance(last_sync_ts, datetime.datetime): date_time.to_timezone(last_sync_ts, date_time.TIMEZONE_UTC)
else: last_sync_ts = date_time.get_current_utc_date_time(as_string = False)
# release the auth-token from the batch:
return await mongo_data_conn.update_one(
collection = self.AUTH_COLLECTION,
filter = {"_id": ObjectId(token_id)},
update = {
"$set": {
"batchId": None,
"syncAfterTs": sync_after_ts,
"lastSyncTs": last_sync_ts
}
},
upsert = False
)
# ┏┓┳┓┳┳┳┓ ┳┓ ┓
# ┃ ┣┫┃┃┃┃ ━━ ┃┃┏┓┃┏┓╋┏┓
# ┗┛┛┗┗┛┻┛ ┻┛┗ ┗┗ ┗┗
controllers_v2/message/mail/all_mail.py
+9 -9
View File
@@ -58,7 +58,7 @@ from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
from models.message.mail.send import MailSendOneResult
# Mail Client(s):
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGMailClient
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGmailClient
# To work with datatypes:
from typing import List, Any
@@ -138,9 +138,9 @@ class AllMailController(MailController):
"""
# Prepare the combined base filter:
sms_filter = {}
for k, v in (base_filter or {}).items(): sms_filter[k] = v
sms_filter["serviceType"] = "sms"
mail_filter = {}
for k, v in (base_filter or {}).items(): mail_filter[k] = v
mail_filter["serviceType"] = "email"
# Invoke the parent's constructor:
MailController.__init__(
@@ -148,7 +148,7 @@ class AllMailController(MailController):
cache = cache,
alert_url = alert_url,
http_client = http_client,
base_filter = sms_filter,
base_filter = mail_filter,
debug = debug,
debug_prefix = debug_prefix,
debug_only_errors = debug_only_errors
@@ -169,7 +169,7 @@ class AllMailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
user_info: CoreUserInfoModel,
inbound_data: OAuthMailAuthorizationRequestData,
session_token: str
@@ -193,7 +193,7 @@ class AllMailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
request_url: str,
inbound_data: dict,
session_token: str = None
@@ -216,7 +216,7 @@ class AllMailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
http_client: httpx.AsyncClient,
auth_token: CoreAuthTokenModel,
force_refresh: bool = False,
@@ -257,7 +257,7 @@ class AllMailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
auth_token: CoreAuthTokenModel,
user_info: CoreUserInfoModel | None,
llm: CoreLLMController = None,
controllers_v2/message/mail/base.py
+6 -6
View File
@@ -58,7 +58,7 @@ from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
from models.message.mail.send import MailSendOneResult
# Mail Client(s):
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGMailClient
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGmailClient
# To work with datatypes:
from typing import List, Any
@@ -190,7 +190,7 @@ class MailController(CoreMessageController, ABC):
for k, v in (base_filter or {}).items(): sms_filter[k] = v
sms_filter["serviceType"] = "email"
# Invoke the parent's constructor:
# Invoke the parents' constructor:
CoreMessageController.__init__(
self,
cache = cache,
@@ -300,7 +300,7 @@ class MailController(CoreMessageController, ABC):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
user_info: CoreUserInfoModel,
inbound_data: OAuthMailAuthorizationRequestData,
session_token: str
@@ -325,7 +325,7 @@ class MailController(CoreMessageController, ABC):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
request_url: str,
inbound_data: dict,
session_token: str = None
@@ -349,7 +349,7 @@ class MailController(CoreMessageController, ABC):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
http_client: httpx.AsyncClient,
auth_token: CoreAuthTokenModel,
force_refresh: bool = False,
@@ -426,7 +426,7 @@ class MailController(CoreMessageController, ABC):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
auth_token: CoreAuthTokenModel,
user_info: CoreUserInfoModel | None,
llm: CoreLLMController = None,
controllers_v2/message/mail/gmail.py
+7 -6
View File
@@ -57,7 +57,7 @@ from models.message.mail.sync import MailSyncOneResult, MailSyncManyResults
from models.message.mail.send import MailSendOneResult
# Mail Client(s):
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGMailClient, SCOPES_GMAIL_MAIL_MANAGEMENT
from utils_v2.goog.controllers.gmail.gmail_client import AsyncGmailClient, SCOPES_GMAIL_MAIL_MANAGEMENT
from utils_v2.goog.models.auth_tokens import GoogleAuthTokens
# To work with datatypes:
@@ -163,7 +163,7 @@ class GmailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
user_info: CoreUserInfoModel,
inbound_data: OAuthMailAuthorizationRequestData,
session_token: str
@@ -230,7 +230,7 @@ class GmailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
request_url: str,
inbound_data: dict,
session_token: str = None
@@ -368,7 +368,7 @@ class GmailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
http_client: httpx.AsyncClient,
auth_token: CoreAuthTokenModel,
force_refresh: bool = False,
@@ -451,7 +451,7 @@ class GmailController(MailController):
mongo_data_conn: AsyncMongo,
user_info: CoreUserInfoModel,
auth_token: CoreAuthTokenModel,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
google_tokens: GoogleAuthTokens,
message_id: str,
llm: CoreLLMController = None,
@@ -495,6 +495,7 @@ class GmailController(MailController):
f"Gmail message '{message_id}' already "
f"sync'd on {mail_records[0].syncTs} (UTC)."
)
print(sync_result)
return sync_result
# Now that we know that we have to fetch the mail from GMail:
@@ -561,7 +562,7 @@ class GmailController(MailController):
self,
sql_conn: AsyncMySQL,
mongo_data_conn: AsyncMongo,
mail_client: AsyncGMailClient,
mail_client: AsyncGmailClient,
auth_token: CoreAuthTokenModel,
user_info: CoreUserInfoModel | None,
llm: CoreLLMController = None,