Squashed 'utils_v2/' changes from ef9630d..271426c

271426c (20241010) Can now cache class methods by accessing a caching object from the class itself!
606b02a Merge commit 'b0e0760fc7bb6336debf621860d1286575a0b8cc'
3d2a723 (20241009) Work in progress.
58266e3 (20241009) 'last decorator' bug-fixed
53e7a39 (20241009) Work in progress.
dbfc17f (20241009) 'last decorator' bug-fixed
11875e6 (20241008) logging decorator and model improved.

git-subtree-dir: utils_v2
git-subtree-split: 271426c19bea8de5edec7ecb79ee04368c47b02f
This commit is contained in:
2024-10-10 11:44:01 +05:30
parent 8494a2c0c0
commit 0450b2a445
17 changed files with 218 additions and 257 deletions
Binary file not shown.
+31 -30
View File
@@ -1691,42 +1691,43 @@ if __name__ == "__main__":
async def main():
# Create an instance of the database connector:
my_fs = AsyncMongoStorage(
connection_string = constants.MONGO_FILE_CONNECTION_STRING,
database_name = constants.MONGO_FILE_DATABASE_NAME,
my_db = AsyncMongo(
connection_string = constants.MONGO_DATA_CONNECTION_STRING,
database_name = constants.MONGO_DATA_DATABASE_NAME,
max_connections = 10,
debug = True
)
# Connect to the database:
await my_fs.connect()
await my_db.connect()
# Keep performing the changes in batches till you have corrections to make:
while True:
# # Get the documents to migrate:
# documents = await my_db.find_many(
# collection = "scriptData",
# filter = {},
# limit = 50,
# projection = {"_id": False}
# )
# # print(json.to_string(documents, default = str))
#
# # Adjust them:
# adjusted_documents = []
# for document in documents:
# script_id = document.pop("scriptId")
# adjusted_document = {
# "scriptId": script_id,
# "desc": "no desc",
# "content": document
# }
# adjusted_documents.append(adjusted_document)
# print(json.to_string(adjusted_documents, default = str))
#
# # Insert the adjusted ones to the new collection:
# response = await my_db.insert_many(
# collection = "_scriptData",
# documents = adjusted_documents
# )
# print("RESPONSE:", response)
# Find all the files that have their metadata as a string:
files = await my_fs.find_many(
filter = {
"metadata": {"$type": "string"}
},
limit = 10
)
print(my_fs.to_json_string(files))
break
# # If no matches were found:
# if not files: break
#
# # Fix the metadata file-by-file:
# for file in files:
# file_id = file["_id"]
# success = await my_fs.replace_metadata_for_one(
# filter = {"_id": file_id},
# replacement = json.from_string(file["metadata"])
# )
# print(file_id, ":", success)
print("Fixes done!")
asyncio.run(main())
+21 -11
View File
@@ -96,7 +96,8 @@ class AsyncMySQL:
"""
A class to work with SQL-based databases. Originally meant to only invoke stored procedures and retrieve them as
JSON-like structures (list or dict).
JSON-like structures (list or dict). The format for the results was very specific to our use case for serving
Bicree's requirement. This may not serve your requirement at all.
:param pool_size: The number of connections to maintain n a pool.
:param args: Any arguments to pass. Not used.
:param kwargs: Pass the connection configuration from here.
@@ -207,8 +208,6 @@ class AsyncMySQL:
await asyncio.sleep(backoff_seconds)
backoff_seconds = backoff_seconds * backoff_multiplier
print("LEN:", len(results))
# If the results are blank:
if len(results) == 0: return {
"status": results[0][0]["status"],
@@ -238,9 +237,6 @@ class AsyncMySQL:
formatted_results["seconds"] = time.perf_counter() - start_ts
# Done here:
print(f"{procedure_name}:")
print(json.to_string(formatted_results))
print("\n")
if return_exception: return formatted_results, exception
else: return formatted_results
@@ -274,12 +270,20 @@ if __name__ == "__main__":
:return: None.
"""
# cred_json = {
# "host": "del.ditscentre.in",
# "user": "bicree",
# "port": 3306,
# "password": "9c3b2808a4aa281129d399fe09e69b53",
# "database": "bicree"
# }
cred_json = {
"host": "del.ditscentre.in",
"user": "bicree",
"user": "caOffice",
"port": 3306,
"password": "9c3b2808a4aa281129d399fe09e69b53",
"database": "bicree"
"password": "jstArchon",
"database": "caOffice"
}
db_conn = AsyncMySQL(
@@ -299,10 +303,16 @@ if __name__ == "__main__":
# )
# )
# result = await db_conn.call_procedure_and_get_json(
# procedure_name = "listSummary",
# procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",)
# )
result = await db_conn.call_procedure_and_get_json(
procedure_name = "listSummary",
procedure_args = ("bd7a6e53-1345-11ef-940c-0cc47a84a0bb",)
procedure_name = "campaign_activity_report",
procedure_args = (10000000,)
)
print("RESULT:", json.to_string(result, default = str))
start_time = time.time()