From 0450b2a445509ef536ccf8598ab31a0e4ba09f18 Mon Sep 17 00:00:00 2001 From: khushal Date: Thu, 10 Oct 2024 11:44:01 +0530 Subject: [PATCH] Squashed 'utils_v2/' changes from ef9630d..271426c 271426c (20241010) Can now cache class methods by accessing a caching object from the class itself! 606b02a Merge commit 'b0e0760fc7bb6336debf621860d1286575a0b8cc' 3d2a723 (20241009) Work in progress. 58266e3 (20241009) 'last decorator' bug-fixed 53e7a39 (20241009) Work in progress. dbfc17f (20241009) 'last decorator' bug-fixed 11875e6 (20241008) logging decorator and model improved. git-subtree-dir: utils_v2 git-subtree-split: 271426c19bea8de5edec7ecb79ee04368c47b02f --- ai/image_classification/async_blur.py | 4 +- ai/image_classification/async_nsfw.py | 2 +- .../async_dichotomous_segmentation.py | 4 +- ai/object_detection/yolo.py | 4 +- api/__pycache__/async_quart.cpython-310.pyc | Bin 23924 -> 23986 bytes .../metrics_prometheus.cpython-310.pyc | Bin 3424 -> 5163 bytes api/async_quart.py | 143 +++++---------- api/{metrics_prometheus.py => log.py} | 163 ++++++------------ api/response.py | 12 ++ cache/__pycache__/__init__.cpython-310.pyc | Bin 0 -> 164 bytes .../async_redis_cache.cpython-310.pyc | Bin 0 -> 13364 bytes cache/async_redis_cache.py | 46 +++++ .../async_mysql_v2.cpython-310.pyc | Bin 0 -> 5474 bytes database/async_mongo_v2.py | 61 +++---- database/async_mysql_v2.py | 32 ++-- image/scanner/scanner.py | 2 +- image/scanner/scanner_v2.py | 2 +- 17 files changed, 218 insertions(+), 257 deletions(-) rename api/{metrics_prometheus.py => log.py} (51%) create mode 100644 cache/__pycache__/__init__.cpython-310.pyc create mode 100644 cache/__pycache__/async_redis_cache.cpython-310.pyc create mode 100644 database/__pycache__/async_mysql_v2.cpython-310.pyc diff --git a/ai/image_classification/async_blur.py b/ai/image_classification/async_blur.py index c921a4e..f69af42 100644 --- a/ai/image_classification/async_blur.py +++ b/ai/image_classification/async_blur.py @@ -11,7 +11,7 @@ OBJECTIVE: To provide an easy way to assess images for blurriness. - Tried and implemented using HuggingFace models. + Tried and implemented using HuggingFace behaviour_models. REFERENCES: @@ -193,7 +193,7 @@ if __name__ == "__main__": async def main(): image = r"/home/developer/Downloads/low-res-check.png" - assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/image_classification/BlurOrBokeh") + assessor = AssessImageBlur(model = r"/home/developer/PycharmProjects/utils/data/ai/behaviour_models/hugging_face/image_classification/BlurOrBokeh") usable = await assessor.is_ok(image) classes = await assessor.classify(image) print("IS OKAY:", usable) diff --git a/ai/image_classification/async_nsfw.py b/ai/image_classification/async_nsfw.py index c59189b..446c34a 100644 --- a/ai/image_classification/async_nsfw.py +++ b/ai/image_classification/async_nsfw.py @@ -11,7 +11,7 @@ OBJECTIVE: To provide an easy way to assess images for adult content. - Tried and implemented using HuggingFace models. + Tried and implemented using HuggingFace behaviour_models. REFERENCES: diff --git a/ai/image_segmentation/async_dichotomous_segmentation.py b/ai/image_segmentation/async_dichotomous_segmentation.py index 04ec460..fb6d6c7 100644 --- a/ai/image_segmentation/async_dichotomous_segmentation.py +++ b/ai/image_segmentation/async_dichotomous_segmentation.py @@ -13,12 +13,12 @@ To provide an easy way to get masks from dichotomous image segmentation. This code uses a very specific model from HuggingFace: "ZhengPeng7/BiRefNet-portrait". - You may experiment with other models too, but make sure that model is made for "dichotomous" behaviour. This + You may experiment with other behaviour_models too, but make sure that model is made for "dichotomous" behaviour. This means that the model should have only two classes like "foreground", and "background". The specified model was trained for implementing portrait mode style blurring of backgrounds. The originally tested model has an MIT license as per their GitHub page. The code in this file may or may not - support drop-in replacement for other models, please be aware about this. + support drop-in replacement for other behaviour_models, please be aware about this. REFERENCES: diff --git a/ai/object_detection/yolo.py b/ai/object_detection/yolo.py index 91747f2..a435ca6 100644 --- a/ai/object_detection/yolo.py +++ b/ai/object_detection/yolo.py @@ -10,7 +10,7 @@ OBJECTIVE: - To provide a class to detect objects in images using YOLO models. + To provide a class to detect objects in images using YOLO behaviour_models. REFERENCES: @@ -166,7 +166,7 @@ class YoloDetect: if __name__ == "__main__": - # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "models", "yolov8x.pt") + # model_file_path = os.path.join(constants.PROJECT_DIRECTORY, "ai", "yolo", "behaviour_models", "yolov8x.pt") model_file_path = r"/home/developer/PycharmProjects/utils/data/ai/models/hugging_face/object_detection/YOLOv10-Document-Layout-Analysis/yolov10x_best.pt" # sample_image_path = r"/home/developer/Downloads/2_cats.jpg" sample_image_path = r"/home/developer/Downloads/flattened_image.jpg" diff --git a/api/__pycache__/async_quart.cpython-310.pyc b/api/__pycache__/async_quart.cpython-310.pyc index e9486874e7560511e1f288c275407e0121c12002..66e8ec4d1a95ef8947770e285afd76cb2ddd5257 100644 GIT binary patch delta 4015 zcmZ`+Yj7LY71rIAWXZB@$?-e3B0G+4MSdrKoA{MDP7Jp5bfTmq6yLS2SeE3gRT3O$ z5^4`El#rx%XelYhfi_?|^rb#1bov8mfu@~lrkw$nwm=7HQf8o(PG>rlw&&cnExApl z*>8`|>z;GZId@lodzO6fIg%N%+tUs3`D^t*#|IWKWfqg7OWv6xVxWYkhjM&5;-2fv z756+}-leAwomC1J%dQxF8S|aC!0#zRf!gG=tg~4Aic8 z=Vf0t&7@fi244+zP$%4LX|^JbOLQI0SupzQtm(9y<|_5T$)ouTCSLxt zxNo67w1m1Ai0|xHx`DcZ)nv6_kiN8QFD*qO#yvjDN}a zs5uL^o@Mb4(|4=01-TZS_%jpWZn$Db0E;1mu*^h(xl^*~#;D@teoHqgPb^yAC8UJk zlV0xFg#1*EXVdG=HOMSU{3ZQ2WKt+pBBKhy1E3jWiZaD2kjxm1gjFT1`qjIlipR`K zPzh}`doX+~5)V`Vcrc{I*k%+|qri#`e=InwtXbG=?;tIS3HwPY*NwWR2&s{MoOPNM z@ra|?j;VS~`dY30AxG_+9IrdtNPoiSlu2{aJW1hZX<1`|cuestr^b}18jOUk1U60r ziN(SO65F%$r3$oIBAOginMYzBXk`X{zj|yEp9}iJRO^0BS66}AcL(#7~~)3 z^sEW5BiBnt6BD`XbnncubGP$*W5y3*+eJ1i%sz8gLUVCx5!6w-{qsHf6J@DS|qMl#|*{p|WR&OD_ytlfE4DwUe4V6*--; z(1}=J7DL#d2Y;AMB zNZ+i>lB&W|_(<-?V`%Q=Z`GEP#}iWBLYggQH~6LI4OI`I26i612Vgcm98uk&$i##~ zZ@vr+=&ds-bn(}k8w-$}wH>OtJu*?#HyH?qH5qi-6TGaY(b0u^^+Fk86Q?Z$^>s)# z0Dz-aTJNZK6k#eli1K#w-?aqmun(a*^nC+bvtTj$uoe>ghG5XhSbQ4JK#v=Wqqg>w z%lx_460exM)od6RL^G-JXh>m$KDPR&BXn7uu@G0sc(e zKH}yX?Jc#qus(WEBDI8Y72yelrw~LoB9Rk`nRbaZiOQz2*HI|!|F3Ckji2o3Y)4lf zONwQ3siY6FXtLTGF5~2`&PwtI97rXW5IT_qzu?!4T#0)-Um;|gpYP4*$G4q#r8EL+ z5z+gwqL}V$Jh#i!g4-uLd&!v6)UP5}1%MG1iHYMVC3PO}n$8!Q6I~%Sw|{i)6o=M| zaWeWh(XW#q?QSG*WAr4Hc)I&dJ>+j~SsU)G`B;FugHd(~j6503L8Aj1L7Yq7Jlxl^ zVKoQ#dzAbEAvM+)`=YrRD-+A7M!IWzXRgqEYt?2s#D}+6l6Ml%Z~v{CL=s;f^h#Z8 zql4$!+(23e6M>P6t(X1D%PHdLJoq4%KrqR78e(@lZvnn8!mPn$2JVRdEZe zcK58&9SpmnN;mk}P($%ILE+}V;p5q$=br5D7x;IE4v}lzI$TF6ZyRne!=ccOp&&d} zpGQ)T!D|lg3v8U;_58u%z9y`zW*>w12UTJI6M;}%5oKmyL{+Pn9SDUYC;h?k+O+_l<;vY^VE{$?CEF~Etm8z z787>DgYYUGCL7C!{+;}Ddp5#*9p10Xi~@83TkOl7Q}rJMHFKP+^TmS6*#@wsQwR zxUbX5rVR#uWzh2(C;`cpn8PM)=Ol zx;tVB+ZvW#$3AnC5CeV>!!<`JDKt=dN@6psYt86uoss9x%KP9!JH*x&%_D2JKs4%~ z*5Be8p7WYT3Dal%bWr+P{Culg8c!n!P6&L;XV@2^9MHBfeK{KSjZ(2$lSS zBd+1EAo&r(-w{4R*o$xuKr_Pu1TR`F3uCZNDC%S+riKI43i~;VO>sfLjQVv5FY!-~ ztUvo%Byq09zb*akvIc*)AV9AeSU*w&2%`v72&WP5M|d0o>#7-#MIs^g2L65<;T>ek t&>Na*JQNA2%#Pmj5OANwzX@l`D~f)S@vBEGNCyAa(Z@{9X36-k{{TbenHT^7 delta 4127 zcmZ`+d2kz78Q-@n$&zf#vLna0BOkG?$VXy3z7yX`OcFV9>>O%}Q2f@85=(NP);V0~ zfS6L~nLG+Jr9+)kLLC|g&R-uJ!ld++Uj|2p~L8ga!lGo2Fr-f;YLV&4lFT*aj7g8!H+>Ir(Hxxw7%>fmZ3 zUHHNNvy@QhRC%ym+$(|=;$9i7RErkYUXp@UYVkrf+-uar1s_enM1t$oiX(owYV)LE z-ByWa(9C&>W~%k)m0$yP(X4qXxSqOcHrzMR995o>=_=}(w*(s?zupLkbY4=h28N#xYn`NuK)uOK& z9Ukr75gHlXJvO*!G}J#l78uoYb`0$u{10B%lFE+W(ZN9f0o}P%)mUUa(YtGi@3OpL z%Yg#29R8u@d$qZOTnQe0%m(-@TnP(+UCO{fer@fN(+y)(@$dqB52;QD?SCP}%X^%a z?i%EqH8kgX+j?YrlUJO-A!micI%N0}Y5{ahLRF74ACej4@tCH@w2*dcTJ_nO7nRUP z&y2(l#b;tPG!dCn6RZve4JdFRBb12Ds^)=Fk|R_@6D%GLG4x09jdO}9dFQ|2kk!{5sGlI<6} zJjt9KMXo~0BGHwE#(Xl{3{SdEJvkoI4y_oy(=$@M5rs-LG0FB|1U7~Ywi+S87d^cz z!uZhRC&6S{?kc5Y2Wp52cB0*bQ7jQnjhe?mtz7_J1c=>0%r^6^LN9+RzmRN9F6Qqh z?uSvwoQtz?JBeNFDo(OOpIknG;v$3$gaiOoA8YheZ@E^M2@EHG=hlY5u`l)W=T0YlYFlv$5M3+Igo(F z7=WI&)EtM^FjZNCm7%DOd&?uM#9WimihQ0gkJ&^PL)=|aP*;uG9SD=?&C0^DN!660 zhN2k&pTbV^y%lSX5{Pwjia%QMnc`UxF3Ri)RD2R)4nZiK=5l3Yw@6GeWI`*I-BU}d zx|2cpp-60cMmqt?iKXYtvaDrKCli(TZ0?m=vvWM=_cs`+f{$58G?GX}Vv`+SsKG__ z$^mr8VNIJ3jmN3#%V7ijb$=5XYD3Y){CjL8M2hBuv|d>)J%R8A3glJ`Z97k`Q!Tev~<&q@$D^Z zYA&FL=&mmUHS3JUHSbh>a#E#tUk3)X-Z|7L;}=^R^O2lQ-(R;QK3Ug)I2?)T3h1&c z{LPj|cOU9C31x&DUfQ~CeIt_10N`kuHac$;iZB%eL~C^Oue3%Q+E8CfO{8weg7ss) zWcD%$bvuoZ&qQHc^m(Cx^=(^*zK)K>>ci7m&m0SE4-D@Mga$`OhDVGNE@#QGn7UP) znVwQv01ah?A0qsS-)P%wD?u{I3);QzRK<v6>?X_9P$?f`&{P+^g~NFG zA3Ceaor_&1B%Unox=+Y7&)ob{si`5*H4*CsMi9HfZT_pxzE+$rF@=j3(*n?ui+v8u zTqGiPEmNwzp*xxzRnl+mL{L* zxn=~O$3NZo)uxrf&)N=!sW&pszKkXr4^lvP3W6wdKd>heOhj1T3Y-LUS`p!Z!!hg53s!S}`hd?x&!NQvA6;>VIS&KIk^Fs0N z@shxX;_raM-A~}>TVqT@s>yHhBZ2+oHh(S9KqmP=0v+XeZt0e(2pq&WkyH}!ZiEvA zGiWqFA0F;6#B%GI(ONY*^<&Rt*X5DjBz83YtWTV*08eEctSb8T+#>TDAxDeq!1VJAEYufk!f-(2Y5 z!+UpE$Zo>#mJ~VIS%?l=?y_jPmv`%O=gJD|Y11r%^B{Ik*p$T9auKZ_bY1FYK;t(? z{V?HoN4L?@0Wan*2(QXz~%|Ym#PxyY-0eJfW5+7abtGAD+6fi|51 zVzt~t-p>$})}Q7lt`1Mep6M2SiuE?dZKUsMezW|+!fJ6jF%zg4G!<_~0qdsc@ zfAGTdE#L$KqoV;|8ha0Dc2_V0TLL~A6t?j9gTF6yp&$!E%;Zmzf$6>%>$% ltT8uw!!;zPO?-G{o8DX=%R>wy*#I diff --git a/api/__pycache__/metrics_prometheus.cpython-310.pyc b/api/__pycache__/metrics_prometheus.cpython-310.pyc index 29709178bcf58cf001a95fe407d73c1a69863c0a..724908a216d73a378997428f0066c51893a99d87 100644 GIT binary patch literal 5163 zcmb7I+jARN8Q+^$D_OoKah$ZkCbXrZMNUKLv}p)&94B#`$i`O4pc!Uov*$!wN7Ab2 ztejZmGG&}c!VIr4ov}N->zY@$k+M4CC)K82wxfuAxZ7Gz@0c4Q4W{W!6n@Zdv;_vky$u81xvgB@=r&i+ zq0P12cD|l(dv&i}s2AGBda*rGpJS0!)Lcnve_#JKgH(IxAy0= znKQ2SZ>3M1}66iM5E z+~k7$l5cYXgxWk6aU}f^b8Q~Vp5V;y$+*+-G1~Fd1S9T8Nhjq`QokK`!Uh+zl7Zb= zxw*2jQe9rz9Kp4^xHNdf+WqRCwWS;1xL94Qjh(viUd699xqtqAvG^59Syh>pzE!D| zU6o%>dVnufezVtZhhj(NZiT%DSF_7WtHq-<5gWV_%M>f6cCVGjU6DjwN;Tc!9lSH; zK?{{k)#QT_Z4dE!1WTO+GNtbJE-%wQic< z_jZjfm!v-}kaj^s?Jsd=%Vo|p2YZU?1mT1E{MdXcon#&>yfQz9d!W~>)EK>Gni;@+ zvMLI|NsuiP6R66Ln5O;=RkNs+Cx+aAy_u%nK=4m{V7BIYC)x=lhx{_xf8!2>029RE z+qK%dKiDY!xC4#Qn=0Uu3$17-OnJP+x=Gwg)r7uRf?YkSPK*MPnf&xm)<}%TudS$s z@vW^xdF#Iq^^BT4dgR9s{jlRNt*`n+aDFTYGDdO)g%ok4nfjgNaYf7%1!mNl;E0_; zFeZHCOJkDg$|QQk@tg)dP^MZIPmNtQoE9VzYJ6=-W|7xxcS+OMGfg{GM3JMlaU3#* z3nJ2DLDHtCkFiLQl&ce36%W~`_p3QgwBdd|S9X*mdF!Dl;gp!nE(~MI1O6m}X@p8W zH^lca|Ml8pGimch26NX+x?C)-??kY-ZJ_cy3n9ruYuV&rR_r8vWR;dYgtPPG2588|R{#+Y zG$0~tN}x<;J|pX4RRM?YEL*}uqd|*zs7F*D0Zgk0C>n6d?4KQZ;W1kr5pKheV3Xu% z_&3p^hwhU#fEC)ZQJe$&Giw(ZJ#xhSuC<@rHMRc*IrOX{&SBO)n$_0BstTfCIWNwk zQLZ+$Qk=y=yhhcxY1)oEk+K<&%C;t_%58_O9#?rSZd@FDr7G$9fg||?Zje)`T#ML> z-^=;jS+jq7E@ zAY;HqER0~Kxu7{ehlMVRq>g7;4vAs$h4qdWQCrY-gfjLK^{=7G3#f)ZhQY5CEc|TP zX80HQ8HeIA_>>%3aqVLe{gfx*Th>%%KkbDgeb;}`OM$5`n@O+5e6n8t<2Y?bvxsRAc$0kK)?IA|lr6|I#enobmKfOGVzReV@<_yB?q%nnT zBp{iKLZCTLPWXM12C39h9>tY`J_09qh<^}7-Ci))`tvn-EF#nBoTcm~WQl)j^L+TMN#d%aJ zpFGe(lDI}wOH^H_>Iy1Eif^Jp9QPy&(w#0>1?pwCy^spzL=VE~(J_%#1tRVf^6~$` zX-QzL&pm72@dgIodg+ez=g^a(Chr zs*ZH$$e@$h{}Dw}yz3c`1(QrB{a@bOrAa}n}JHAN^lZPulJ-9tAmS zVSQum?n>?U%Dv6t?!7y;Rg9N|8><`r!p6!ie|@QT`!X`IENfNn^=8s-#Rv=zgRuxG z9DbyA6u)bU+(w5aGSz7XGaFD;W>JbkiA)0$`FrCFi{+l#myA{XrEuH$-TY@p>g?yR z$2x$tWwWMnAFB?2d-iaKjOBZSQ}WVd$Tt0cj-mqrx2o)0E4BVvvYztt;$nNJ!r*k` zP9;j(I^ffhf|gH&0atvPKjSIs$d>ZRq(+m(Nc`#ACnEh&7~w5*|Xq}|M7 z_EXT6IeX>}q=BHCMPbu#1{6W62CXD5MeAm5ij)p1TmG53pFhBv0j0cZ|MX^tr~VsY zkORp|r7{*WT0p=?*+kJ#Xuu}?BKH~iyJ!5?`V^soem{hgUQ)J{Rk@(d;5bo=S6WEj z!G!MB4~ZId@0`(J(7d??~~3T%|x(xrp}Q3Bjyf8p=oVI93jUf1mK7(2n$f&YUql)3X*#5fH4` z#2vcGmOD~8T>w!o+#(p8x#erLID4d5LDY(2-gTy5{QyM{4y1%yZ$;w8s zymqfz)4^IMcw|77l}P<2u|cCe?`+2+>10pNhYutnkqIslTw|$h(d&%7?a;0$?@AVP zUL_6HiHu`-Za^t{R7zacUf;BW$_|2w)nceop?#~Mc2XqaWQVU;b6xWNo&RuQxA zV-~-omE+uGHroZ1#8Zk5JR>Z*gR+ZTEXC3+Qys`877@#`+-<~io1{F#lWg=F5p{+c zsi(@RXib;X3?E@*pg^Xa!HA8siQA~0Ws__Q)*MguMtN=%^)cL$)o41??(Bmj_yGQ3 zOW9PEINu@pyrW5T!D}}IE~L5IZZuqROU4%6c9qMChEH#J;u;q|bt^&bI;Vjbxb;r> zCpm_l@OAA>_&|SzQ{iv%3-&o4h+4%zd4B1^+cw5sT|M)3s6BzX0MiH8qIJBEC|W0g zm?1#2t^sNcts(~(9bF`WOQZl~d@TrCwBj)?XXy4TlnS@zbINa4cq^!RO-Gly&+AtP zqEnG<>N%cokeCdAGA`||DNQT@BW_>=+qje4{&m=`A1bc7@NQzN8^n6XI=X81 zanE8{d3IgtN{U+;n4GxOgIa^jM2m|npzOe-W6$X82d)Tc-~*HJD%|&JBr^33Q>E3W zL&ogcrH>1ArSQ>(PYbIhy0CnqSdvEFz0B+Wz&Tp2S#hu!2OY%Z5T;-!ZAx9P0!!%) zIQuHO$%d`jIIHPeXKaVPUysaoFTwQ}I6yrJ%~b+sY6CL^J)*=E6{gYGdS)MC6j(3N ztHS zYXPO3z~w7`vH@-(I#wTfV(8oZte(_5qucJv4}~4ASUlyJA&u|vN|RFNRUj>=)$98e z%qiG@I@+n&h#Y_m(@z6fNGHT1m|*)~E15_{x6-1t;ntdzI#{Nvyvcrla47%>4@8BP@1@yor_&L#NS~6Q83%>*>ts z;{gS_h6&Xe3-6_0>ArOXa$S8FVMg_76{WUVC`tYN@@lCwt;*9sIzQjIm1mw(IbZP_ z^KPp)58hYGMM2+E^}nS7IApuHt8j+#*ZW()!r~YM(+%M^;aAeAta;&VCO0Eg_`wfm zW6)%F_;e~<&WykN9o+j_0J1?y0%vjZKcb~IOUpkN?*aJ*hbcuKDxqT7%PgcfQ7_g< zo2p-`sy4&FGiMH~a6xS;d51`nKO`~T`7+R@eXL%CczBK0@gGsx&hG1O{R8w)>_e}~ z_jIgth0@B>!s=1Skg=AiH3Jy~a>m3VGy%WW{9C@%TkvmbxUCj%GBFdL$jx?T;*;f-Gli8w zClRlKka$%=RE;Vr#X$v1Iq{AHrCH>G=mn{Xc5{bb!-SUBv4+Rmb$(n0+&=}NBWrI0 sC$OcBY9yhu+js*TPX>t&J@*NTMa1Dwhh1{gI&Be4i4kigZR?Nz2cc%fJ^%m! diff --git a/api/async_quart.py b/api/async_quart.py index 0c29f69..8749a76 100644 --- a/api/async_quart.py +++ b/api/async_quart.py @@ -46,13 +46,8 @@ from utils_v2.string import json from utils_v2.date_time import date_time from utils_v2.security import sanitizers from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.api.log import APILogModel from utils_v2.api.response import ResponseModel -from utils_v2.api.metrics_prometheus import ( - TOTAL_REQUEST_COUNT, - LIVE_REQUEST_COUNT, - REQUEST_LATENCY, - MetricsAPI -) # To work with date and time: import time @@ -60,11 +55,9 @@ import datetime # System-level activities: import io -import distro -import socket -import platform +import os -# For Pydantic data-models: +# For Pydantic data-behaviour_models: import pydantic # For hashing and shortening the hash: @@ -90,11 +83,6 @@ import asyncio # ***************************************************************************************************************** -# Info for logging that will stay constant during runtime: -SERVER_HOSTNAME = socket.gethostname() -PLATFORM_INFO = platform.uname() -HOST_OS = distro.name(True) - # Chars to choose from for random strings: ALPHANUMERIC_CHARS = string.ascii_letters + string.digits @@ -359,7 +347,7 @@ def set_api_version(api_version): # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -417,7 +405,7 @@ def read_input( # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -556,7 +544,7 @@ def validate_input( # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -572,6 +560,8 @@ def validate_input( def log_request_to_mongo( attr_name, collection: str = "logs", + api_version: str = None, + project: str = None, log_type: str = None, operation: str = None, log_input: bool = True, @@ -586,6 +576,8 @@ def log_request_to_mongo( :param attr_name: The name of the variable that holds the instance of 'AsyncMongo'. It should be accessible in the scope of 'current_app'. :param collection: The name of the collection to write the log into. + :param api_version: The version code of the API endpoint that is being logged. + :param project: The name of the project that the endpoint was built for. :param log_type: A hint to identify what the log was for. :param operation: A hint to identify what was action was being performed. :param log_input: Whether, or not, you would like to log the input that came in. @@ -602,6 +594,9 @@ def log_request_to_mongo( # Let the next in-line decorator know that it has been wrapped: kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 + # Set the api version as needed: + kwargs["api_version"] = kwargs.get("api_version", api_version) + # Make variables and extract available info.: exception = None response = None @@ -635,7 +630,8 @@ def log_request_to_mongo( elif isinstance(response, tuple): response_to_log, http_code_to_log = response else: response_to_log, http_code_to_log = str(response), 200 - # Try to get the information about the request: + # Try to get the information about the request. + # There will be no data in any of these if the decorator was used to catch start-up and shut-down events. request_method = None request_url = None request_route = None @@ -653,46 +649,42 @@ def log_request_to_mongo( except: pass # Construct the log: - # for k in sensitive_keys: kwargs.get("inbound_headers", {}).pop(k, None) - # for k in sensitive_keys: kwargs.get("inbound_data", {}).pop(k, None) - log_json = { - "hostname": SERVER_HOSTNAME, - "os": f"{HOST_OS}", - "cpu": f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})", - "logId": kwargs.get("log_id"), - "logChain": kwargs.get("inbound_headers", {}).get("X-Log-Chain"), - "log": log_type, - "operation": operation, - "apiVer": kwargs.get("api_version"), - "method": request_method, - "url": request_url, - "route": request_route, - "ts": request_ts, - "tat": time.perf_counter() - start_ts, - "cpuTime": time.process_time() - cpu_start_ts, - "headers": kwargs.get("inbound_headers"), - "data": kwargs.get("inbound_data") if log_input else "not logged", - "files": { + api_log = APILogModel( + project = project, + log = log_type, + operation = operation, + apiVer = kwargs.get("api_version"), + logId = kwargs.get("log_id"), + logChain = kwargs.get("inbound_headers", {}).get("X-Log-Chain"), + method = request_method, + url = request_url, + route = request_route, + ts = request_ts, + tat = time.perf_counter() - start_ts, + cpuTime = time.process_time() - cpu_start_ts, + headers = kwargs.get("inbound_headers"), + data = kwargs.get("inbound_data") if log_input else "not logged", + files = { k: { "name": v["name"], "size": v["size"] } for k, v in kwargs.get("inbound_files", {}).items() }, - "exception": None if exception is None else describe_exception(exception), - "response": response_to_log, - "httpCode": http_code_to_log - } + exception = None if exception is None else describe_exception(exception), + response = response_to_log, + httpCode = http_code_to_log + ) # Write the log: app_attr = getattr(current_app, attr_name) inserted_id = await app_attr.insert_one( collection = collection, - document = log_json + document = api_log.model_dump() ) # Return the response from the wrapped function. if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -709,7 +701,9 @@ def should_not_be_under_maintenance(attr_name): """ Use this decorator to reject a request when the app is being marked as "under-maintenance". You will need to create - a boolean variable within the scope of the 'current_app' for this to work. + a boolean variable within the scope of the 'current_app' for this to work. An alternate to this is to set the value + in an environment variable named 'IS_UNDER_MAINTENANCE' to a string value of either 'True' or 'False' for + multi-worker deployments. :param attr_name: The name of the boolean variable that will hold the information about the app being under maintenance. If its value is True at the time of checking, the incoming request will be rejected. :return: The decorator factory. @@ -726,14 +720,16 @@ def should_not_be_under_maintenance(attr_name): # Get the attribute and check if it indicates that the app is under maintenance, # call the wrapped function if not under maintenance: app_attr = getattr(current_app, attr_name) - if app_attr: response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() + env_attr = True if os.environ.get("IS_UNDER_MAINTENANCE", "False").lower() == "true" else False + if app_attr or env_attr: + response = ResponseModel(status_code = StatusCodes.DOWN_FOR_MAINTENANCE).for_quart() else: response = await func(*args, **kwargs) kwargs["decorator_count"] -= 1 # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -780,7 +776,7 @@ def only_whitelisted_ips(attr_name): # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -888,7 +884,7 @@ def limit_rate( # Return the response from the function call: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -927,7 +923,7 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): # Done here: if ( - kwargs["decorator_count"] == 1 and + kwargs["decorator_count"] == 0 and isinstance(response, ResponseModel) ): response = response.for_quart() return response @@ -950,51 +946,6 @@ def handle_cancelled_request(cleanup_func = None, cleanup_coro = None): return decorator -# --------------------------------------------------------------------------------------------------------------------- - - -def measure_metrics_for_prometheus(): - - """ - Use this decorator to automatically measure metrics for using in Prometheus. - :return: The decorator factory. - """ - - def decorator(func): - - @wraps(func) - async def wrapper(*args, **kwargs): - - # Let the next in-line decorator know that it has been wrapped: - kwargs["decorator_count"] = kwargs.get("decorator_count", 0) + 1 - - # We use the context of the measurement class: - async with MetricsAPI( - method = f"{request.method}", - endpoint = str(request.url_rule.rule) - ) as metrics: - - # Invoke the wrapped function: - response = await func(*args, **kwargs) - kwargs["decorator_count"] -= 1 - - # Interpret the HTTP code: - if isinstance(response, ResponseModel): _, metrics.http_code = response.for_quart() - elif isinstance(response, tuple): metrics.http_code = response[1] - else: metrics.http_code = 200 - - # Done here: - if ( - kwargs["decorator_count"] == 1 and - isinstance(response, ResponseModel) - ): response = response.for_quart() - return response - - return wrapper - - return decorator - - # ***************************************************************************************************************** # ***** **** # *** MAIN PROGRAM *** diff --git a/api/metrics_prometheus.py b/api/log.py similarity index 51% rename from api/metrics_prometheus.py rename to api/log.py index f7e9e3e..9f5acef 100644 --- a/api/metrics_prometheus.py +++ b/api/log.py @@ -3,15 +3,14 @@ AUTHOR: Khushal P Soonderji - Sharvil J Daiya DATE: - Saturday, 28th Sept., 2024 + Thursday, 12th Sept., 2024 OBJECTIVE: - To have one place from where several metrics are measured using easy to use context managers. + To have a structure to the response sent from the API calls. REFERENCES: @@ -31,16 +30,21 @@ # ***************************************************************************************************************** -# To make sibling directories accessible for imports: -import sys -sys.path.append(".") -sys.path.append("..") +# System-level activities: +import distro +import socket +import platform -# To measure time: -import time +# For data-modelling: +from pydantic import BaseModel, Field +from typing import Any, Optional, List, Literal -# To capture metrics for Prometheus: -from prometheus_client import Counter, Summary, Gauge +# To work with date and time: +import datetime + +# My utils: +from utils_v2.api.codes import StatusCodes, HttpCodes +from utils_v2.date_time import date_time # ***************************************************************************************************************** @@ -50,31 +54,11 @@ from prometheus_client import Counter, Summary, Gauge # ***************************************************************************************************************** -# Define the Prometheus metrics. -# FOR THE MICROSERVICE AS A WHOLE: -WORKER_COUNT = Counter( - name = "ms_workers_active_total", - documentation = "The number of threads for the microservice being monitored.", - labelnames = ["project_name", "service_name", "host_name"] -) - -# Define the Prometheus metrics. -# FOR INDIVIDUAL API ENDPOINTS: -REQUEST_LATENCY = Summary( - name = "http_request_latency_seconds", - documentation = "Latency of HTTP requests in seconds.", - labelnames = ["method", "endpoint", "http_status"] -) -TOTAL_REQUEST_COUNT = Counter( - name = "http_requests_total", - documentation = "Total HTTP requests.", - labelnames = ["method", "endpoint", "http_status"] -) -LIVE_REQUEST_COUNT = Gauge( - name = "http_requests_live_total", - documentation = "To check if an API endpoint is being served right now.", - labelnames = ["endpoint"] -) +# Info for logging that will stay constant during runtime: +SERVER_HOSTNAME = str(socket.gethostname()) +PLATFORM_INFO = platform.uname() +HOST_OS = str(distro.name(True)) +HOST_CPU = f"{PLATFORM_INFO.processor} ({PLATFORM_INFO.machine})" # ***************************************************************************************************************** @@ -94,59 +78,39 @@ LIVE_REQUEST_COUNT = Gauge( # ***************************************************************************************************************** -class MetricsAPI: +class APILogModel(BaseModel): - """ - Use this class through its context manager to automatically measure all the metrics in one place. - This was originally created to measure the performance of API endpoints made in Quart, but it should work with - other frameworks as well. - """ + # To identify the machine the code is running on. + # DO NOT MODIFY THESE: + hostname: str = SERVER_HOSTNAME + os: str = HOST_OS + cpu: str = HOST_CPU - def __init__( - self, - method = None, - endpoint = None, - raise_exception = False - ): + # To identify the project and actions: + project: Optional[str] = None + log: str + operation: str + apiVer: Optional[str] = None + logId: Optional[str] = None + logChain: Optional[str] = None - # Make provisions for things to note. - # NOTE: THESE MUST BE SET FROM OUTSIDE: - self.method = method - self.endpoint = endpoint - self.http_code = None - self.__raise_exception = raise_exception + # Timing metrics: + ts: datetime.datetime + tat: float + cpuTime: float - async def __aenter__(self): + # To understand the request that came in: + method: Optional[str] = None + url: Optional[str] = None + route: Optional[str] = None + headers: Optional[Any] = None + data: Optional[Any] = None + files: Optional[Any] = None - # Note down the start time immediately: - self.__start_ts = time.perf_counter() - self.__cpu_start_ts = time.process_time() - - # Note down the metrics: - LIVE_REQUEST_COUNT.labels(self.endpoint).inc(1) - - # Setup done: - return self - - async def __aexit__(self, exc_type, exc_value, traceback): - - # Note down the metrics: - LIVE_REQUEST_COUNT.labels(self.endpoint).dec(1) - - TOTAL_REQUEST_COUNT.labels( - self.method, - self.endpoint, - self.http_code - ).inc() - - REQUEST_LATENCY.labels( - self.method, - self.endpoint, - self.http_code - ).observe(time.perf_counter() - self.__start_ts) - - # Handle the exception as per the user's preference: - return False if self.__raise_exception else True + # To understand the output that went out: + exception: Optional[Any] = None + response: Optional[Any] = None + httpCode: Optional[int] = None # ***************************************************************************************************************** @@ -168,31 +132,8 @@ class MetricsAPI: if __name__ == "__main__": - import asyncio - import random - from prometheus_client import generate_latest + my_log = APILogModel( + log = "internal" + ) - async def simulate_endpoint(): - - async with MetricsAPI( - method = random.choice(["GET", "POST"]), - endpoint = f"https://my.domain.com/api/{random.choice([0, 1, 2, 3])}" - ) as metrics: - - # Simulate some action on some endpoint: - await asyncio.sleep(1.0) - - # Note down the values: - metrics.http_code = 200 - - async def main(): - - print("Simulating endpoints...") - tasks = [simulate_endpoint() for _ in range(250)] - await asyncio.gather(*tasks) - print("Done!") - - print("METRICS:") - print(generate_latest().decode()) - - asyncio.run(main()) + print(my_log) diff --git a/api/response.py b/api/response.py index 3c51696..02e3d72 100644 --- a/api/response.py +++ b/api/response.py @@ -30,9 +30,11 @@ # ***************************************************************************************************************** +# For data-modelling: from pydantic import BaseModel from typing import Any, Optional, List +# My utils: from utils_v2.api.codes import StatusCodes, HttpCodes @@ -64,6 +66,11 @@ from utils_v2.api.codes import StatusCodes, HttpCodes class ResponseModel(BaseModel): + """ + A model for how the response should be when developing API endpoints. + """ + + # The fields that you want in your response: status_code: StatusCodes message: Optional[str | List] = None data: Optional[Any] = None @@ -74,6 +81,11 @@ class ResponseModel(BaseModel): def for_quart(self): + """ + Call this when you are using either Flask or Quart as your framework. + :return: The output as expected by Flask and Quart. + """ + # Construct the basic structure: response_dict = { "status": 1 if self.status_code.value[0] else 0, diff --git a/cache/__pycache__/__init__.cpython-310.pyc b/cache/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..06f9307256d55010da1340723a036cace7ff4320 GIT binary patch literal 164 zcmd1j<>g`kf^+M?r-A6lAOaaM0yz#qT+9L_QW%06G#UL?G8BP?5yUTd{fzwFRQ;6H zvecaXg481YfXd{I#G>4QqWrAXhZD~clZ<<^HqjkMh5QVc0=2%0oSTD0Y*K(54i5o6Lj)w?sx zndwn=50^Wbl_^uObg&F0PJ$pYkfq@ZFS!KB2LdDx6d)J>1$_t*B!}o0K*&Xoocw-O z-90@!yNYc1kN}y*RM%8jy{dZk-mBkxRqIbpRSbL@vxW7ay>1x)!XMdB4nHp78~hlF zFjfp<3ae$dt(Ba~e6D4;^DB9kw_AmFaiys8`Btf2UMZ`*)vByiRJ)0l32|5yMd>|r zWl}sW%A$hY)Yl9#Atv85#N<6|W!gI+rfymCnrdOKSYlcnc+XflC=QB4cpefn;vqa| zJZtA6Z|0tH-$XCg4fU-aaldRDMy2B5bMejP*REY()IV78jYcnMxGm=m=Z5chgeNze z`sJ$^moI1PEjM~H5boBjv#=00oQqxAY|T2SPM$igt6%%-*DqgMzWUZk8{g?l|8`S& zj@xlOH`sDE-7P2dolRffayFYFbbM|j@H@wywPwo;oMwkLQL$5Z+HR-g$vI8_`sFK^ zuU}rebooYRObb<~5r*AhaejVtb8~L-vskhu9aE@PO;dM_mgG+ef9Y;gqH2jtboGrg6 z9pQ#9TT5?Fb$WHpah;&I8hCH_yiVxU-B!!NINB~22@AuH0=MlY1M`Hl*6Y;6rr*)) zl~!nWy1g)%RV}0!^jab5ta%}Zfkx7A^Ofq9WA@a1JXdf!>prItHl*KMZzu|z7_&F* z%3XJx9X;a3t}ER(1_~OBV45bBFlLMaBaOT}-KN~qt->Z|1^EiV@ebJIu7#d-HXF^l z9)apWXoj_WLFlY{V6x{*PsB8(7xrXFQyZ>Z*l{e55!jHWR#e`5Z?Ox~l-#2`N?d|-;g zhdEtp8sAc_s>P)-4p3We;QPrzB2G-TFqn6{&6>NuUh4&T*$bPkpmzJzyjtRUrAIX^ zYcr^+lDY0yR5{mb1|jT5_k4o`;`$sk`WIA%S~HwG*YdHr!TGs#<%dPI2N=b$U`XuK zd@o!(@$xdv#sTRz1Fz=Yse4`O(}P@;yX9>~`P*)*=S2lAtkub=RI7EMg0)(d3qlzc z8*b2OHCLko4E3qsI1?3mU8pr%bn3pqQ#9ATAdCvDZs46c9hFzloK9as+EFF9?uDS{ zh1FtYyK+5<3b!`Zb5iSZZM}Kh>qHZ}iyFI(Doj05;~*=Vw#t{F?p?nVc-35#R}7Mm zfL)PIOQBb3n!k%NUnsl5Q6$z~CtorTn*1Kk*`>nI51U6#JJ+B6W!NmKHxDd1i-$a+ z^Di>bK|C;Zxf(z@T`L!_f&!9)HDlTj_aFP54%G~*IhoRw3#3k2l=Dm!?NUPFEffVD z()5J=s62(tzMF!i;$jPU6w>woTfi^91-M}-m1bxYOdUz~#@nuJx~tehlr@Ij?%qJ3 z-nCIa{eN(ukhJ;S3$hzY8D;5S@E7>;t{K9RVl%O}bJ!KkPy-NH>U38C6yfdwtB>6P z);~YU@8k#eP9fbD@;gP87sAp`c~HQvP*J-=u|GA)!NM0uc7+mlg;I7`SceJ!v3Z@l zgO%AGN@zPdC=NQXS<1B!9>Zcar`$v~ntpRBL+xf+84N z=hP|;i$2GK%R`{JEQ#mjVIe}ApQW~ zfFrW*R&2)}HMx&#lY918WRbGIat_2{g3sWgM{$xlH2@mR@8df(w@3TWeX-6G%R3}H zw7h4N4x)$lAoeahjE(RU@@4!Q0ZjO99T0Zu}3b7wn^h(+jyJxPH;nh71pR6K|h)3|85J$x^ zJSWAY;xRm@#0%mHZyNr=lj2M7z6h6XH2WE3RHtI@faBtHM41 zmuyKAC8K|IS69#*U1{Gw>0W?dGpRBj9AFp+QRhqI}rvw!ElF$kzfUBgc|Y zmV3jQR87@#-%kD(&G(*5vJxY-uCn@meEGY5X;H`(UuclHs=_?R)H()>58mXKL; zHRapJZkqOM>EUWu4E5o?cHnp2hQkU^H{9yNcWyOr(bJ=Iy4LbtK35qQm`!^GtOoi( zxoE;$m_>_#58z{@q_2b{a2Bg+lf(`65tLBwa7`1@SOhL*TND|pr+Seg2-ph1nbhV zjP)lN#8`B0Hn1gna@N5xJAOFJP(ey9X!x7({u%8c-L@A5F5HXMqjwOMkm25>YhCHB zHSg%D=)4npcho2qT@~_S2w}W~A;l$01zd)X7c`A%y4k6>dcwQpgHw0t#2l^dvPrdC zz14&aMNf{u$+9_mVf=l<>RNz)yc!}aVU2B~XU6Lexf=_BgyI;{vGJOSka1n3A{5l9A;ND4CrlYi5>Bj&ipPO9%~ z=RQg7?2f5)6@Iz0-&IS##ZklAwjhq7fJt#-5rnAtD{Q#pkE@rXw_mk=UEhzb#owO)(rzaG;9 zDX0xCoAQW2cFsZAFA2uc{Aiz_nv8fO`eDRRNAmn$h_Ie|1IrV?8meY&w>Yjv=VE8e z>7@2bja9i{%9{ASme7_ML&G3PMo7XOSIUQ&uAb&@%JJlXW!g#H?joi|yRV(Zddfe6 z&}z(oj5RB{r}PJ-NWt-D8z{)@Y%xs50>gE2!FCT+P~HKS6;uLWJV z?kVr3I)QjLLavgdjmjb3DMoxzuccbZcqx@`3$fW6?eK07WtVD}l0p6}=R*5(cyh)p z<#YfDVV^NvnKY<#;>m}3vRJX1?hu0NI?`EExX?U@Ks!nZ;#mUk^Blmu!Wloz;*4eA zafNW|2<{>{ts>(v#kV~PE3LNK2DW}+kB+w)um5n_ufO^FP3v8=ba8dRtjC7+3hpjQX|3`+!Sl+MoL3cRi=HRZly;& zbIvl3HR2wX`hf&&7a-XzVOosR0KY;&5<)TJ656k^$-wU(o6It7Nt3gIjfqc6Fz2K{ zL}m3sOXd^{Vl=i=b+2098)4Mz%P#HWLmFc2(t7DOnOt zjZFGKQKW3@%nUHn7(y7SJAzHPJnY~j5_k>CyJo)<8ar@#Hmv)~@wus78`}u&t!GSQ z+cb^-}$8fM6P!5;JdN>~I_`QG< znm89RQcdJeJ4ZOGIN* zY9Y!KO$1SiXL_}sY{@sl0eKxsRKA>?W+;^Sz=~HWJ>=NE!NK2FlQ{(02#$b+b=NA| zrfnAftB{+vD(19Vx@-NS|Hy9XjoaUd!~G$KfyfFEY6|K*RT;|iNJg; zk+oILEKx4AL^yE662XraD#s;a8!S^RU8YMdpKyk1KU|pD5M&qRbJ!mA>UH>40B(Ep zed)?Fgk5nsUuSnRB+CiJWsiwcM3pEpuV8)R!8}dIGERxuU-7VM+tZ&VOttO)qr2HU zY`8yXe-G5&rKpp$hMa;}r(C@OTLoBbY~*g)@w^Pa=OzKC*Hyl9 zg08cR<=`k0yOhuKc3whY!~swMrP#?DNMiOThrzVsi-~dng4z`OwI5@J;0UPYm7846 zWhOU}L^d74|6#EY>6y}&$7W%HYtFc{b@ym#(zbBCraVEjVm_+WY1+2U{!_cxevFQ! zC#2%JBf%er-&5r2{aG9|<49q{f|mnPz!7sE9#3IVP^$R&puo5Ap;Rbn zV1~s(?w%QzHp&Pf<%RVlEU5CUa`$;?3}xLe4vIH{nxU+ft!ogO671;#^g1ypDId$0 zuYMN_twg2tPKj_2LW7*Lss;BCz>-utZn(GMVFfrc*0?S)pLK@1GHR-C8UR)600uJp z&cpDMfC7!1O^qOwrvT?c$D?Eu!V+H`9V`f#I$cV*3YhE=tVk**#MAw&rK*b;@l}{?{JxiW(!wUaG;){JEjNZ>poiw zuz;l9k8JMZD$;X>k)9^*9(6*whF8@|Ilx11GoeoM1PH6+6?vPrM~$lr`d+1m&EchWcY4CI zD!EDOSk8i;*`|d*`xkc3{;cv@p~OuLQ5fpoXZ?qE>l{YDG~(UxNci)W>qV%Ixr5Nd zecrR6av;XK18)Z^W`{OU?wCWZ0%{@pdJNaU7;aF?H;?-mwy+T$HmOiLAFEbTsa651 zbr7mm;z=KFgjn*@cX2NywZw&YaOVTcg{M-ky!-X6a$Wb}OG5PZj;K~%YgtA>zNlis z4-cTL<689Vmsvd|zf+?yGbD;zJWx$7%6kgDBY>GuC94^kIi^YgML}Mh1MOup%-Z8U zt2KsO5gqPzGE-bqKDw56N-M-yi9bbM3;k2f5rak(3WLSoyU*gG)E6^SuVlR#&n~iQ zF64JveJ=s0VznvYPa*>WuUNPwQNiCq^9WKc&~V!K6lg&7dJgT94+CkbJa6E~*k*BP zOr8-!laC0CQYj#J%IxR1PQ&cmvDt^j$Ff;X&A$B(WL-%navDs#gq5k;e?xk>lZB8t zoN(c_U^A4AYBQW%P>W9*Rz1Y*vGs?rdbl&@WzF0FLg`I~EynSBdi$B&`JU@RlC(L1u+=*9i@NtEqI&;>o- zRA7D|KgI;+xiNtu$}o2T+y~ZuUQVM?EI^262l-%Dy|aNbaQt-7Oy0w=;rf~qt-=<} z`8Mw4DntRKp+w7;=Tp%tzEjkPRoloZ(Xw0%Ql5$y@FruL$zhc~!3mN$E5M>U(s7>0 z)z?ngTx;S+BtpB~LyrSh9Iryk^pO(;E`bv`dw{^Ae!6v{e%fO~GO3Nah#PD4LAqM1 zleZaKIod-B++_uX;q}(FVGr@le*N25Y5*|B z77q}@SOVe@ShA4L%^0jE8xT=&s16u5A!^PhVhrmF!g5AK#J9<$v4v}qPzX;%Wkg?VYCZO{Ke%s5 z>HA|^Abr7QdI}cEI&3-wU0LY$@NO%lTH^SB2_v{7dDsIeN{A=CGm1ir*jviL+d2ZNd_`Wvc?t`5wynYE!`Bf$=cZIn>DKZI+JfQ*^{h}_bIafz$La&bH7vPy4t{&vJ8I<6~w>*4OTuBJ?ypD_=01%n~Y4KrlVHuvNDC3;QG7aP^ zAwc#$lmBPA|N2w*KUOs9H-(zc50oS@m#Ktkr~Hhc-=VPPq(oHvOm=XQX5|2a*fh zb1~56az~|R-J=C>>jU?)3h47%homIvqbzmVj_RSVfa#el2XZ)H4lK&aUMHhoYMN?> urE^;0&r>6U=aAUOq%)c0Uny5IE6-P6oIYiI$M{ZZvSi^8r3YWIfA-%$qL6g} literal 0 HcmV?d00001 diff --git a/cache/async_redis_cache.py b/cache/async_redis_cache.py index 14df725..a0ab8c0 100644 --- a/cache/async_redis_cache.py +++ b/cache/async_redis_cache.py @@ -123,6 +123,52 @@ def cache_it(cache = None, expiry = 120): return decorator +# --------------------------------------------------------------------------------------------------------------------- + + +def cache_class_methods(attr_name, expiry = 120): + + """ + This decorator factory takes an instance of the async caching class 'AsyncRedisCache' and holds your data there. + If a subsequent call is made to the same decorated function with the same inputs, the result is fetched from the + cache instead of going through the whole function again. + :param attr_name: The name of the variable that has an instance of "AsyncRedisCache". + :param expiry: The time in seconds after which the cached data must be cleared. + :return: The decorator that automatically caches your data. + """ + + def decorator(func): + + @wraps(func) + async def wrapper(self, *args, **kwargs): + + # Get the cache object first: + cache_obj = getattr(self, attr_name) + + # We first use the name of the function and the inputs given to it to generate a key for Redis with the + # simple hashing and shortening by way of base64 strings: + inputs_given = func.__name__ + str([_ for _ in args]) + str(kwargs) + sha256_hash = hashlib.sha256() + sha256_hash.update(inputs_given.encode("utf-8")) + hashed_key = sha256_hash.digest() + base64_key = base64.b64encode(hashed_key).decode("utf-8") + + # Now we check if we have the value in cache: + try: response = await cache_obj.get(base64_key, raise_exception = True) + + # If the key doesn't exist, we pass through the function and store the results. + except: + response = await func(self, *args, **kwargs) + await cache_obj.set(key = base64_key, value = response, expiry = expiry) + + # Return the response from the wrapped function. + return response + + return wrapper + + return decorator + + # ***************************************************************************************************************** # ***** **** # *** CLASSES *** diff --git a/database/__pycache__/async_mysql_v2.cpython-310.pyc b/database/__pycache__/async_mysql_v2.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b97ae9bf49442879d05ef5bb60cecdf0cfa4234f GIT binary patch literal 5474 zcmai2&2t<_6`!x&o&C^iB`d)WsnjGD;%rDHlQn1WgM8y zJk)jVXl&6~J-5~@bV0wn}TIBiiy>8FTq&r8TxTb~IOAtG=SK z0<+)Kn7wbTjq@5SZX1HJ5-Y!_txd2=HpZ&&>1&g`a8NkZ)=u&A!4#il6*hiTS28`o zYS}Y>>a_M7jZJ)H?vs4RifZ*!z5kR-4t{g5uU=lhHk;kh#CJA3@ut^wt~x7G6f!Qh zg6!eK-0I>8-6at)uluy~+__}and>yBpLU)-_w4zMcKPL37w1>6e1DW-HFDOuCV z7MJE1S4IR{nwdKiboq^?*OuoN?s<3{TGW=LKHW2>r|Y`3w&EznwRXkl^MZS=1-{;C zG`R4IaP(*5<|VZF5<0G}=`aR^lnQmg448$h#d0i4%tU_($ ztOm8&Y+^;LpOU4yxEuQ4?cxx80}Bs+76#Oo(c*8RON~@>wbVqfGabE=m?^GX#y&l< z81=a%Pm-|QO}ZEYCI{(F(wcJ>13?mjA*c9uoZ2tu#f>^d#( zg$c1lA?|{3H`?Y-oJ0ajYm3O|tRtW`FJzA3i3s>E6uik>ju#I%esyJe>6vDr7)8fV z6vvazAWob}I4tm!`n0pU$(@Zzw7i5av<4Q8JIy49>SJe@i>?#5xgTr4H^NdQL=Hs2yH5{S znr(Z+gHc9N(~W~3pH;$!oz^-RPPF0pQ5bSR38GLPZp#b81kEAqC8m2;@_mZ zgHeYBQ;cnBXWChUV0bZh57+iy#&=aI5$AKfFAX<>Mn`xgMdm}BT=3&HZWGom^=xHt zPU@`I4ViBXz7brP)gbhn9meOQFyYsex-BQ%W8-vP+E$)AF_p<8A38DF3PN{yNYZS1 z*Y76qu8$C$+=Lhf*}a@Pv1nW1G`Xl-(u#R=Ll%#Bu#{l=OqN@km;(2wS7$b(7N21- z#AejyV&-bs-}FT5s))9rqv>xHw}m(@BD6?Cw50GXS6&kG#@tb>p2T=BL|T zX}fL^28rvgVg0y*E0Qz6h z5@X9`Iow%jfDMWv%={4WvZei2|G9oeW7cI2AY#9P(F(9ADRy7bwZ5)vz425#D5WaO zFz^%2o1p6JKh}vh8|~?P0H0nsbfCo;=njA1ghPxNErAAn)JqT;l%w3onS6Mnvgcz{ zo(0f!L^umi2}dkk+5;U*gRR$1S%5u~yJFy7R|BgVMeXd|>t#V_CCzr!mbMo}O5?Je z$)l`PltCcIh^nBhB8X(Q!4nsz6(%mNC93qaxOm;?ZSp)Zj{CZyR4R*nI8By0s@~jfd+~ixbv9L4c6-y(n9=P_;{8|q`9s9g0`>s?X<9>nRerGukhV9ZHgPLEfWm2_ey#J|`{4 zWB+dBDqWy_#gAI;i+`Z?Xz?kn_sob*kt4XnoM`$&Gr~-K5rY9IK<6xNrck5c`6@4w zdA}p#NJyI&Abx3Y@B~4t*_34^OQ66-=Dl{ChYSKDWs+tuXv;!~fF1LsUdk}1I^18K zr5w3e!E&5r%_{P=@k z)9>FeUh)gsoa!lt5?m_;Y$}gU6z=s4@HU%i{bFLIsDj{8&-Y8c3cQz+Xr%Es$uqUf zpn5|Krx5L|?hK+`8PU#8%l)E?aJc_*Im}@lTq(Y#MWz8=zo~2CSdu%)r>Fq4b>NRy znw!%4*3MW`NbP+T3aH=SNo{7FK?FuK&uGVY2R5bchX&IEjpaW?6g_@+AIbtK?ToTt zIi@@)9%^{E+FM3{tY1yX(h6b1t}cF>R?`YAV#l!y+Rl#>Ba=y;#Hs#xIu7Y9@jGD4 zIA|0lD`B=wmE%Xq7j}M`on%@#P0uSAv@|dNm{!u_2K*g#lsVyrDC8~>+2hnvOq98}$$49vO&&@U zaa`ttgtxHMHt)vuT9$_m-$ItDp6O<5%JFs2-;Opm;53aw4mE63&i14&DW6WNz1Mb~pq?34Z-aIuoS3t}MB3D`K4{jVrFZ)A5?w6Y(H1 zoS{xdb_I}1b3ddx!tIfcl4C(1W`2Mc6RWM+hW(Ij+7H+z<1D?Bp;tdqG_*YxW3GE5 zE|7UrQ&^G5;9bOn6-|5vL$wt?3KWlEF%oxe4tqn^QUL!^VqxjRBSw1p6WdUxPphkK z-oXId018xfJX|<0x6PDOpR6ErY%zu(ARv%=OL0WR=9) z%B3bM{;`|*27si_h7*qW7Kf2=6*8g%MjNY6IC-2lngNc6pbqSc!~X)(=$ zB=(W+p@s}X3Qfx!D7<;_57a-hxD_XJ!pA3?dQMuKQJhGN$|-5JBauj(3R!$&VB#UX zwwOS7n-shxPy1f0?FEg{1p)-SLDGfY6Hi?aZoL01O}qrMUg6r}%ImMK&O#3b#y328 zSf-bPQfp=>WF0c6szYJYxR^w4j4u#jLz=jvV9Y5zfG+W|LU{+4TUY6!_%=B1)@(_+ zvP=5K2Rbb#$--_zT0x->K%T37W6mIcFD*J(mS&w^5uX|qZs5a{vXuIRs>F+YxPuvY0YYsr*%aqUEC(HGbGWq^OQX%i zD>Pw5u^b