j4UdZddlmZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZejeZe ddZd ed<d Zd Zd d dZdZddddZdZgedRZd(dZd)d ZGd!d"Zd#Zd*d'ZdS)+aRequest log: one SQLite row per ``/v1/*`` request, for the dashboard and ``/admin/stats``. Rows are written by ``api.middleware.RequestLogMiddleware``. Code deeper in the stack (the gateway) adds to the in-flight row through ``current_record``. Async image/video submits return 200 straight away, so the task's outcome is written back onto the submit row when it finishes (``finish_task``) and counts as an error. All database work runs in a worker thread; a failed write is logged and dropped, never surfaced to the client. ) annotationsN) ContextVar)Path)Anycurrent_record)defaultz!ContextVar[dict[str, Any] | None])tsmethodpathmodelkey_idkey_name account_id status_code latency_msstreampollerror client_ip user_agenttask_ida# CREATE TABLE IF NOT EXISTS requests ( id INTEGER PRIMARY KEY AUTOINCREMENT, ts REAL NOT NULL, method TEXT NOT NULL, path TEXT NOT NULL, model TEXT, key_id TEXT, key_name TEXT, account_id TEXT, status_code INTEGER NOT NULL, latency_ms INTEGER NOT NULL, stream INTEGER NOT NULL DEFAULT 0, poll INTEGER NOT NULL DEFAULT 0, error TEXT, client_ip TEXT, user_agent TEXT, task_id TEXT ); CREATE INDEX IF NOT EXISTS idx_requests_ts ON requests (ts); CREATE INDEX IF NOT EXISTS idx_requests_key ON requests (key_id, ts); CREATE INDEX IF NOT EXISTS idx_requests_account ON requests (account_id, ts); CREATE INDEX IF NOT EXISTS idx_requests_task ON requests (task_id); TEXTINTEGER) task_statustask_msz.(status_code >= 400 OR task_status = 'failed'))i<)Qi)i: i0*)1h24h7d)2xx3xx4xx5xxfailedrstrreturnNonecJt}|||d<dSdS)zORecord the account serving the current request (the last one wins on failover).Nr)rget)rrecords 9C:\Projects\Muse2API\src\muse2api\services\request_log.py note_accountr-Fs1    ! !F )| sorted_values list[int]qfloat int | Nonec |sdStdtt|dz tj|t|zdz }||S)Nr)maxminlenmathceil)r/r1idxs r, _percentiler<Ms] t aS''!+TYq3};M;M7M-N-NQR-RSS T TC  r.c eZdZd-dZd.dZdZd/d Zd/d Zd0d Z d1d2dZ d3dZ d4dZ dddddddd d!d5d)Z d6d+Z d7d,ZdS)8 RequestLogr rr'r(c`||_d|_tj|_i|_dSN)r _conn threadingLock_lock_early)selfr s r,__init__zRequestLog.__init__Vs* 04 ^%% (* r.sqlite3.Connectionc |j|jjddt j|jdd}tj|_|d|d| td|dD}t D]$\}}||vr|d |d |%||_|jS) NT)parentsexist_okF)check_same_threadisolation_levelzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALch|] }|d S)r5.0rs r, z!RequestLog._db..fsNNNQAaDNNNr.zPRAGMA table_info(requests)z ALTER TABLE requests ADD COLUMN  ) rAr parentmkdirsqlite3connectRow row_factoryexecute executescript_SCHEMA_ADDED_COLUMNSitems)rFconnhavecolkinds r,_dbzRequestLog._db^s :  I  " "4$ " ? ? ??49W[\\\D&{D  LL2 3 3 3 LL4 5 5 5   w ' ' 'NN$,,/L"M"MNNND+1133 R R Td??LL!PC!P!P$!P!PQQQDJzr.cz|j5||g|RcdddS#1swxYwYdSr@)rDrd)rFfnargss r,_callzRequestLog._callms Z ) )2dhhjj(4((( ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) ) )s 044cK tj|jdd{VdS#t$r$td|jYdSwxYw)NcdSr@rO)dbs r,z!RequestLog.open..ss4r.zfailed to open request log %s)asyncio to_threadrh Exceptionlog exceptionr rFs r,openzRequestLog.openqsz F#DJ@@ @ @ @ @ @ @ @ @ @ F F F MM949 E E E E E E Fs!'*AAcLKdfd }tj|d{VdS)Nr'r(cj5j jd_ddddS#1swxYwYdSr@)rDrAcloserrsr,_closez RequestLog.close.._closeys & &:)J$$&&&!%DJ & & & & & & & & & & & & & & & & & &s(>AAr'r()rmrn)rFrws` r,rvzRequestLog.closexsP & & & & & & '''''''''''r.r+dict[str, Any]c KdddtfdtDddtdddttzdd rdnd dfd } t jj|d{VdS#t$rt dYdSwxYw)Nr)rrc3BK|]}|VdSr@)r*)rQcr+s r, z!RequestLog.add..s-33aFJJqMM333333r.zINSERT INTO requests (z, z ) VALUES (?)rrrkrHr'r(c|r:jdx}r|t|dSdSdSr@)r[rEpop _FINISH_SQL)rkearlyrowrFsqlrs r,runzRequestLog.add..runsk JJsC  /T[__Wd%C%CCE / ;..... / / / /r.zfailed to write request log rowrkrHr'r() tupleCOLUMNSjoinr8r*rmrnrhrorprq)rFr+rrrrs`` @@@r,addzRequestLog.adds2q33F33333733333etyy'9'9eeTYYsUXY`UaUaOaEbEbeee .C$$fjj.C.C / / / / / / / / / =#DJ44 4 4 4 4 4 4 4 4 4 = = = MM; < < < < < < =s! C$C+*C+Nrr&status finished_atr2r str | NonercK||||fdfd } tjj|d{VdS#t$rtdYdSwxYw) zAWrite an async task's outcome onto the request that submitted it.rkrHr'r(c6|tjsuj<t jdkrUjt tjt jdkQdSdSdS)N)r[rrowcountrEr8rnextiter)rkrgrFrs r,rz#RequestLog.finish_task..runs::k4009 =(, G$$+&&,,KOODdk):):$;$;<<<$+&&,,,,  = =-,r.Nz#failed to record outcome of task %sr)rmrnrhrorprq)rFrrrrrrrgs`` @r, finish_taskzRequestLog.finish_tasks UJ@ = = = = = = = = J#DJ44 4 4 4 4 4 4 4 4 4 J J J MM? I I I I I I Js 8%A! A!tasks list[Any]cKd|DsdStdz tj|jfdd{VdS#t$rt dYdSwxYw)zFFill in outcomes for submit rows logged before outcomes were recorded.cg|]>}|j |jj|j|jpidd|jf?S)messageN)finishedrvalue updated_atrr*id)rQts r, z-RequestLog.backfill_tasks..sW...!*. 2/B/B9/M/MtUVUYZ...r.Nz AND task_status IS NULLc0|Sr@) executemany)rkrowsrs r,rlz+RequestLog.backfill_tasks..s2>>#t;T;Tr.z failed to backfill task outcomes)rrmrnrhrorprq)rFrrrs @@r,backfill_taskszRequestLog.backfill_taskss.....  F66 >#DJ0T0T0T0T0TUU U U U U U U U U U > > > MM< = = = = = = >s$A$A.-A.retention_daysintc8Ktj|dzz  tj|jfdd{V}n+#t$rt dYdSwxYw|jr!t d|j||jS)Nrc2|dfS)Nz!DELETE FROM requests WHERE ts < ?)r[)rkcutoffs r,rlz"RequestLog.prune..srzz2UX^W`'a'ar.zfailed to prune request logrz-pruned %d request log rows older than %s days) timermrnrhrorprqrinfo)rFrcurrs @r,prunezRequestLog.prunes~55 ) aaaaccccccccCC    MM7 8 8 811  < d HHDclTb c c c|s#A$A*)A*Fdr)r rrr since hide_pollslimitoffsetr r float | Nonerboolrrch Kdggc} |r*| d ||r*| d ||dkr| tn@|tvr7t|ddz} | d | | dzgz |rk| d|d d d d d d} | d z|r*| d ||r| dd| d fd } t j|j| d{VS)Nz1=1z key_id = ?zaccount_id = ?r%rrz$status_code >= ? AND status_code < ?zpath LIKE ? ESCAPE '\'\z\\%z\%_z\_zts >= ?zpoll = 0z AND rkrHr'ryc|dd}|ddg}d|D|dS)Nz$SELECT COUNT(*) FROM requests WHERE rzSELECT * FROM requests WHERE z" ORDER BY id DESC LIMIT ? OFFSET ?c,g|]}t|SrO)_rowrPs r,rz1RequestLog.query..run..s333T!WW333r.)datatotal)r[fetchonefetchall)rkrrrgcondrrs r,rzRequestLog.query..runsJJLdLLdSS\\^^_`aE::XXXX&$&&v&(((0  43d333eDD Dr.rkrHr'ry) appendFAILEDSTATUS_CLASSESrreplacerrmrnrh)rFr rrr rrrrwherelowescapedrrgrs `` @@r,queryzRequestLog.querysgr t  LL & & & KK     $ LL) * * * KK # # # X   LL ~ % %fQi..3&C LL? @ @ @ S#)$ $D  ' LL2 3 3 3ll40088eDDLLSRWXXG KK# & & &   LL # # # KK     % LL $ $ $||E"" E E E E E E E E E&tz3777777777r.windowc Kt\}tj}||z zzz t| z zdz d fcd fd d  fd }tj|j|d{VS)Nr5zts >= ? AND poll = 0rkrHcolsr&r'list[dict[str, Any]]c |d|dtdd|d }d|DS)NzSELECT z, COUNT(*) AS count, SUM(z ) AS errors FROM requests WHERE z GROUP BY z ORDER BY count DESC LIMIT 50c,g|]}t|SrOdictrPs r,rz5RequestLog.stats..grouped..***DGG***r.r[rr)rkrrrgrs r,groupedz!RequestLog.stats..groupeds::[$[[[['+[[7;[[[hjj +*T*** *r.rycX|dtd  \}}}d|d d D} fdt D}|dtd d  g RD]-\}}}d |cxkr krn||c||d <||d <. ||||r||z nd t |dt |dd |d |d |d |d|d S)NzSELECT COUNT(*), COALESCE(SUM(z@), 0), COALESCE(SUM(status_code >= 500), 0) FROM requests WHERE cg|] }|d S)rrOrPs r,rz1RequestLog.stats..run..s.\\\!1\\\r.z&SELECT latency_ms FROM requests WHERE z ORDER BY latency_msc(g|]}|zzdddS)r)rcounterrorsrO)rQibucketstarts r,rz1RequestLog.stats..run..s,cccQEAJ.aHHcccr.z9SELECT CAST((ts - ?) / ? AS INTEGER) AS b, COUNT(*), SUM(z) FROM requests WHERE z GROUP BY brrrgg?gffffff?)p50p95zkey_id, key_namerr r) rrbucket_secondsrr server_errors error_raterby_key by_accountby_model by_statusseries)r[rrranger<)rkrrr latenciesrr;rerrsrgrrr n_bucketsrrs r,rzRequestLog.stats..runs+-::SSSLPSS,,hjj )E6=\\rzzSSSSUY([([\\\IcccccRWXaRbRbcccF$&JJK!KK9=KKKF*T**%,%, N N UD''''i'''''BG?F3K(&+h*? "( !.05>funn3&1)T&B&B&1)T&B&BDD!'"&899%gb,77#GB00$WR77  r.)rkrHrr&r'rr)WINDOWSrrrmrnrh) rFrlengthnowrrgrrrrrs ` @@@@@@r,statszRequestLog.statss ikkv&(61F:u/0014 +eX d + + + + + + +            >&tz3777777777r.cKtjdz d fd }tj|j|d{VdS) zPer-key request counts in one GROUP BY: everything kept (see retention), the last 24h, and the failures among those (4xx/5xx or a failed task). Task polls are excluded, as in ``stats``.rrkrHr'rc|dtdf}d|DS)NzSELECT key_id, key_name, MAX(ts) AS last_request_at, COUNT(*) AS total, COALESCE(SUM(ts >= ?), 0) AS requests_24h, COALESCE(SUM(ts >= ? AND zk), 0) AS errors_24h FROM requests WHERE poll = 0 AND key_id IS NOT NULL GROUP BY key_id ORDER BY total DESCc,g|]}t|SrOrrPs r,rz5RequestLog.key_usage..run..$rr.r)rkrrs r,rz!RequestLog.key_usage..runs\::6,2666    !) +*T*** *r.N)rr)rkrHr'r)rrmrnrh)rFrrs @r, key_usagezRequestLog.key_usagesh e# + + + + + +g.? C.P.P(P(P(P(P(P(PQQQr.)r rr'r()r'rHrx)r+ryr'r()NN) rr&rr&rr2rrrrr'r()rrr'r()rr2r'r)r rrrrrr rrrrrrrrrr'ry)rr&r'ry)r'ry)__name__ __module__ __qualname__rGrdrhrsrvrrrrrrrrOr.r,r>r>Us=++++    )))FFFF((((====&NRJJJJJ$ > > > >    37QU)-$*.5!$A#8#8#8#8#8#8J/8/8/8/8bRRRRRRr.r>zUPDATE requests SET task_status = ?, task_ms = CAST((? - ts) * 1000 AS INTEGER), error = COALESCE(error, ?), account_id = COALESCE(account_id, ?) WHERE task_id = ? AND poll = 0rR sqlite3.Rowryct|}t|dt|dc|d<|d<|S)Nrr)rr)rRds r,rr0s< QA!!H+..QvYAhK6 Hr.)rr&r'r()r/r0r1r2r'r3)rRrr'ry) __doc__ __future__rrmloggingr9rWrBr contextvarsrpathlibrtypingr getLoggerrrpr__annotations__rr]r^rrrSTATUS_FILTERSr-r<r>rrrOr.r,rs#"""""  """"""g!!5?J?OY]4^4^4^^^^^ Z 4"(I>> :M9N O O-,>,8,,****QRQRQRQRQRQRQRQRj%       r.