§
     �’jI&  ã                   ó&  — d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d
ZdZ G d„ dee¦  «        Z G d„ d¦  «        Z	 ddee         dededededeeef         dz  ddfd„Z G d„ d¦  «        ZdS )é    N)Údefaultdict)ÚEnum)ÚQueueÚget_context)ÚBaseContext)ÚBaseProcess)ÚSynchronized)ÚEmpty)ÚAnyÚIterableÚTypeiX  éÈ   c                   ó   — e Zd ZdZdZdZdS )ÚQueueSignalsÚstopÚconfirmÚerrorN)Ú__name__Ú
__module__Ú__qualname__r   r   r   © ó    úNC:\Projects\DSCons\.venv\Lib\site-packages\qdrant_client\parallel_processor.pyr   r      s   € € € € € Ø€DØ€GØ€E€E€Er   r   c                   óZ   — e Zd Zedededd fd„¦   «         Zdee         dee         fd„ZdS )ÚWorkerÚargsÚkwargsÚreturnc                 ó   — t          ¦   «         ‚©N©ÚNotImplementedError)Úclsr   r   s      r   ÚstartzWorker.start   s   € å!Ñ#Ô#Ð#r   Úitemsc                 ó   — t          ¦   «         ‚r    r!   )Úselfr%   s     r   ÚprocesszWorker.process   s   € Ý!Ñ#Ô#Ð#r   N)r   r   r   Úclassmethodr   r$   r   r(   r   r   r   r   r      sq   € € € € € Øð$˜#ð $¨ð $°ð $ð $ð $ñ „[ð$ð$˜X cœ]ð $¨x¸¬}ð $ð $ð $ð $ð $ð $r   r   Úworker_classÚinput_queueÚoutput_queueÚnum_active_workersÚ	worker_idr   r   c                 óL  ‡— |€i }t          j        d|› dt          j        ¦   «         › �¦  «         	  | j        d	i |¤Ž}dt
          t                   fˆfd„}|                      |¦   «         ¦  «        D ]}|                     |¦  «         ŒnJ# t          $ r=}	t          j
        |	¦  «         |                     t          j        ¦  «         Y d}	~	nd}	~	ww xY w‰                     ¦   «          |                     ¦   «          ‰                     ¦   «          |                     ¦   «          |                     ¦   «         5  |xj        dz  c_        ddd¦  «         n# 1 swxY w Y   t          j        d|› d�¦  «         dS # ‰                     ¦   «          |                     ¦   «          ‰                     ¦   «          |                     ¦   «          |                     ¦   «         5  |xj        dz  c_        ddd¦  «         n# 1 swxY w Y   t          j        d|› d�¦  «         w xY w)
zç
    A worker that pulls data pints off the input queue, and places the execution result on the output queue.
    When there are no data pints left on the input queue, it decrements
    num_active_workers to signal completion.
    NzReader worker: z PID: r   c               3   ó`   •K  — 	 ‰                      ¦   «         } | t          j        k    rd S | V — Œ+r    )Úgetr   r   )Úitemr+   s    €r   Úinput_queue_iterablez%_worker.<locals>.input_queue_iterable7   s<   øè è € ðØ"—’Ñ(Ô(�Ø�<Ô,Ò,Ð,Ø�EØ�
�
�
ð	r   é   zReader worker z	 finishedr   )ÚloggingÚinfoÚosÚgetpidr$   r   r   r(   ÚputÚ	ExceptionÚ	exceptionr   r   ÚcloseÚjoin_threadÚget_lockÚvalue)
r*   r+   r,   r-   r.   r   Úworkerr3   Úprocessed_itemÚes
    `        r   Ú_workerrC   !   sè  ø€ ð €~Øˆå„LÐA 9ÐAÐAµB´I±K´KÐAÐAÑBÔBÐBð!<Ø#�Ô#Ð-Ð- fÐ-Ð-ˆð	¥h­s¤mð 	ð 	ð 	ð 	ð 	ð 	ð %ŸnšnÐ-AÐ-AÑ-CÔ-CÑDÔDð 	-ð 	-ˆNØ×Ò˜^Ñ,Ô,Ð,Ð,ð	-øåð -ð -ð -ÝÔ˜!ÑÔÐØ×Ò�Ô+Ñ,Ô,Ð,Ð,Ð,Ð,Ð,Ð,øøøøð-øøøð 	×ÒÑÔÐØ×ÒÑÔÐØ×ÒÑ!Ô!Ð!Ø× Ò Ñ"Ô"Ð"à×(Ò(Ñ*Ô*ð 	*ð 	*ØÐ$Ô$¨Ñ)Ð$Ô$ð	*ð 	*ð 	*ñ 	*ô 	*ð 	*ð 	*ð 	*ð 	*ð 	*ð 	*øøøð 	*ð 	*ð 	*ð 	*õ 	ŒÐ: iÐ:Ð:Ð:Ñ;Ô;Ð;Ð;Ð;øð 	×ÒÑÔÐØ×ÒÑÔÐØ×ÒÑ!Ô!Ð!Ø× Ò Ñ"Ô"Ð"à×(Ò(Ñ*Ô*ð 	*ð 	*ØÐ$Ô$¨Ñ)Ð$Ô$ð	*ð 	*ð 	*ñ 	*ô 	*ð 	*ð 	*ð 	*ð 	*ð 	*ð 	*øøøð 	*ð 	*ð 	*ð 	*õ 	ŒÐ: iÐ:Ð:Ð:Ñ;Ô;Ð;Ð;øøøsg   ²AB ÂE= Â
CÂ3CÃ
E= ÃCÃE= Ä;EÅEÅEÅ=A%H#Ç"G?Ç3H#Ç?HÈH#ÈHÈH#c            	       ó  — e Zd ZdZdefdedee         dedz  defd„Z	de
d	dfd
„Zdee
         de
de
d	ee
         fd„Zdee
         de
de
d	ee
         fd„Zdee
         de
de
d	ee
         fd„Zdd„Zddedz  d	dfd„Zdd„Zdd„ZdS )ÚParallelWorkerPooluK  Process-based worker pool that streams items through `Worker` subclasses.

    Spawns `num_workers` child processes via `multiprocessing`. Items pushed onto
    the input queue are picked up by any free worker; results return on the output
    queue. Backpressure is bounded by `num_workers * max_internal_batch_size`.

    Three map variants:

    - `unordered_map` â€” yields in completion order.
    - `semi_ordered_map` â€” same, but each result carries its source index.
    - `ordered_map` â€” yields in input order (buffers out-of-order results).

    Worker failures surface as `QueueSignals.error`, causing the active map call
    to raise `RuntimeError` after terminating remaining workers. Drain the map
    generator (or close it) so `join` runs in the `finally` block; relying on
    `__del__` for cleanup is a fallback only.
    NÚnum_workersr@   Ústart_methodÚmax_internal_batch_sizec                 ó®   — || _         || _        d | _        d | _        t	          |¦  «        | _        g | _        | j        |z  | _        d| _        d | _	        d S )NF)
r*   rF   r+   r,   r   ÚctxÚ	processesÚ
queue_sizeÚemergency_shutdownr-   )r'   rF   r@   rG   rH   s        r   Ú__init__zParallelWorkerPool.__init__j   s`   € ð #ˆÔØ&ˆÔØ)-ˆÔØ*.ˆÔÝ +¨LÑ 9Ô 9ˆŒØ,.ˆŒØÔ*Ð-DÑDˆŒØ"'ˆÔØ48ˆÔÐÐr   r   r   c                 óf  — | j                              | j        ¦  «        | _        | j                              | j        ¦  «        | _        | j                              d| j        ¦  «        }t          |t          ¦  «        sJ ‚|| _	        t          d| j        ¦  «        D ]”}t          | j         d¦  «        sJ ‚| j                              t          | j        | j        | j        | j	        ||                     ¦   «         f¬¦  «        }|                     ¦   «          | j                             |¦  «         Œ•d S )NÚir   ÚProcess)Útargetr   )rJ   r   rL   r+   r,   ÚValuerF   Ú
isinstanceÚ	BaseValuer-   ÚrangeÚhasattrrQ   rC   r*   Úcopyr$   rK   Úappend)r'   r   Ú	ctx_valuer.   r(   s        r   r$   zParallelWorkerPool.start{   s  € Øœ8Ÿ>š>¨$¬/Ñ:Ô:ˆÔØ œHŸNšN¨4¬?Ñ;Ô;ˆÔà”H—N’N 3¨Ô(8Ñ9Ô9ˆ	Ý˜)¥YÑ/Ô/Ð/Ð/Ð/Ø"+ˆÔå˜q $Ô"2Ñ3Ô3ð 	+ð 	+ˆIÝ˜4œ8 YÑ/Ô/Ð/Ð/Ð/Ø”h×&Ò&ÝàÔ%ØÔ$ØÔ%ØÔ+ØØ—K’K‘M”Mðð 'ñ 
ô 
ˆGð �MŠM‰OŒOˆOØŒN×!Ò! 'Ñ*Ô*Ð*Ð*ð	+ð 	+r   Ústreamr   c              /   ó
  K  — 	  | j         d	i |¤Ž | j        €
J d¦   «         ‚| j        €
J d¦   «         ‚d}d}|D ]ø}|                      ¦   «          ||z
  | j        k     r-	 | j                             ¦   «         }n\# t          $ r d }Y nNw xY w	 | j                             t          ¬¦  «        }n(# t          $ r}|  	                    ¦   «          |‚d }~ww xY w|�<|t          j        k    r#|  	                    ¦   «          t          d¦  «        ‚|V — |dz  }| j                             |¦  «         |dz  }Œùt          | j        ¦  «        D ]&}	| j                             t          j        ¦  «         Œ'||k     rb| j                             t          ¬¦  «        }|t          j        k    r#|  	                    ¦   «          t          d¦  «        ‚|V — |dz  }||k     °b| j        €
J d¦   «         ‚| j        €
J d¦   «         ‚|                      ¦   «          | j                             ¦   «          | j                             ¦   «          | j        r4| j                             ¦   «          | j                             ¦   «          d S | j                             ¦   «          | j                             ¦   «          d S # | j        €
J d¦   «         ‚| j        €
J d¦   «         ‚|                      ¦   «          | j                             ¦   «          | j                             ¦   «          | j        r3| j                             ¦   «          | j                             ¦   «          w | j                             ¦   «          | j                             ¦   «          w xY w)
NzInput queue was not initializedz Output queue was not initializedr   ©ÚtimeoutzThread unexpectedly terminatedr4   zInput queue is NonezOutput queue is Noner   )r$   r+   r,   Úcheck_worker_healthrL   Ú
get_nowaitr
   r1   Úprocessing_timeoutÚjoin_or_terminater   r   ÚRuntimeErrorr9   rV   rF   r   Újoinr<   rM   Úcancel_join_threadr=   )
r'   r[   r   r   ÚpushedÚreadr2   Úout_itemrB   Ú_s
             r   Úunordered_mapz ParallelWorkerPool.unordered_map“   sÜ  è è € ð4	0ØˆDŒJÐ Ð ˜Ð Ð Ð àÔ#Ð/Ð/Ð1RÑ/Ô/Ð/ØÔ$Ð0Ð0Ð2TÑ0Ô0Ð0àˆFØˆDØð ð �Ø×(Ò(Ñ*Ô*Ð*Ø˜D‘= 4¤?Ò2Ð2ð(Ø#'Ô#4×#?Ò#?Ñ#AÔ#A˜˜øÝ ð (ð (ð (Ø#'˜˜˜ð(øøøð Ø#'Ô#4×#8Ò#8ÕASÐ#8Ñ#TÔ#T˜˜øÝ ð  ð  ð  Ø×.Ò.Ñ0Ô0Ð0Ø˜øøøøð øøøð Ð'Ø¥<Ô#5Ò5Ð5Ø×.Ò.Ñ0Ô0Ð0Ý*Ð+KÑLÔLÐLØ"�N�N�NØ˜A‘I�DØÔ ×$Ò$ TÑ*Ô*Ð*Ø˜!‘��å˜4Ô+Ñ,Ô,ð 8ð 8�ØÔ ×$Ò$¥\Ô%6Ñ7Ô7Ð7Ð7à˜’-�-ØÔ,×0Ò0Õ9KÐ0ÑLÔL�Ø�|Ô1Ò1Ð1Ø×*Ò*Ñ,Ô,Ð,Ý&Ð'GÑHÔHÐHØ���Ø˜‘	�ð ˜’-�-ð Ô#Ð/Ð/Ð1FÑ/Ô/Ð/ØÔ$Ð0Ð0Ð2HÑ0Ô0Ð0Ø�IŠI‰KŒKˆKØÔ×"Ò"Ñ$Ô$Ð$ØÔ×#Ò#Ñ%Ô%Ð%ØÔ&ð 0ØÔ ×3Ò3Ñ5Ô5Ð5ØÔ!×4Ò4Ñ6Ô6Ð6Ð6Ð6àÔ ×,Ò,Ñ.Ô.Ð.ØÔ!×-Ò-Ñ/Ô/Ð/Ð/Ð/øð Ô#Ð/Ð/Ð1FÑ/Ô/Ð/ØÔ$Ð0Ð0Ð2HÑ0Ô0Ð0Ø�IŠI‰KŒKˆKØÔ×"Ò"Ñ$Ô$Ð$ØÔ×#Ò#Ñ%Ô%Ð%ØÔ&ð 0ØÔ ×3Ò3Ñ5Ô5Ð5ØÔ!×4Ò4Ñ6Ô6Ð6Ð6àÔ ×,Ò,Ñ.Ô.Ð.ØÔ!×-Ò-Ñ/Ô/Ð/Ð/øøøsV   „AJ, ÁA8Á7J, Á8BÂJ, ÂBÂJ, Â B,Â+J, Â,
CÂ6CÃCÃDJ, Ê,CNc                 ó>   —  | j         t          |¦  «        g|¢R i |¤ŽS r    )rj   Ú	enumerate)r'   r[   r   r   s       r   Úsemi_ordered_mapz#ParallelWorkerPool.semi_ordered_mapÊ   s-   € Ø!ˆtÔ!¥)¨FÑ"3Ô"3ÐE°dÐEÐEÐE¸fÐEÐEÐEr   c              /   ó¶   K  — t          t          ¦  «        }d} | j        |g|¢R i |¤ŽD ].\  }}|||<   ||v r |                     |¦  «        V — |dz  }||v ° Œ/d S )Nr   r4   )r   Úintrm   Úpop)r'   r[   r   r   ÚbufferÚnext_expectedÚidxr2   s           r   Úordered_mapzParallelWorkerPool.ordered_mapÍ   s–   è è € Ý�SÑ!Ô!ˆØˆà.˜Ô.¨vÐG¸ÐGÐGÐGÀÐGÐGð 	#ð 	#‰IˆC�ØˆF�3‰KØ 6Ð)Ð)Ø—j’j Ñ/Ô/Ð/Ð/Ð/Ø Ñ"�ð   6Ð)Ð)øð	#ð 	#r   c                 óÌ   — | j         D ][}|                     ¦   «         sE|j        dk    r:d| _        |                      ¦   «          t          d|j        › d|j        › �¦  «        ‚Œ\dS )zJ
        Checks if any worker process has terminated unexpectedly
        r   TzWorker PID: z# terminated unexpectedly with code N)rK   Úis_aliveÚexitcoderM   rb   rc   Úpid©r'   r(   s     r   r_   z&ParallelWorkerPool.check_worker_health×   s†   € ð ”~ð 	ð 	ˆGØ×#Ò#Ñ%Ô%ð ¨'Ô*:¸aÒ*?Ð*?Ø*.�Ô'Ø×&Ò&Ñ(Ô(Ð(Ý"Øe 7¤;ÐeÐeÐSZÔScÐeÐeñô ð øð		ð 	r   r4   r^   c                 óÖ   — d| _         | j        D ]@}|                     |¬¦  «         |                     ¦   «         r|                     ¦   «          ŒA| j                             ¦   «          dS )zM
        Emergency shutdown
        @param timeout:
        @return:
        Tr]   N)rM   rK   rd   rv   Ú	terminateÚclear)r'   r^   r(   s      r   rb   z$ParallelWorkerPool.join_or_terminateã   sr   € ð #'ˆÔØ”~ð 	$ð 	$ˆGØ�LŠL ˆLÑ)Ô)Ð)Ø×ÒÑ!Ô!ð $Ø×!Ò!Ñ#Ô#Ð#øØŒ×ÒÑÔÐÐÐr   c                 ót   — | j         D ]}|                     ¦   «          Œ| j                              ¦   «          d S r    )rK   rd   r|   ry   s     r   rd   zParallelWorkerPool.joinð   s<   € Ø”~ð 	ð 	ˆGØ�LŠL‰NŒNˆNˆNØŒ×ÒÑÔÐÐÐr   c                 ój   — | j         D ]*}|                     ¦   «         r|                     ¦   «          Œ+dS )a  
        Terminate processes if the user hasn't joined. This is necessary as
        leaving stray processes running can corrupt shared state. In brief,
        we've observed shared memory counters being reused (when the memory was
        free from the perspective of the parent process) while the stray
        workers still held a reference to them.
        For a discussion of using destructors in Python in this manner, see
        https://eli.thegreenplace.net/2009/06/12/safely-using-destructors-in-python/.
        N)rK   rv   r{   ry   s     r   Ú__del__zParallelWorkerPool.__del__õ   sF   € ð ”~ð 	$ð 	$ˆGØ×ÒÑ!Ô!ð $Ø×!Ò!Ñ#Ô#Ð#øð	$ð 	$r   )r   N)r4   )r   r   r   Ú__doc__ÚMAX_INTERNAL_BATCH_SIZEro   r   r   ÚstrrN   r   r$   r   rj   rm   rt   r_   rb   rd   r   r   r   r   rE   rE   W   s®  € € € € € ðð ð, $(Ø'>ð9ð 9àð9ð �V”ð9ð ˜D‘jð	9ð
 "%ð9ð 9ð 9ð 9ð"+˜cð + dð +ð +ð +ð +ð050 H¨S¤Mð 50¸#ð 50Èð 50ÐQYÐZ]ÔQ^ð 50ð 50ð 50ð 50ðnF x°¤}ð F¸Sð FÈCð FÐT\Ð]`ÔTað Fð Fð Fð Fð# (¨3¤-ð #¸ð #Àsð #ÈxÐX[Ì}ð #ð #ð #ð #ð
ð 
ð 
ð 
ðð ¨¨t©ð ¸Dð ð ð ð ðð ð ð ð
$ð $ð $ð $ð $ð $r   rE   r    )r5   r7   Úcollectionsr   Úenumr   Úmultiprocessingr   r   Úmultiprocessing.contextr   Úmultiprocessing.processr   Úmultiprocessing.sharedctypesr	   rU   Úqueuer
   Útypingr   r   r   ra   r�   r‚   r   r   ro   ÚdictrC   rE   r   r   r   ú<module>rŒ      sÉ  ðØ €€€Ø 	€	€	€	Ø #Ð #Ð #Ð #Ð #Ð #Ø Ð Ð Ð Ð Ð Ø .Ð .Ð .Ð .Ð .Ð .Ð .Ð .Ø /Ð /Ð /Ð /Ð /Ð /Ø /Ð /Ð /Ð /Ð /Ð /Ø BÐ BÐ BÐ BÐ BÐ BØ Ð Ð Ð Ð Ð Ø &Ð &Ð &Ð &Ð &Ð &Ð &Ð &Ð &Ð &ð Ð àÐ ðð ð ð ð �3˜ñ ô ð ð$ð $ð $ð $ð $ñ $ô $ð $ð %)ð3<ð 3<Ø�v”,ð3<àð3<ð ð3<ð "ð	3<ð
 ð3<ð ��c�ŒN˜TÑ!ð3<ð 
ð3<ð 3<ð 3<ð 3<ðlj$ð j$ð j$ð j$ð j$ñ j$ô j$ð j$ð j$ð j$r   