
    !<i                        U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
m
Z
mZmZ d dlmZ d dlmZmZmZmZmZmZ d dlZej        dk     rd dlmZ d dlmZ d d	lmZmZmZ d d
l m!Z! d dl"m#Z#m$Z$m%Z% d dl&m'Z' ddl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9m:Z: ddl;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZK d dlLmMZM  ed          ZN G d deO          ZP ejQ        eR          ZSejT        eUd<    ejV        eR          ZWeeUd<    G d de          ZX G d d          ZYdeZde[fdZ\dS )     N)contextmanager)datetime	timedeltatimezone)TracebackType)Any	Coroutine	GeneratorMappingProtocolcast)      )ExceptionGroup)trace)Status
StatusCodeTracer)Redis)ConnectionError	LockErrorResponseError)Self   )	ConcurrencyLimit
DependencyFailedDependency	PerpetualRetryTimeoutget_single_dependency_of_type'get_single_dependency_parameter_of_typeresolved_dependencies)Docket	ExecutionRedisMessageRedisMessageIDRedisReadGroupResponse)compact_signatureget_signature)QUEUE_DEPTHREDIS_DISRUPTIONSSCHEDULE_DEPTHTASK_DURATIONTASK_PUNCTUALITYTASKS_COMPLETEDTASKS_FAILEDTASKS_PERPETUATEDTASKS_REDELIVEREDTASKS_RETRIEDTASKS_RUNNINGTASKS_STARTEDTASKS_STRICKENTASKS_SUCCEEDEDhealthcheck_servermetrics_server)suppress_instrumentationd   millisecondsc                   (     e Zd ZdZdef fdZ xZS )ConcurrencyBlockedz:Raised when a task cannot start due to concurrency limits.	executionc                 j    || _         t                                          d|j         d           d S )NzTask z blocked by concurrency limits)rA   super__init__key)selfrA   	__class__s     /Users/kimhansen/Desktop/03 Workspace/ceo-agents/chl-effectiveness/mcp-servers/whoop/.venv/lib/python3.11/site-packages/docket/worker.pyrD   zConcurrencyBlocked.__init__L   s6    "NNNNOOOOO    )__name__
__module____qualname____doc__r%   rD   __classcell__)rG   s   @rH   r@   r@   I   sU        DDP) P P P P P P P P P PrI   r@   loggertracerc                   P    e Zd Zdee         deeez           deeef         fdZdS )_stream_due_taskskeysargsreturnc                 
   K   d S N )rF   rS   rT   s      rH   __call__z_stream_due_tasks.__call__V   s      #rI   N)	rJ   rK   rL   liststrfloattupleintrY   rX   rI   rH   rR   rR   U   sU        I%)#+%6	sCx     rI   rR   c                    0   e Zd ZU dZeed<   eed<   eed<   eed<   eed<   eed<   eed<   e	ed	<   e	ed
<   dd ed           ed           ed           ed          ddfdededz  dededededed	e	d
e	ddfdZ
eded         fd            ZdefdZdee         dz  dedz  dedz  ddfdZdeeef         fdZdeeef         fdZedddd ed           ed           ed           ed          ddd ddd!gfd"ed#ededz  dededededed	e	d
e	d$e	d%edz  d&edz  d'ee         ddfd(            ZdHd)ZdHd*Zeeef         ed+<   d,eeef         ddfd-ZdId.e	ddfd/ZdId0ed.e	fd1Z d0ed2e!j"        ddfd3Z#dHd4Z$d0ed5e%e&z  ddfd6Z'd7e%ddfd8Z(d7e%d9eee)f         d:e*de+fd;Z,d7e%d9eee)f         de	fd<Z-d7e%d9eee)f         d=ede	fd>Z.dHd?Z/e0defd@            Z1dAedefdBZ2dCedefdDZ3dHdEZ4d0ed7e%de	fdFZ5d0ed7e%ddfdGZ6dS )JWorkera  A Worker executes tasks on a Docket.  You may run as many workers as you like
    to work a single Docket.

    Example:

    ```python
    async with Docket() as docket:
        async with Worker(docket) as worker:
            await worker.run_forever()
    ```
    docketnameconcurrencyredelivery_timeoutreconnection_delayminimum_check_intervalscheduling_resolutionschedule_automatic_tasks!suppress_internal_instrumentationN
      )minutesseconds   r=   TrU   c
                     || _         |p(t          j                     dt          j                     | _        || _        || _        || _        || _	        || _
        || _        |	| _        d S )N#)ra   socketgethostnameosgetpidrb   rc   rd   re   rf   rg   rh   ri   )
rF   ra   rb   rc   rd   re   rf   rg   rh   ri   s
             rH   rD   zWorker.__init__r   sr     Cv133CCbikkCC	&"4"4&<#%:"(@%1R...rI   )NNNc              #   z   K   | j         r-t                      5  dV  ddd           dS # 1 swxY w Y   dS dV  dS )a  Suppress OTel auto-instrumentation for internal Redis operations.

        When suppress_internal_instrumentation is True (default), this context manager
        suppresses OpenTelemetry auto-instrumentation spans for internal Redis polling
        operations like XREADGROUP, XAUTOCLAIM, and Lua script evaluations. This prevents
        thousands of noisy spans per minute from overwhelming trace storage.

        Task execution spans and user-facing operations (schedule, cancel, etc.) are
        NOT suppressed.
        N)ri   r;   rF   s    rH   _maybe_suppress_instrumentationz&Worker._maybe_suppress_instrumentation   s       1 	)++                    EEEEEs   *..c                 n   K   t          j        |                                           | _        i | _        | S rW   )asynciocreate_task
_heartbeat_heartbeat_task_execution_countsrw   s    rH   
__aenter__zWorker.__aenter__   s0      &24??3D3DEE!#rI   exc_type	exc_value	tracebackc                    K   | ` | j                                         	 | j         d {V  n# t          j        $ r Y nw xY w| `d S rW   )r~   r}   cancelrz   CancelledError)rF   r   r   r   s       rH   	__aexit__zWorker.__aexit__   su       "##%%%	&&&&&&&&&% 	 	 	D	   s   - ??c                 J    i | j                                         d| j        iS )Nzdocket.worker)ra   labelsrb   rw   s    rH   r   zWorker.labels   s3    
k  ""
TY
 
 	
rI   c                 `    i |                                  | j        j        | j        j        dS )N)zdocket.queue_keyzdocket.stream_key)r   ra   	queue_key
stream_keyrw   s    rH   _log_contextzWorker._log_context   s;    
kkmm
 $ 5!%!7
 
 
 	
rI   zredis://localhost:6379/0r<   Fzdocket.tasks:standard_tasksdocket_nameurluntil_finishedhealthcheck_portmetrics_porttasksc                   K   t          |          5  t          |          5  t          ||          4 d{V }|D ]}|                    |           t	          ||||||||	|
	  	        4 d{V 	 }t          j                    }ddt          ddffdt          t          d          rF|
                    t          j        fd	           |
                    t          j        fd
           	 |r't          j        |                                          n&t          j        |                                           d{V  n# t
          j        $ r Y nw xY wt          t          d          r>|                    t          j                   |                    t          j                   nY# t          t          d          r?|                    t          j                   |                    t          j                   w w xY wddd          d{V  n# 1 d{V swxY w Y   ddd          d{V  n# 1 d{V swxY w Y   ddd           n# 1 swxY w Y   ddd           dS # 1 swxY w Y   dS )a  Run a worker as the main entry point (CLI).

        This method installs signal handlers for graceful shutdown since it
        assumes ownership of the event loop. When embedding Docket in another
        framework (e.g., FastAPI with uvicorn), use Worker.run_forever() or
        Worker.run_until_finished() directly - those methods do not install
        signal handlers and rely on the framework to handle shutdown signals.
        )port)rb   r   N)	ra   rb   rc   rd   re   rf   rg   rh   ri   sig_namerU   c                     t                               d|            r*                                s                                 d S d S d S )Nz,Received %s, initiating graceful shutdown...)rO   infodoner   )r   run_tasks    rH   handle_shutdownz#Worker.run.<locals>.handle_shutdown   sa    JH   $ .HMMOO .$OO-----. . . .rI   SIGTERMc                        d          S )Nr   rX   r   s   rH   <lambda>zWorker.run.<locals>.<lambda>   s    OOI4N4N rI   c                        d          S )NSIGINTrX   r   s   rH   r   zWorker.run.<locals>.<lambda>   s    ??83L3L rI   )r9   r:   r$   register_collectionr`   rz   get_running_loopr[   hasattrsignaladd_signal_handlerr   r   r{   run_until_finishedrun_foreverr   remove_signal_handler)clsr   r   rb   rc   rd   re   rf   rg   rh   ri   r   r   r   r   ra   	task_pathworkerloopr   r   s                      @@rH   runz
Worker.run   s     6 $45557	F 7	F---7	F 7	F ;C888 3F 3F 3F 3F 3F 3F 3FF!& : :I..y9999 %!$/+=+=/E.C1I:[
 
 
/F /F /F /F /F /F /F /F  
 #355D:>H.# .$ . . . . . . vy11 //"N,N,N,N,N   //"M+L+L+L+L  F) '.':6;T;T;V;V'W'WHH'.': & 2 2 4 4( (H '"1    #6955 F 66v~FFF 66v}EEE #6955 F 66v~FFF 66v}EEEEF[/F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F /F	3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F 3F	7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	F 7	Fs   I=I%9I8B H%	9AEF:E# F:"E##F:&AH%	:AHH%	I%
H//I2H/3I6I%
II%II%I=%I)	)I=,I)	-I==JJc                 >   K   |                      d           d{V S )z8Run the worker until there are no more tasks to process.FforeverN_runrw   s    rH   r   zWorker.run_until_finished  s,      YYuY---------rI   c                 >   K   |                      d           d{V S )zRun the worker indefinitely.Tr   Nr   rw   s    rH   r   zWorker.run_forever  s,      YYtY,,,,,,,,,rI   r~   iterations_by_keyc                 p   K   d D              _         dt          dt          f fd} j        j                            |           	                                   d{V   j        j                            |           i  _         dS #  j        j                            |           i  _         w xY w)al  
        Run the worker until there are no more tasks to process, but limit specified
        task keys to a maximum number of iterations.

        This is particularly useful for testing self-perpetuating tasks that would
        otherwise run indefinitely.

        Args:
            iterations_by_key: Maps task keys to their maximum allowed executions
        c                     i | ]}|d S )r   rX   ).0rE   s     rH   
<dictcomp>z&Worker.run_at_most.<locals>.<dictcomp>#  s    !F!F!FS#q!F!F!FrI   rA   rU   c                 T    | j         }|vrdS j        |         |         k    rdS dS )NFT)rE   r~   )rA   rE   r   rF   s     rH   has_reached_max_iterationsz6Worker.run_at_most.<locals>.has_reached_max_iterations%  s>    -C+++u%c*.?.DDDt5rI   N)r~   r%   boolra   strike_listadd_conditionr   remove_condition)rF   r   r   s   `` rH   run_at_mostzWorker.run_at_most  s       "G!F4E!F!F!F		) 		 		 		 		 		 		 		 		 	--.HIII	())+++++++++K#445OPPP%'D""" K#445OPPP%'D"''''s   B (B5r   c                 
  K   |                                   	 	 | j                                        4 d {V }|                     ||           d {V cd d d           d {V  S # 1 d {V swxY w Y   n# t          $ r} t          j        d|                                            t          	                    d| j
        d           t          j        | j
                                                   d {V  Y nw xY w)NTr   r   z,Error connecting to redis, retrying in %s...)exc_info)_startup_logra   redis_worker_loopr   r,   addr   rO   warningre   rz   sleeptotal_seconds)rF   r   r   s      rH   r   zWorker._run7  s     	M
M;,,.. K K K K K K K%!%!2!25'!2!J!JJJJJJJK K K K K K K K K K K K K K K K K K K K K K K K K K K K K" M M M!%a777B+!    
 mD$;$I$I$K$KLLLLLLLLLLLM		Ms5   A: A(A: (
A22A: 5A26A: :BD Dr   c           	         K   t          j                    } j        r                                  d {V  t          j                             |                    }i i  j                                         dt          f fd}dt          dt          f fddt          dt          f fd	 ddt          dt          d	t          dt          f fd
}dfd}dt          dt          dd f fdd}	 |s|sr |             d {V   j        t                    z
  dk    r2t          j         j                                                   d {V  efD ]u}	 |	           d {V D ]Y\  }
}|
dk    }|D ]K\  }}|s ||||           d {V }|s.                     |           d {V   |           d {V  LZdk    r nv|ss |             d {V }||nA# t           j        $ r/ r*t&                              dt                               Y nw xY wr't          j        ddi d {V   |             d {V  |                                 | d {V  d S # r't          j        ddi d {V   |             d {V  |                                 | d {V  w xY w)NrU   c                    K   t                               d                                           4 d {V } |                     j        j                   |                     j        j                   |                                  d {V }|d         }|d         }|dk    p|dk    cd d d           d {V  S # 1 d {V swxY w Y   d S )NzChecking for workextrar   r   )	rO   debugpipelinexlenra   r   zcardr   execute)r   results
stream_len	queue_lenlog_contextr   rF   s       rH   check_for_workz+Worker._worker_loop.<locals>.check_for_workW  s\     LL,KL@@@~~'' 7 7 7 7 7 7 78dk4555t{4555+3+;+;+=+=%=%=%=%=%=%=$QZ
#AJ	!A~6Q7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7s   A4C
CCr   c           
         K   t                               d           	                                 5  |                     j        j        j        j        j        t          j	        
                                dz            d           d {V ^}}}d d d            n# 1 swxY w Y   nY# t          $ rL}dt          |          v r5j                                         d {V   |            d {V cY d }~S  d }~ww xY wd|fgS )NzGetting redeliveriesr     z0-0)rb   	groupnameconsumernamemin_idle_timestart_idcountNOGROUP   __redelivery__)rO   r   rx   
xautoclaimra   r   worker_group_namerb   r^   rd   r   r   r[   _ensure_stream_and_group)r   _redeliverieseavailable_slotsget_redeliveriesr   rF   s       rH   r   z-Worker._worker_loop.<locals>.get_redeliveriesa  s     LL/{LCCC99;; 
 
050@0@![3"&+"?%)Y&) 3AACCdJ' ' "'- 1A 	1 	1 	+ 	+ 	+ 	+ 	+ 	+'A|a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 !   A&&+>>@@@@@@@@@!1!1%!8!8888888888888	
 '566sH   B2 A%B&B2 &B**B2 -B*.B2 2
D<A D<DDDc           
        K   t                               d           j        j                            d          }	                                 5  |                     j        j        j        j        j	        di|rdn(t          j                                        dz                       d {V }d d d            n# 1 swxY w Y   nY# t          $ rL}dt          |          v r5j                                         d {V   |            d {V cY d }~S  d }~ww xY w|r3|s1t!          j        j                                                   d {V  |S )	NzGetting new deliveriesr   z	memory://>r   r   )r   r   streamsblockr   r   )rO   r   ra   r   
startswithrx   
xreadgroupr   rb   r   r^   rf   r   r   r[   r   rz   r   )r   	is_memoryresultr   r   get_new_deliveriesr   rF   s       rH   r   z/Worker._worker_loop.<locals>.get_new_deliveriesv  s     LL1LEEE 22;??I99;; 	 	#(#3#3"&+"?%)Y!%!7 =$Uaa !<!J!J!L!Lt!STT- $4 $ $      F	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 !   A&&+>>@@@@@@@@@!3!3E!:!:::::::::::::	
  Q QmD$?$M$M$O$OPPPPPPPPPMsJ    C A'C;C CC CC 
D)A D$D)#D$$D)F
message_idmessageis_redeliveryc                 8  K   	 t          j        	j        ||           d {V }n5# t          $ r(}t                              d|           Y d }~dS d }~ww xY wt          j        	                    |          |j	                  }| |<   |
|<   dz  dS )N)redeliveredzUnable to start task: %sr   F)rb   r   T)
r%   from_messagera   
ValueErrorrO   errorrz   r{   _executerE   )r   r   r   rA   r   taskactive_tasksr   r   rF   task_executionss         rH   
start_taskz'Worker._worker_loop.<locals>.start_task  s      

"+"8Km# # #      		    .%    
 uuuuu &t}}Y'?'?imTTTD!+L$-OD! q O4s   "( 
AAAc                    K   d D             } | D ]}                     |          }                     |           	 | d {V   |           d {V  H# t          $ r}t                              d|j        j                   t          j        t          j	                  t          z   |j        _        |j                            |           d {V  Y d }~d }~ww xY wd S )Nc                 :    h | ]}|                                 |S rX   )r   )r   r   s     rH   	<setcomp>zGWorker._worker_loop.<locals>.process_completed_tasks.<locals>.<setcomp>  s%    LLL		LtLLLrI   u7   🔒 Task %s blocked by concurrency limit, reschedulingr   )reschedule_message)popr@   rO   r   rA   rE   r   nowr   utcCONCURRENCY_BLOCKED_RETRY_DELAYwhenschedule)	completed_tasksr   r   r   ack_messager   r   r   r   s	       rH   process_completed_tasksz4Worker._worker_loop.<locals>.process_completed_tasks  sD     LLLLLO' N N)--d33
##D)))NJJJJJJJ%+eZ8888888888) N N NLLQ) !    !X\225TT K$ +..*.MMMMMMMMMMMMMMNN Ns   A
C&$A8C!!C&c                   K   t                               d           |                                 4 d {V }|                    j        j        j        j        |           |                    j        j        |           |                                 d {V  d d d           d {V  d S # 1 d {V swxY w Y   d S )NzAcknowledging messager   )	rO   r   r   xackra   r   r   xdelr   )r   r   r   r   rF   s      rH   r	  z(Worker._worker_loop.<locals>.ack_message  s[     LL0LDDD~~'' 
) 
) 
) 
) 
) 
) 
)8K*K1  
 K*   &&(((((((((
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
) 
)s   A&B33
B= B=Tr   r   z0Shutdown requested, finishing %d active tasks...r   return_exceptionsFrU   N)rz   Eventrh   '_schedule_all_automatic_perpetual_tasksr{   _scheduler_looprc   r   r   r   r(   r'   r&   lenr   rf   r   _delete_known_taskr   rO   r   gatherset)rF   r   r   worker_stoppingscheduler_taskr   r   r
  has_worksourcer   messagesr   r   r   task_startedr	  r   r   r   r   r   r   s   ``              @@@@@@@rH   r   zWorker._worker_loopG  sh     !-//( 	A>>@@@@@@@@@ ,  88
 
 BD?A*''))	7d 	7 	7 	7 	7 	7 	7 	7 	7	7% 	74J 	7 	7 	7 	7 	7 	7 	7 	7 	7*	E 	6L 	 	 	 	 	 	 	 	 	: #(	 	&	!	  	 		 	 	 	 	 	 	 	 	 	4	N 	N 	N 	N 	N 	N 	N 	N 	N 	N,	)U 	) 	)4 	) 	) 	) 	) 	) 	) 	) +	! 6X 6 6--/////////"&"2S5F5F"F"a''!-(C(Q(Q(S(STTTTTTTTT/1CD  F6<fUmm0C0C0C0C0C0C E E,
H(26G(G3; 	E 	E/J#* ) (1; *G]2 2 , , , , , ,L $0 E&*&=&=eW&M&M M M M M M M M&1k%&D&D D D D D D D D	E '!++ ,  6| 6%3^%5%5555555H7  6X 6 6: % 	 	 	 F%%%    	  0nlKdKKKKKKKKK--/////////!!!            0nlKdKKKKKKKKK--/////////!!!         s,   C9H
 	J 
;IJ IJ AKr  c                 .  K   t          t          |                    d                    }t          j        }|                                 }|                                r|r	 t                              d|           | 	                                5   || j
        j        | j
        j        gt          j        t          j                                                  | j
        j        g           d{V \  }}ddd           n# 1 swxY w Y   |dk    r4t                              d||| j
        j        | j
        j        |           n-# t&          $ r  t                              dd	|
           Y nw xY wt+          j        | j                                                   d{V  n6# t+          j        | j                                                   d{V  w xY w|                                |t                              d|           dS )z7Loop that moves due tasks from the queue to the stream.a_  
            local total_work = redis.call('ZCARD', KEYS[1])
            local due_work = 0

            if total_work > 0 then
                local tasks = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])

                for i, key in ipairs(tasks) do
                    local hash_key = ARGV[2] .. ":" .. key
                    local task_data = redis.call('HGETALL', hash_key)

                    if #task_data > 0 then
                        local task = {}
                        for j = 1, #task_data, 2 do
                            task[task_data[j]] = task_data[j+1]
                        end

                        redis.call('XADD', KEYS[2], '*',
                            'key', task['key'],
                            'when', task['when'],
                            'function', task['function'],
                            'args', task['args'],
                            'kwargs', task['kwargs'],
                            'attempt', task['attempt']
                        )
                        redis.call('DEL', hash_key)

                        -- Set run state to queued
                        local run_key = ARGV[2] .. ":runs:" .. task['key']
                        redis.call('HSET', run_key, 'state', 'queued')

                        -- Publish state change event to pub/sub
                        local channel = ARGV[2] .. ":state:" .. task['key']
                        local payload = '{"type":"state","key":"' .. task['key'] .. '","state":"queued","when":"' .. task['when'] .. '"}'
                        redis.call('PUBLISH', channel, payload)

                        due_work = due_work + 1
                    end
                end
            end

            if due_work > 0 then
                redis.call('ZREMRANGEBYSCORE', KEYS[1], 0, ARGV[1])
            end

            return {total_work, due_work}
            zScheduling due tasksr   )rS   rT   Nr   z#Moved %d/%d due tasks from %s to %szError in scheduler loopTr   r   zScheduler loop finished)r   rR   register_scriptsysmaxsizer   is_setrO   r   rx   ra   r   r   r   r  r   r  	timestamprb   	Exception	exceptionrz   r   rg   r   )rF   r   r  stream_due_tasks
total_workr   due_works          rH   r  zWorker._scheduler_loop  s      /3!!.5 58/
 8/
t +
''))!((** 	Pj 	PP3;GGG99;;  1A1A"k3T[5KL&l8<88BBDDdkFVW2 2 2 , , , , , ,(J               a<<LL= "-.) !         -!% !      mD$>$L$L$N$NOOOOOOOOOOgmD$>$L$L$N$NOOOOOOOOOO3 "((** 	Pj 	P6 	.kBBBBBsO   #0E A'D:E D

E D
=E F+ 'E63F+ 5E66F+ +3Gc                 ^  K   | j                                         4 d {V }	 |                    | j         j         ddd          4 d {V  | j         j                                        D ]S}t          |t                    }||j        s"|j	        } | j         
                    ||                       d {V  T	 d d d           d {V  n# 1 d {V swxY w Y   n"# t          $ r Y d d d           d {V  d S w xY wd d d           d {V  d S # 1 d {V swxY w Y   d S )Nz:perpetual:lockrj   F)timeoutblocking)rE   )ra   r   lockrb   r   valuesr"   r   	automaticrJ   r   r   )rF   r   task_function	perpetualrE   s        rH   r  z.Worker._schedule_all_automatic_perpetual_tasks_  s     ;$$&& 	 	 	 	 	 	 	% ::{'888"u &   H H H H H H H H *.):)A)A)C)C H H$K)9% %	 %,$(2 %$+4EdkoomoEEGGGGGGGGGGHH H H H H H H H H H H H H H H H H H H H H H H H H H H     '	 	 	 	 	 	 	 	 	 	 	 	 	 	$%	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	sY   D+C'A3CC'
C	C'"C	#C'&D'
D1DDD
D&)D&execution_or_messagec                   K   t          |t                    r|j        }n.|                    d          x}r|                                }nd S t
                              d|                                            | j        j	         d| }|
                    |dd           d {V  | j                            |          }| j                            |          }|                    ||           d {V  d S )Ns   keyzDeleting known taskr   z:runs:known	stream_id)
isinstancer%   rE   getdecoderO   r   r   ra   rb   hdelknown_task_keystream_id_keydelete)rF   r   r2  rE   	bytes_keyruns_keyr:  r;  s           rH   r  zWorker._delete_known_tasku  s      *I66 	&*CC.226:::Y 	""$$CCF*$2C2C2E2EFFFk&33c33jj7K888888888 33C8811#66ll>=99999999999rI   rA   c           
        K   i |                                  |                                }i |                                 |                                }|                                }| j        j                            |          r| j                                        4 d {V }| 	                    ||           d {V  d d d           d {V  n# 1 d {V swxY w Y   t                              d||           t          j        d|ddiz             d S |j        | j        v r| j        |j        xx         dz  cc<   t!          j                    }||j                                        z
  }i |d|i}d}t'          j        d|           |j        rt+          j        d|           t-          j        d|           t/          j        ||           |j        dk    rdnd	}	t                              d
|	t7          |          ||           |                    | j                   d {V  i }
d}t<                              |j         j!        tD          j#        j$        i |                                 |                                d|j         j!        i|%                                          5 }	 tM          | |          4 d {V }
tO          |
tP                    }|ry|j)        sr| j                                        4 d {V }| *                    ||           d {V }|stW          |          d}d d d           d {V  n# 1 d {V swxY w Y   d |
,                                D             }|rTt[          dd.                    |/                                          z   d |0                                D                       tO          |
tb                    }|r|j2        | j3        k    rdd |
,                                D             }tc          | j3                  }|4                                 | 5                    |||           d {V }nc| 5                    ||
|           d {V }nEtc          | j3                  }|4                                 | 5                    ||
|           d {V }t!          j                    |z
  x}|d<   tm          j        d|           |7                    tq          tr          j:                             | ;                    ||
ty          |                     d {V }|r|=                    d            d {V  nd }|| j        j>        rt          j@        |          }t          jB        |          C                    d          }|j        }t          | j        j>        E                                          }| j        jF        G                    |d|i|           d {V  |=                    |           d {V  |rdnd}	t                              d
|	t7          |          ||           d d d           d {V  n# 1 d {V swxY w Y   n# tV          $ r  t          $ r}t!          j                    |z
  x}|d<   t          j        d|           |J                    |           |7                    tq          tr          jK        t          |                               | M                    ||
           d {V }|s+| ;                    ||
ty          |                     d {V }d }| j        j>        rt          j@        |          }t          jB        |          C                    d          }|j        }t          | j        j>        E                                          }| j        jF        G                    |d|i|           d {V  t          |          j!         dt          |           }|O                    ||           d {V  |rdnd}	t          P                    d
|	t7          |          ||           Y d }~nd }~ww xY w|r_| j                                        4 d {V }| Q                    ||           d {V  d d d           d {V  n# 1 d {V swxY w Y   t-          j        d|           t          j        d|           t          j        ||           n# |r_| j                                        4 d {V }| Q                    ||           d {V  d d d           d {V  n# 1 d {V swxY w Y   t-          j        d|           t          j        d|           t          j        ||           w xY w	 d d d            d S # 1 swxY w Y   d S )Nu   🗙 %sr   r   zdocket.wherer   punctualityg        u   ↬u   ↪z
%s [%s] %sFzcode.function.name)kind
attributeslinksTc                 D    i | ]\  }}t          |t                    ||S rX   )r6  r   r   kvs      rH   r   z#Worker._execute.<locals>.<dictcomp>  s?     + + + Aq%a)9::+1+ + +rI   z1Failed to resolve dependencies for parameter(s): z, c                     g | ]	}|j         
S rX   )r   )r   
dependencys     rH   
<listcomp>z#Worker._execute.<locals>.<listcomp>  s+       $. !+ 0  rI   c                 D    i | ]\  }}t          |t                    ||S rX   )r6  r    rE  s      rH   r   z#Worker._execute.<locals>.<dictcomp>  s>     4 4 4$(Aq'1!W'='=4 !14 4 4rI   durationrm   )
result_keyasciidata)ttlu   ↫u   ↩z: )Tr   specific_labelsr   general_labels	call_reprra   r   is_strickenr   r  rO   r   r7   r   rE   r~   timer  r$  r6   r   r3   r5   r/   recordattemptr   msclaimrb   rP   start_as_current_spanfunctionrJ   r   SpanKindCONSUMERincoming_span_linksr#   r!   r   is_bypassed_can_start_taskr@   itemsr   joinrS   r.  r    baserd   start_run_function_with_timeoutr8   
set_statusr   r   OK_perpetuate_if_requestedr   mark_as_completedexecution_ttlcloudpickledumpsbase64	b64encoder8  r^   r   result_storageputr%  r1   record_exceptionERRORr[   _retry_if_requestedtypemark_as_failedr&  _release_concurrency_slotr0   r.   )rF   rA   r   counter_labelscallr   re  r@  rL  arrowdependenciesacquired_concurrency_slotspanconcurrency_limit	can_startdependency_failuresuser_timeoutlimited_dependencieslimited_timeoutr   rd   rescheduledrM  pickled_resultencoded_resultttl_secondsr   retriedpickled_exceptionencoded_exception	error_msgs                                  rH   r   zWorker._execute  s     L**,,L	0I0I0K0KLHDKKMMHY-E-E-G-GH""$$;"..y99 	{((** @ @ @ @ @ @ @e--eY?????????@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @ NN9d+N>>>q.NH3M"MNNNF=D222"9=111Q6111	in66888AAm[AA!^,,,  	5!!^444!^,,,^<<<"*Q..EL%K$kRRR oodi(((((((((.0$)!))'(++--++-- %i&8&A 
 //11 * 	
 	
 [	? Q?0yAA a a a a a a a\(E$&6) )% *	=2C2O	= $(;#4#4#6#6 = = = = = = =%.2.B.B5).T.T(T(T(T(T(T(TI#, D&8&C&C C8<5= = = = = = = = = = = = = = = = = = = = = = = = = = =+ +$0$6$6$8$8+ + +'
 + 
, S"&)),?,D,D,F,F"G"G!H 2E2L2L2N2N  	 	 	 $Aw#W#WL# ',t/FFF4 4,8,>,>,@,@4 4 40
 /6d6M.N.NO+11333+/+J+J )+?, , & & & & & &FF
 ,0+J+J )<, , & & & & & &FF
 .5T5L-M-M**00222'+'F'F%|5G( ( " " " " " " :>u9LLH{:6#'>:::OOF:=$9$9:::(,(E(E!<81L1L1L) ) # # # # # #K # Q (99T9JJJJJJJJJJ &*
!-$+2K--8->v-F-FN-3-=n-M-M-T-T '. .N *3J*-dk.G.U.U.W.W*X*XK"&+"<"@"@ *V^,D+ #A # #        (99Z9PPPPPPPPP%0;EEeEKK$eR\\4{     a a a a a a a a a a a a a a a a a a a a a a a a a a aD &    " " "59Y[[55HH;z2 N333%%a(((z'7Q @ @AAA $ 8 8L Q QQQQQQQ $($A$A!<81L1L1L% %      G
 "
;, 
(3(9!(<(<%(.(89J(K(K(R(R) )% "+J"%dk&?&M&M&O&O"P"PK+488"V->$?[ 9         
  $Aww/;;3q66;;	..yZ.PPPPPPPPP!(3e   %Ht; !        A"J - O#{0022 O O O O O O Oe"<<UINNNNNNNNNO O O O O O O O O O O O O O O O O O O O O O O O O O O !"n555#A~666$X~>>>> - O#{0022 O O O O O O Oe"<<UINNNNNNNNNO O O O O O O O O O O O O O O O O O O O O O O O O O O !"n555#A~666$X~>>>>>w[	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	? [	?s   )C
C"%C"<g>Z>Y40MY4
MY4!M"L Y4"Z4
Y>	>ZY>	Zd,bG"b>d,bd,!g,c	g
c%	%g(c%	)Ag,"ge=	+g=
fg
fAggg#&g#r{  r+  c           	      J  K   t          t          d d t          f          |j        |j        i i |j        |          }t          j        |          }	 |                                s|	                                
                                }|                                r|                                 no	 t          j        t          j        |          |           d {V }||                                s|                                 S S # t          j        $ r Y w xY w|                                s|                                 n.# |                                s|                                 w w xY w	 | d {V S # t          j        $ r t          j        w xY w)N)r+  )r   r	   r   r\  rT   kwargsrz   r{   r   	remainingr   expiredr   wait_forshieldTimeoutErrorr   )rF   rA   r{  r+  	task_coror   r  r   s           rH   rf  z!Worker._run_function_with_timeoutL  s      dD#o&I&" 	
 	
	 "9--	iikk #--//==??	??$$ KKMMM#*#3t,,i$ $ $      F " 99;;  +   H 99;;  99;; 	'::::::% 	' 	' 	'&&	's7   A#E 8/D D# E "D##E +E:>F F"c                   K   t          |t                    }|sdS |j        |j        |j        k    rdS t	          j        t          j                  |j        z   |_	        |xj        dz  c_        |
                    d           d {V  t          j        di |                                 |                                           dS )NFr   T)replace)r!   r   attemptsrX  r   r  r   r  delayr  r  r4   r   r   rS  )rF   rA   r{  retrys       rH   rt  zWorker._retry_if_requestedt  s      
 .lEBB 	5>%)*;u~*M*M5!hl33ekA	Q   .........!LL1I1I1K1KLMMMtrI   rL  c                   K   t          |t                    }|sdS |j        r'| j                            |j                   d {V  dS t          j        t          j	                  }t          |||j        z   |z
            } | j                            |j        ||j                  |j        i |j         d {V  t!          j        di |                                 |                                           dS )NFr   T)r!   r   	cancelledra   r   rE   r   r  r   r  maxeveryr  r\  rT   r  r2   r   r   rS  )rF   rA   r{  rL  r1  r  r  s          rH   ri  zWorker._perpetuate_if_requested  s      2,	JJ	 	5 	+$$Y]3333333335l8<((3io-899Jdk!!)"4dIMJJ^

 
 	
 	
 	
 	
 	
 	
 	

 	a!PDKKMM!PY5M5M5O5O!PQQQtrI   c           
          t                               d| j                   | j        j                                        D ];\  }}t                               d|t          t          |                               <d S )Nz,Starting worker %r with the following tasks:z* %s(%s))rO   r   rb   ra   r   rb  r)   r*   )rF   	task_namer   s      rH   r   zWorker._startup_log  su    BDINNN#{06688 	W 	WOItKK
I/@tATAT/U/UVVVV	W 	WrI   c                     | j         j        S rW   )ra   workers_setrw   s    rH   r  zWorker.workers_set  s    {&&rI   worker_namec                 6    | j                             |          S rW   )ra   worker_tasks_set)rF   r  s     rH   r  zWorker.worker_tasks_set  s    {++K888rI   r  c                 6    | j                             |          S rW   )ra   task_workers_set)rF   r  s     rH   r  zWorker.task_workers_set  s    {++I666rI   c                   K   	 	 t          j        t          j                                                  }| j        j        | j        j        z  }||                                z
  }t          | j        j
                  }| j                                        4 d {V }|                                 5  |                                4 d {V }|                    | j        d|           |                    | j        | j        |i           |D ]K}|                     |          }|                    |d|           |                    || j        |i           L |j        |                     | j                  g|R   |                    |                     | j                  t-          |t/          d                               |                                 d {V  d d d           d {V  n# 1 d {V swxY w Y   |                                4 d {V }|                    | j        j                   |                    | j        j        d|           |                    | j        j        |d           |                                 d {V }	d d d           d {V  n# 1 d {V swxY w Y   d d d            n# 1 swxY w Y   |	d         }
|	d         }|	d         }t;          j        |
|z   | j                                                   tA          j        || j                                                   d d d           d {V  n# 1 d {V swxY w Y   n# tB          j"        $ r Y d S tF          $ rY tI          j%        d|                                            tL          '                    dd| (                                           Y n>tR          $ r2 tL          '                    dd| (                                           Y nw xY wtC          j*        | j        j                                                   d {V  S)	NTr   r   rm   z+inf   zError sending worker heartbeatr  )+r   r  r   r  r$  ra   heartbeat_intervalmissed_heartbeatsr   rZ   r   r   rx   r   zremrangebyscorer  zaddrb   r  saddr  expirer  r   r   r   r   zcountr   r+   r  r   r-   rz   r   r   r,   r   rO   r&  r   r%  r   )rF   r  maximum_ageoldest
task_namesrr   r  r  r   stream_depthoverdue_depthschedule_depths                rH   r|   zWorker._heartbeat  sy     ;	P8l8<00::<<K2T[5RR  {88:::!$+"344
;,,.. M M M M M M M!==?? J J#$::<< 5 5 5 5 5 5 58$55d6F6RRR$MM$*:TY<LMMM-7 R R	373H3H3S3S 0 ( 9 9:JAv V V V (.>C@P Q Q Q Q)HM$*?*?	*J*JXZXXXX$OO $ 5 5di @ @ #K11E1E1E F F  
 #+"2"2"4"444444445 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5 5" $%::<< J J J J J J J8$MM$+*@AAA$OODK,A1cJJJ$OODK,A3OOO7?7G7G7I7I1I1I1I1I1I1IGJ J J J J J J J J J J J J J J J J J J J J J J J J J J%J J J J J J J J J J J J J J J2 $+1:L$+AJM%,QZNOL=$@$+BTBTBVBVWWW"&~t{7I7I7K7KLLL?M M M M M M M M M M M M M M M M M M M M M M M M M M MB )   "   !%a777  4!++-- !     
      4!++-- !      - > L L N NOOOOOOOOOw;	Ps   BM. M2KDG6$K6
H  KH K!A<J/K/
J99K<J9=K MK	MK	A6M
M. 
M&&M. )M&*M. .P AP!8PPc           	      
  K   t          |j        t                    }|sdS 	 |                    |j                  }n# t
          $ r Y dS w xY w|j        p| j        j        }| d|j         d| }d}t          j
        t          j                                                  }| j                                        dz   }	|                    |d|t#          |j                  |j        ||	           d{V }
t)          |
          S )z6Check if a task can start based on concurrency limits.T:concurrency::a  
        local key = KEYS[1]
        local max_concurrent = tonumber(ARGV[1])
        local task_key = ARGV[2]
        local current_time = tonumber(ARGV[3])
        local slot_timeout = tonumber(ARGV[4])

        -- Clean up stale slots from crashed workers or orphaned tasks
        redis.call('ZREMRANGEBYSCORE', key, 0, current_time - slot_timeout)

        -- Check if this task already has a slot (from a previous delivery attempt)
        local slot_time = redis.call('ZSCORE', key, task_key)
        if slot_time then
            local age = current_time - slot_time
            if age < slot_timeout then
                -- Fresh slot - another delivery is still executing this task
                return 0
            else
                -- Stale slot - original worker probably crashed, allow takeover
                redis.call('ZADD', key, current_time, task_key)
                return 1
            end
        end

        -- No existing slot for this task - check if we can acquire a new one
        if redis.call('ZCARD', key) < max_concurrent then
            redis.call('ZADD', key, current_time, task_key)
            return 1
        else
            return 0
        end
        rk   r   N)r"   r\  r   get_argumentargument_nameKeyErrorscopera   rb   r   r  r   r  r$  rd   r   evalr[   max_concurrentrE   r   )rF   r   rA   r~  argument_valuer  concurrency_key
lua_scriptcurrent_timeslot_timeoutr   s              rH   ra  zWorker._can_start_task  sK      D 0
 
 ! 	4	&334E4STTNN 	 	 	44	 "';4;+;UU#4#BUU^UU 	
B  |HL11;;== .<<>>Bzz!011M
 
 
 
 
 
 
 
 F||   = 
A
Ac                 &  K   t          |j        t                    }|sdS 	 |                    |j                  }n# t
          $ r Y dS w xY w|j        p| j        j        }| d|j         d| }|	                    ||j
                   d{V  dS )z/Release a concurrency slot when task completes.Nr  r  )r"   r\  r   r  r  r  r  ra   rb   zremrE   )rF   r   rA   r~  r  r  r  s          rH   rw  z Worker._release_concurrency_slot@  s      
 D 0
 
 ! 	F	&334E4STTNN 	 	 	FF	 "';4;+;UU#4#BUU^UU 	
 jj)-88888888888r  r  r  )7rJ   rK   rL   rM   r$   __annotations__r[   r^   r   r   rD   r   r
   rx   r   r   ru  BaseExceptionr   r   r   r   r   classmethodrZ   r   r   r   dictr   r   r   r   rz   r  r  r  r%   r&   r  r   r   r    r   rf  rt  ri  r   propertyr  r  r  r|   ra  rw  rX   rI   rH   r`   r`   [   s        
 
 NNN
III!!!!!!!!%%%%$$$$""""'++++
  (1	!(<(<(<(1	!(<(<(<,5I3,G,G,G+49#+F+F+F)-26S SS DjS 	S
 &S &S !*S  )S #'S ,0S 
S S S S, ;K1L    ^"$    
!}%,! !4'! !4'	!
 
! ! ! !
S) 
 
 
 

gc3h/ 
 
 
 
  $-(1	!(<(<(<(1	!(<(<(<,5I3,G,G,G+49#+F+F+F)-26$'+#'9:PF PFPF PF Dj	PF
 PF &PF &PF !*PF  )PF #'PF ,0PF PF *PF DjPF CyPF  
!PF PF PF [PFd. . . .- - - - CH~%%%(7383D ( ( ( ( (>M M$ M4 M M M M t! t! t! t! t! t! t!l`C`C !`C 
	`C `C `C `CD   ,::2;l2J:	: : : :(A?	 A?d A? A? A? A?F&'&' 3
?+&' 	&'
 
&' &' &' &'P 3
?+ 
	   ( 3
?+ 	
 
   4W W W W
 'S ' ' ' X'9C 9C 9 9 9 97# 7# 7 7 7 7<P <P <P <P|O5 OY O4 O O O Ob99'09	9 9 9 9 9 9rI   r`   rn   rU   c                 ,    | dk     r	| dz  ddS | ddS )Nr<   r   z6.0frY  zs rX   rm   s    rH   rY  rY  [  s3    }}D.)))))"""""rI   )]rz   rn  loggingrt   r   rr   r!  rV  
contextlibr   r   r   r   typesr   typingr   r	   r
   r   r   r   rl  version_infoexceptiongroupr   opentelemetryr   opentelemetry.tracer   r   r   redis.asyncior   redis.exceptionsr   r   r   typing_extensionsr   r{  r   r   r   r   r   r    r!   r"   r#   ra   r$   r%   r&   r'   r(   rA   r)   r*   instrumentationr+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   #opentelemetry.instrumentation.utilsr;   r  r%  r@   	getLoggerrJ   rO   Loggerr  
get_tracerrP   rR   r`   r\   r[   rY  rX   rI   rH   <module>r     s      				   



  % % % % % % 2 2 2 2 2 2 2 2 2 2       E E E E E E E E E E E E E E E E    g------       : : : : : : : : : :       F F F F F F F F F F " " " " " "
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
              8 7 7 7 7 7 7 7                                   & I H H H H H
 #,)"="="= P P P P P P P P +*844 4 4 4!!(++ + + +       }9 }9 }9 }9 }9 }9 }9 }9@ # ## # # # # # #rI   