��Yf�d�@s�ddgZddlZddlZddlZddlZddlZddlZddlZddlm Z ddlm Z mZdZdZ dZej�Zdd �Zd d�ZGdd �d e�ZGdd�d�Zdd�ZGdd�de�Zdfdddd�Zdd�ZGdd�de�ZGdd�de�ZeZGdd�de�ZGdd�de�ZGd d!�d!e�Z Gd"d�de�Z!dS)#�Pool� ThreadPool�N�)�util)�get_context�TimeoutError�cCstt|��S)N)�list�map)�args�r�9/opt/alt/python35/lib64/python3.5/multiprocessing/pool.py�mapstar+srcCsttj|d|d��S)Nrr)r � itertools�starmap)rrrr �starmapstar.src@s(eZdZdd�Zdd�ZdS)�RemoteTracebackcCs ||_dS)N)�tb)�selfrrrr �__init__6szRemoteTraceback.__init__cCs|jS)N)r)rrrr �__str__8szRemoteTraceback.__str__N)�__name__� __module__�__qualname__rrrrrr r5src@s(eZdZdd�Zdd�ZdS)�ExceptionWithTracebackcCsDtjt|�||�}dj|�}||_d||_dS)N�z """ %s""")� traceback�format_exception�type�join�excr)rr rrrr r<s zExceptionWithTraceback.__init__cCst|j|jffS)N)�rebuild_excr r)rrrr � __reduce__Asz!ExceptionWithTraceback.__reduce__N)rrrrr"rrrr r;srcCst|�|_|S)N)r� __cause__)r rrrr r!Dsr!cs@eZdZdZ�fdd�Zdd�Zdd�Z�S)�MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be safely sent through the socket.csAt|�|_t|�|_tt|�j|j|j�dS)N)�reprr �value�superr$r)rr r&)� __class__rr rPszMaybeEncodingError.__init__cCsd|j|jfS)Nz(Error sending result: '%s'. Reason: '%s')r&r )rrrr rUs zMaybeEncodingError.__str__cCsd|jj|fS)Nz<%s: %s>)r(r)rrrr �__repr__YszMaybeEncodingError.__repr__)rrr�__doc__rrr)rr)r(r r$Lsr$Fc'Cs7|dks0t|�tkr*|dks0t�|j}|j}t|d�rk|jj�|jj�|dk r�||�d}x�|dks�|r!||kr!y |�} Wn&t t fk r�tjd�PYnX| dkr�tjd�P| \} }}} }yd|| |�f}WnUt k r}}z5|r_|tk r_t||j�}d|f}WYdd}~XnXy|| ||f�Wnbt k r�}zBt||d�}tjd|�|| |d|ff�WYdd}~XnXd} } }}} }|d7}q�Wtjd |�dS) Nr�_writerz)worker got EOFError or OSError -- exitingzworker got sentinel -- exitingTFrz0Possible encoding error while sending result: %szworker exiting after %d tasks)r�int�AssertionError�put�get�hasattrr+�close�_reader�EOFError�OSErrorr�debug� Exception�_helper_reraises_exceptionr� __traceback__r$)�inqueue�outqueue�initializer�initargsZmaxtasksZwrap_exceptionr.r/Z completed�task�job�i�funcr�kwds�result�e�wrappedrrr �worker]sF0 ! ,rEcCs |�dS)z@Pickle-able helper function for use by _guarded_task_generation.Nr)Zexrrr r7�sr7c@s�eZdZdZdZdd�Zddfdddd�Zdd �Zd d�Zdd �Z dd�Z fidd�Zddd�Zddd�Z ddddd�Zdd�Zddd�Zddd�Zfidddd �Zdddd!d"�Zdddd#d$�Zed%d&��Zed'd(��Zed)d*��Zed+d,��Zd-d.�Zd/d0�Zd1d2�Zd3d4�Zed5d6��Zed7d8��Z d9d:�Z!d;d<�Z"dS)=rzS Class which supports an async version of applying functions to arguments. TcOs|jj||�S)N)�_ctx�Process)rrrArrr rG�szPool.ProcessNcCs#|pt�|_|j�tj�|_i|_t|_||_ ||_ ||_|dkrvtj �psd}|dkr�td��|dk r�t|�r�td��||_g|_|j�tjdtjd|f�|_d|j_t|j_|jj�tjdtjd|j|j|j|j|jf�|_d|j_t|j_|jj�tjdtjd|j|j |jf�|_!d|j!_t|j!_|j!j�t"j#||j$d|j|j%|j|j|j|j|j!|jfdd�|_&dS) Nrz&Number of processes must be at least 1zinitializer must be a callable�targetrTZexitpriority�)'rrF� _setup_queues�queue�Queue� _taskqueue�_cache�RUN�_state�_maxtasksperchild�_initializer� _initargs�os� cpu_count� ValueError�callable� TypeError� _processes�_pool�_repopulate_pool� threadingZThreadr�_handle_workers�_worker_handler�daemon�start� _handle_tasks� _quick_put� _outqueue� _task_handler�_handle_results� _quick_get�_result_handlerrZFinalize�_terminate_pool�_inqueue� _terminate)r� processesr;r<Zmaxtasksperchild�contextrrr r�sT z Pool.__init__cCswd}xjttt|j��D]M}|j|}|jdk r"tjd|�|j�d}|j|=q"W|S)z�Cleanup after any worker processes which have exited due to reaching their specified lifetime. Returns True if any workers were cleaned up. FNzcleaning up worker %dT)�reversed�range�lenrZ�exitcoderr5r)rZcleanedr?rErrr �_join_exited_workers�s" zPool._join_exited_workerscCs�x�t|jt|j��D]�}|jdtd|j|j|j|j |j |jf�}|jj|�|j jdd�|_ d|_|j�tjd�qWdS)z�Bring the number of pool processes up to the specified number, for use after reaping workers which have exited. rHrrGZ PoolWorkerTzadded workerN)rnrYrorZrGrErircrRrSrQ�_wrap_exception�append�name�replacer_r`rr5)rr?�wrrr r[�s# zPool._repopulate_poolcCs|j�r|j�dS)zEClean up any exited workers and start replacements for them. N)rqr[)rrrr �_maintain_pool�szPool._maintain_poolcCsL|jj�|_|jj�|_|jjj|_|jjj|_ dS)N) rFZSimpleQueuerircr+�sendrbr2�recvrf)rrrr rJ�szPool._setup_queuescCs.|jtkst�|j|||�j�S)z6 Equivalent of `func(*args, **kwds)`. )rPrOr-�apply_asyncr/)rr@rrArrr �apply�sz Pool.applycCs|j||t|�j�S)zx Apply `func` to each element in `iterable`, collecting the results in a list that is returned. )� _map_asyncrr/)rr@�iterable� chunksizerrr r szPool.mapcCs|j||t|�j�S)z� Like `map()` method but the elements of the `iterable` are expected to be iterables as well and will be unpacked as arguments. Hence `func` and (a, b) becomes func(a, b). )r|rr/)rr@r}r~rrr rszPool.starmapcCs|j||t|||�S)z= Asynchronous version of `starmap()` method. )r|r)rr@r}r~�callback�error_callbackrrr � starmap_asyncszPool.starmap_asyncccs�y>d}x1t|�D]#\}}||||fifVqWWn@tk r�}z ||dt|fifVWYdd}~XnXdS)z�Provides a generator of tasks for imap and imap_unordered with appropriate handling for iterables which throw exceptions during iteration.rN��)� enumerater6r7)rZ result_jobr@r}r?�xrCrrr �_guarded_task_generationszPool._guarded_task_generationrcCs�|jtkrtd��|dkret|j�}|jj|j|j||�|j f�|S|dkswt �tj|||�}t|j�}|jj|j|jt |�|j f�dd�|D�SdS)zP Equivalent of `map()` -- can be MUCH slower than `Pool.map()`. zPool not runningrcss"|]}|D]}|Vq qdS)Nr)�.0�chunk�itemrrr � @szPool.imap..N)rPrOrV�IMapIteratorrNrMr.r��_job�_set_lengthr-r� _get_tasksr)rr@r}r~rB�task_batchesrrr �imap's" z Pool.imapcCs�|jtkrtd��|dkret|j�}|jj|j|j||�|j f�|S|dkswt �tj|||�}t|j�}|jj|j|jt |�|j f�dd�|D�SdS)zL Like `imap()` method but ordering of results is arbitrary. zPool not runningrcss"|]}|D]}|Vq qdS)Nr)r�r�r�rrr r�[sz&Pool.imap_unordered..N)rPrOrV�IMapUnorderedIteratorrNrMr.r�r�r�r-rr�r)rr@r}r~rBr�rrr �imap_unorderedBs" zPool.imap_unorderedcCs_|jtkrtd��t|j||�}|jj|jd|||fgdf�|S)z; Asynchronous version of `apply()` method. zPool not runningrN)rPrOrV�ApplyResultrNrMr.r�)rr@rrArr�rBrrr rz]s +zPool.apply_asynccCs|j||t|||�S)z9 Asynchronous version of `map()` method. )r|r)rr@r}r~rr�rrr � map_asynchszPool.map_asyncc Cs�|jtkrtd��t|d�s6t|�}|dkrztt|�t|j�d�\}}|rz|d7}t|�dkr�d}tj |||�}t |j|t|�|d|�} |jj |j| j||�df�| S)zY Helper function to implement map, starmap and their async counterparts. zPool not running�__len__N�rrr�)rPrOrVr0r �divmodrorZrr�� MapResultrNrMr.r�r�) rr@r}Zmapperr~rr�Zextrar�rBrrr r|ps&( zPool._map_asynccCsrtj�}xB|jtks6|jrP|jtkrP|j�tjd�qW|j j d�tjd�dS)Ng��?zworker handler exiting) r\�current_threadrPrOrN� TERMINATErw�time�sleeprMr.rr5)�pool�threadrrr r]�s* zPool._handle_workersc Cs�tj�}x+t|jd�D] \}}d}z�x�|D]�}|jrXtjd�Py||�Wq;tk r�} zN|dd�\} }y|| j|d| f�Wnt k r�YnXWYdd} ~ Xq;Xq;W|rtjd�|r�|dnd}||d�wPWdd}}} XqWtjd�yFtjd�|j d�tjd�x|D]}|d�qkWWntk r�tjd �YnXtjd �dS)Nz'task handler found thread._state != RUNrFzdoing set_length()rztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exitingr�)r\r��iterr/rPrr5r6�_set�KeyErrorr.r4) � taskqueuer.r:r��cacher�ZtaskseqZ set_lengthr=rCr>�idx�prrr ra�sB zPool._handle_taskscCs tj�}x�y |�}Wn)ttfk rGtjd�dSYnX|jrt|jtksft�tjd�P|dkr�tjd�P|\}}}y||j ||�Wnt k r�YnXd}}}qWx�|r�|jtkr�y |�}Wn)ttfk r,tjd�dSYnX|dkrItjd�q�|\}}}y||j ||�Wnt k r�YnXd}}}q�Wt|d�rtjd�y2x+td�D]}|j j�s�P|�q�WWnttfk r�YnXtjdt|�|j�dS) Nz.result handler got EOFError/OSError -- exitingz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr2z"ensuring that outqueue is not full� z7result handler exiting: len(cache)=%s, thread._state=%s)r\r�r4r3rr5rPr�r-r�r�r0rnr2�pollro)r:r/r�r�r=r>r?�objrrr re�s\ zPool._handle_resultsccsDt|�}x1ttj||��}|s1dS||fVqWdS)N)r��tupler�islice)r@�it�sizer�rrr r�szPool._get_taskscCstd��dS)Nz:pool objects cannot be passed between processes or pickled)�NotImplementedError)rrrr r"szPool.__reduce__cCs5tjd�|jtkr1t|_t|j_dS)Nzclosing pool)rr5rPrO�CLOSEr^)rrrr r1s z Pool.closecCs0tjd�t|_t|j_|j�dS)Nzterminating pool)rr5r�rPr^rj)rrrr � terminates zPool.terminatecCsqtjd�|jttfks(t�|jj�|jj�|j j�x|j D]}|j�qYWdS)Nzjoining pool)rr5rPr�r�r-r^rrdrgrZ)rr�rrr rs z Pool.joincCsZtjd�|jj�x9|j�rU|jj�rU|jj�tj d�qWdS)Nz7removing tasks from inqueue until task handler finishedr) rr5Z_rlock�acquire�is_aliver2r�ryr�r�)r9�task_handlerr�rrr �_help_stuff_finish(s zPool._help_stuff_finishc Cs�tjd�t|_t|_tjd�|j||t|��|j�sit|�dksit�t|_|jd�tjd�t j �|k r�|j�|r�t|dd�r�tjd�x'|D]} | j dkr�| j�q�Wtjd�t j �|k r!|j�tjd�t j �|k rJ|j�|r�t|dd�r�tjd �x8|D]0} | j�rwtjd | j�| j�qwWdS)Nzfinalizing poolz&helping task handler/workers to finishrzjoining worker handlerr�zterminating workerszjoining task handlerzjoining result handlerzjoining pool workerszcleaning up worker %d)rr5r�rPr�ror�r-r.r\r�rr0rpr��pid) �clsr�r9r:r�Zworker_handlerr�Zresult_handlerr�r�rrr rh1s8 $ zPool._terminate_poolcCs|S)Nr)rrrr � __enter___szPool.__enter__cCs|j�dS)N)r�)r�exc_typeZexc_valZexc_tbrrr �__exit__bsz Pool.__exit__)#rrrr*rrrGrrqr[rwrJr{r rr�r�r�r�rzr�r|�staticmethodr]rarer�r"r1r�rr��classmethodrhr�r�rrrr r�sF 8 .< .c@s^eZdZdd�Zdd�Zdd�Zddd �Zdd d�Zdd �ZdS)r�cCsJtj�|_tt�|_||_||_||_|||j s< *��&%@