U ��.e(�!@s�ddlmZmZmZddlmZmZddlmZm Z ddl Z ddlZddlm Z ddlmZmZmZmZddlmZdd lmZdd lmZzddlmZWnek r�eZYnXedd �eD��Zedd �eD��Zedd �eD��Zeeddg�BZdZej �r(e�!edd�e"d�d�Z#n e�!e�Z#e$ddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5g �Z%e�!d6�Z&iZ'Gd7d8�d8e(�Z)d9d:�Z*Gd;d<�d�d>e+�Z,Gd?d@�d@e-�Z.GdAdB�dBe(�Z/GdCdD�dDe(�Z0dEdF�Z1dS)G�)�absolute_import�division�unicode_literals)� text_type�binary_type)�http_client�urllibN)�webencodings�)�EOF�spaceCharacters�asciiLetters�asciiUppercase)�_ReparseException)�_utils)�StringIO)�BytesIOcCsg|]}|�d��qS��ascii��encode��.0�item�r�E/usr/lib/python3.8/site-packages/pip/_vendor/html5lib/_inputstream.py� srcCsg|]}|�d��qSrrrrrrrscCsg|]}|�d��qSrrrrrrrs�>�.)�sumr#�r%rrr�_bufferedBytes^szBufferedStream._bufferedBytescCs<|j�|�}|j�|�|jdd7<t|�|jd<|Sr')r"r2r#�appendr$r()r%r1�datarrrr/as zBufferedStream._readStreamcCs�|}g}|jd}|jd}|t|j�kr�|dkr�|j|}|t|�|kr`|}|||g|_n"t|�|}|t|�g|_|d7}|�||||��||8}d}q|r�|�|�|��d�|�S)Nrr �)r$r(r#r6r/�join)r%r1ZremainingBytes�rvZbufferIndexZbufferOffsetZbufferedDataZbytesToReadrrrr0hs$ zBufferedStream._readFromBufferN)�__name__� __module__�__qualname__�__doc__r&r+r.r2r5r/r0rrrrr!9s r!cKs�t|tj�s(t|tjj�r.t|jtj�r.d}n&t|d�rJt|�d�t �}n t|t �}|r�dd�|D�}|rvt d|��t|f|�St|f|�SdS)NFr2rcSsg|]}|�d�r|�qS)Z _encoding)�endswith)r�xrrrr�s z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r) � isinstancerZHTTPResponserZresponseZaddbase�fp�hasattrr2r� TypeError�HTMLUnicodeInputStream�HTMLBinaryInputStream)�source�kwargsZ isUnicodeZ encodingsrrr�HTMLInputStream�s�� rIc@speZdZdZdZdd�Zdd�Zdd�Zd d �Zdd�Z d d�Z ddd�Zdd�Zdd�Z ddd�Zdd�ZdS)rE��Provides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing incorrect byte-sequences and also provides column and line tracking. i(cCsZtjsd|_ntd�dkr$|j|_n|j|_dg|_td�df|_|� |�|_ |��dS)�Initialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) Nu􏿿r r�utf-8�certain)r�supports_lone_surrogates�reportCharacterErrorsr(�characterErrorsUCS4�characterErrorsUCS2ZnewLines�lookupEncoding�charEncoding� openStream� dataStream�reset)r%rGrrrr&�s zHTMLUnicodeInputStream.__init__cCs.d|_d|_d|_g|_d|_d|_d|_dS)N�r)r*� chunkSize�chunkOffset�errors�prevNumLines�prevNumCols�_bufferedCharacterr4rrrrV�szHTMLUnicodeInputStream.resetcCst|d�r|}nt|�}|S�zvProduces a file object from source. source can be either a file object, local filename or a string. r2)rCr�r%rGr"rrrrT�s z!HTMLUnicodeInputStream.openStreamcCsT|j}|�dd|�}|j|}|�dd|�}|dkr@|j|}n||d}||fS)N� rrr )r*�countr[�rfindr\)r%r,r*ZnLinesZpositionLineZlastLinePosZpositionColumnrrr� _position�s z HTMLUnicodeInputStream._positioncCs|�|j�\}}|d|fS)z:Returns (line, col) of the current position in the stream.r )rcrY)r%�line�colrrrr$�szHTMLUnicodeInputStream.positioncCs6|j|jkr|��stS|j}|j|}|d|_|S)zo Read one character from the stream or queue if available. Return EOF when EOF is reached. r )rYrX� readChunkrr*)r%rY�charrrrrg�s zHTMLUnicodeInputStream.charNcCs�|dkr|j}|�|j�\|_|_d|_d|_d|_|j�|�}|j rX|j |}d|_ n|s`dSt |�dkr�t|d�}|dks�d|kr�dkr�nn|d|_ |dd�}|jr�|�|�|� d d �}|� dd �}||_t |�|_dS) NrWrFr r� ��i��z r`� T)�_defaultChunkSizercrXr[r\r*rYrUr2r]r(�ordrO�replace)r%rXr7Zlastvrrrrf�s0 z HTMLUnicodeInputStream.readChunkcCs(ttt�|��D]}|j�d�qdS)N�invalid-codepoint)�ranger(�invalid_unicode_re�findallrZr6)r%r7�_rrrrP%sz*HTMLUnicodeInputStream.characterErrorsUCS4cCs�d}t�|�D]�}|rqt|��}|��}t�|||d��rrt�|||d��}|tkrl|j � d�d}q|dkr�|dkr�|t|�dkr�|j � d�qd}|j � d�qdS)NF�rnTrii��r )rp�finditerrl�group�startrZisSurrogatePairZsurrogatePairToCodepoint�non_bmp_invalid_codepointsrZr6r()r%r7�skip�matchZ codepointr)Zchar_valrrrrQ)s"�z*HTMLUnicodeInputStream.characterErrorsUCS2Fc Cs�zt||f}WnNtk r^d�dd�|D��}|s@d|}t�d|�}t||f<YnXg}|�|j|j�}|dkr�|j|jkr�q�n0|� �}||jkr�|� |j|j|��||_q�|� |j|jd��|��sdq�qdd�|�}|S)z� Returns a string of characters from the stream up to but not including any character in 'characters' or EOF. 'characters' must be a container that supports the 'in' method and iteration over its characters. rWcSsg|]}dt|��qS)z\x%02x)rl)r�crrrrNsz5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N)�charsUntilRegEx�KeyErrorr9�re�compileryr*rYrX�endr6rf) r%Z charactersZopposite�charsZregexr:�mr�rrrr� charsUntil@s, z!HTMLUnicodeInputStream.charsUntilcCs@|dk r<|jdkr.||j|_|jd7_n|jd8_dSr')rYr*rX)r%rgrrr�ungetos zHTMLUnicodeInputStream.unget)N)F)r;r<r=r>rkr&rVrTrcr$rgrfrPrQr�r�rrrrrE�s & /rEc@sLeZdZdZddd�Zdd�Zd d �Zddd�Zd d�Zdd�Z dd�Z dS)rFrJN�windows-1252TcCs\|�|�|_t�||j�d|_d|_||_||_||_||_ ||_ |�|�|_|� �dS)rKi�dN)rT� rawStreamrEr&�numBytesMeta�numBytesChardet�override_encoding�transport_encoding�same_origin_parent_encoding�likely_encoding�default_encoding�determineEncodingrSrV)r%rGr�r�r�r�r�Z useChardetrrrr&�szHTMLBinaryInputStream.__init__cCs&|jdj�|jd�|_t�|�dS)Nrrm)rSZ codec_info�streamreaderr�rUrErVr4rrrrV�szHTMLBinaryInputStream.resetcCsDt|d�r|}nt|�}z|�|��Wnt|�}YnX|Sr^)rCrr.r+r!r_rrrrT�s z HTMLBinaryInputStream.openStreamcCs�|��df}|ddk r|St|j�df}|ddk r:|St|j�df}|ddk rX|S|��df}|ddk rt|St|j�df}|ddk r�|dj�d�s�|St|j�df}|ddk r�|S|�r`zddl m }Wntk r�YnvXg}|�}|j�s,|j �|j�}|�s�q,|�|�|�|�q�|��t|jd�}|j �d�|dk �r`|dfSt|j�df}|ddk �r�|Std�dfS)NrMrZ tentativezutf-16)�UniversalDetector�encodingr�)� detectBOMrRr�r��detectEncodingMetar��name� startswithr�Z%pip._vendor.chardet.universaldetectorr��ImportErrorZdoner�r2r�r6Zfeed�close�resultr.r�)r%ZchardetrSr�ZbuffersZdetectorr#r�rrrr��sP z'HTMLBinaryInputStream.determineEncodingcCs�t|�}|dkrdS|jdkr(td�}nT||jdkrH|jddf|_n4|j�d�|df|_|��td|jd|f��dS)N��utf-16be�utf-16lerLrrMzEncoding changed from %s to %s)rRr�rSr�r.rVr)r%ZnewEncodingrrr�changeEncodings z$HTMLBinaryInputStream.changeEncodingc Cs�tjdtjdtjdtjdtjdi}|j�d�}|�|dd��}d}|sp|�|�}d}|sp|�|dd ��}d }|r�|j� |�t |�S|j� d �dSdS)z�Attempts to detect at BOM at the start of the stream. If an encoding can be determined from the BOM return the name of the encoding otherwise return NonerLr�r�zutf-32lezutf-32be�N�rsr)�codecs�BOM_UTF8�BOM_UTF16_LE�BOM_UTF16_BE�BOM_UTF32_LE�BOM_UTF32_BEr�r2�getr.rR)r%ZbomDict�stringr�r.rrrr�s2� zHTMLBinaryInputStream.detectBOMcCsH|j�|j�}t|�}|j�d�|��}|dk rD|jdkrDtd�}|S)z9Report the encoding declared by the meta element rNr�rL)r�r2r��EncodingParserr.�getEncodingr�rR)r%r#�parserr�rrrr�9sz(HTMLBinaryInputStream.detectEncodingMeta)NNNNr�T)T)r;r<r=r>r&rVrTr�r�r�r�rrrrrF�s� * >"rFc@s�eZdZdZdd�Zdd�Zdd�Zdd �Zd d�Zdd �Z dd�Z dd�Zeee �Z dd�Zee�Zefdd�Zdd�Zdd�Zdd�ZdS)� EncodingBytesz�String-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raisedcCst�||��S�N)r1�__new__�lower�r%�valuerrrr�LszEncodingBytes.__new__cCs d|_dS)Nr)rcr�rrrr&PszEncodingBytes.__init__cCs|Sr�rr4rrr�__iter__TszEncodingBytes.__iter__cCs>|jd}|_|t|�kr"t�n|dkr.t�|||d�S)Nr r�rcr(� StopIterationrD�r%�prrr�__next__WszEncodingBytes.__next__cCs|��Sr�)r�r4rrr�next_szEncodingBytes.nextcCsB|j}|t|�krt�n|dkr$t�|d|_}|||d�Sr'r�r�rrr�previouscszEncodingBytes.previouscCs|jt|�krt�||_dSr��rcr(r�)r%r$rrr�setPositionlszEncodingBytes.setPositioncCs*|jt|�krt�|jdkr"|jSdSdS)Nrr�r4rrr�getPositionqs zEncodingBytes.getPositioncCs||j|jd�S�Nr )r$r4rrr�getCurrentByte{szEncodingBytes.getCurrentBytecCsH|j}|t|�kr>|||d�}||kr4||_|S|d7}q||_dS)zSkip past a list of charactersr N�r$r(rc�r%r�r�rzrrrrx�s zEncodingBytes.skipcCsH|j}|t|�kr>|||d�}||kr4||_|S|d7}q||_dSr�r�r�rrr� skipUntil�s zEncodingBytes.skipUntilcCs>|j}|||t|��}|�|�}|r:|jt|�7_|S)z�Look for a sequence of bytes at the start of a string. If the bytes are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone)r$r(r�)r%r1r�r7r:rrr� matchBytes�s zEncodingBytes.matchBytescCsR||jd��|�}|dkrJ|jdkr,d|_|j|t|�d7_dSt�dS)z�Look for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the matchNrrr T)r$�findrcr(r�)r%r1ZnewPositionrrr�jumpTo�s zEncodingBytes.jumpToN)r;r<r=r>r�r&r�r�r�r�r�r��propertyr$r��currentByte�spaceCharactersBytesrxr�r�r�rrrrr�Hs r�c@sXeZdZdZdd�Zdd�Zdd�Zdd �Zd d�Zdd �Z dd�Z dd�Zdd�ZdS)r�z?Mini parser for detecting character encoding from meta elementscCst|�|_d|_dS)z3string - the data to work on for encoding detectionN)r�r7r��r%r7rrrr&�s zEncodingParser.__init__c Cs�d|jfd|jfd|jfd|jfd|jfd|jff}|jD]Z}d}|D]D\}}|j�|�rFz|�}Wq�WqFtk r�d}Yq�YqFXqF|s:q�q:|jS) Ns�r7r�r4rrrr��szEncodingParser.handleCommentcCs�|jjtkrdSd}d}|��}|dkr,dS|ddkr\|ddk}|r�|dk r�||_dSq|ddkr�|d}t|�}|dk r�||_dSq|ddkrtt|d��}|��}|dk rt|�}|dk r|r�||_dS|}qdS) NTFrs http-equivr scontent-type�charsetscontent) r7r�r��getAttributer�rR�ContentAttrParserr��parse)r%Z hasPragmaZpendingEncoding�attrZtentativeEncoding�codecZ contentParserrrrr��s8zEncodingParser.handleMetacCs |�d�S)NF)�handlePossibleTagr4rrrr��sz%EncodingParser.handlePossibleStartTagcCst|j�|�d�S)NT)r�r7r�r4rrrr��s z#EncodingParser.handlePossibleEndTagcCsb|j}|jtkr(|r$|��|��dS|�t�}|dkrD|��n|��}|dk r^|��}qLdS)NTr)r7r��asciiLettersBytesr�r�r��spacesAngleBracketsr�)r%ZendTagr7rzr�rrrr��s z EncodingParser.handlePossibleTagcCs|j�d�S)Nrr�r4rrrr�szEncodingParser.handleOthercCs�|j}|�ttdg�B�}|dkr&dSg}g}|dkr>|r>q�nX|tkrR|��}q�nD|dkrhd�|�dfS|tkr�|�|��n|dkr�dS|�|�t|�}q.|dkr�|� �d�|�dfSt|�|��}|dk�r2|}t|�}||k�r t|�d�|�d�|�fS|tk�r$|�|��q�|�|�q�nJ|dk�rJd�|�dfS|tk�rd|�|��n|dk�rrdS|�|�t|�}|t k�r�d�|�d�|�fS|tk�r�|�|��n|dk�r�dS|�|��q|dS) z_Return a name,value pair for the next attribute in the stream, if one is found, or None�/)rNN�=)r�rr8)�'�"r)r7rxr�� frozensetr9�asciiUppercaseBytesr6r�r�r�r�)r%r7rzZattrNameZ attrValueZ quoteCharrrrr�s` zEncodingParser.getAttributeN) r;r<r=r>r&r�r�r�r�r�r�r�r�rrrrr��s$r�c@seZdZdd�Zdd�ZdS)r�cCs ||_dSr�)r7r�rrrr&fszContentAttrParser.__init__cCs�z�|j�d�|jjd7_|j��|jjdkss� �� JgIh6'