Sindbad~EG File Manager

�

&Μg�$���	dd
lZdd
l
Zdd
lZdg
Zej
dd�ZGd�d�ZGd�d�ZGd�d	�Z	y
)
�N�RobotFileParser�RequestRatezrequests secondsc�X�eZ
dZ	dd
�
Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zy
)rc�z�g|_g|_
d|_d
|_d
|_|j|
�

d|_y)NFr)�entries�sitemaps�
default_entry�disallow_all�	allow_all�set_url�last_checked��self�urls  �//usr/local/lib/python3.12/urllib/robotparser.py�__init__zRobotFileParser.__init__s;�������
�!���!���������S�����c

��	|jS�
N)
r
�
rs
 r�mtimezRobotFileParser.mtime%s��	�� � � rc
�8�	d
dl}
|
j
�|_
y)Nr)�timer
)rrs  r�modifiedzRobotFileParser.modified.s��	�	� �I�I�K��rc�r�	|
|_tjj|
�
d
d\|_|_y)N�
�)r�urllib�parse�urlparse�host�pathrs  rrzRobotFileParser.set_url6s0��:����%�|�|�4�4�S�9�!�A�>���	�4�9rc
�
�		tjj|j�
}
|
j	�}|j|j
d
�
j��

y#tjj$rT}|jdvrd|_n4|jdk\r |jdkrd|_Yd}~yYd}~yYd}~yYd}~yd}~w
wxYw
)Nzutf-8)i�
i�
Ti�
i�
)
r�request�urlopenr�readr�decode�
splitlines�error�	HTTPError�coder
r)r�
f�raw�errs    rr&zRobotFileParser.read;s���B�		9����&�&�t�x�x�0�A��&�&�(�C��J�J�s�z�z�'�*�5�5�7�8���|�|�%�%�	&��x�x�:�%�$(��!����S��S�X�X��^�!%����&4��"��	&�s�)A+�+C�;C�Cc��d
|
jvr|j�|
|_
yy|jj|
�

y�N�
*)�
useragentsr	r�append)r�entrys  r�
_add_entryzRobotFileParser._add_entryHs=���%�"�"�"��!�!�)�%*��"�*�
�L�L����&rc���	d
}t
�}|j�
|
D�]�}|s4|dk(r
t
�}d
}n"|dk(r|j|�

t
�}d
}|jd�
}|d
k\r|d|}|j	�}|s
�h|jdd�}t
|�
dk(s
��|d
j	�j�|d
<tjj|dj	��
|d<|d
dk(rB|dk(r|j|�

t
�}|jj|d�

d}�
�*|d
dk(r3|d
k7s�
�9|jjt|dd��

d}�
�e|d
d	k(r3|d
k7s�
�t|jjt|dd
��

d}�
��|d
dk(r?|d
k7s�
��|dj	�j�rt!|d�
|_d}�
��|d
dk(r�|d
k7s�
��|djd
�
}t
|�
dk(rk|d
j	�j�rJ|dj	�j�r)t%t!|d
�
t!|d�
�|_d}���|d
dk(s���|j(j|d�

���|dk(r|j|�

yy)Nrr��
#�
:z
user-agent�disallowF�allowTzcrawl-delayzrequest-rate�
/�sitemap)�Entryrr5�find�strip�split�len�lowerrr�unquoter2r3�	rulelines�RuleLine�isdigit�int�delayr�req_rater)r�lines�stater4�line�
i�numberss       rrzRobotFileParser.parseQs���	�������
�
���D���A�:�!�G�E��E��a�Z��O�O�E�*�!�G�E��E��	�	�#��A��A�v��B�Q�x���:�:�<�D����:�:�c�1�%�D��4�y�A�~��q�'�-�-�/�/�/�1��Q�� �,�,�.�.�t�A�w�}�}��?��Q���
�7�l�*��
�z�����.� %����$�$�+�+�D��G�4��E��!�W�
�*��
�z����.�.�x��Q���/G�H� !���!�W��'��
�z����.�.�x��Q���/F�G� !���!�W�
�-��
�z� �
�7�=�=�?�2�2�4�*-�d�1�g�,�E�K� !���!�W��.��
�z�"&�q�'�-�-��"4����L�A�-�'�!�*�2B�2B�2D�2L�2L�2N� '�
�
� 0� 0� 2� :� :� <�-8��W�Q�Z��#�g�VW�j�/�-Z�E�N� !���!�W�	�)�
�M�M�(�(��a��1�o
�p
�A�:��O�O�E�"�rc�d�	|jr
y
|jr
y|js
y
tjjtjj
|�
�
}tjjdd|j|j|j|jf�
}tjj|�
}|sd}|jD]&}|j|
�
s
�|j|�
c
S|j r|j j|�
Sy)NFT�r<)r
rr
rrr rD�
urlunparser"�params�query�fragment�quoter�
applies_to�	allowancer	)r�	useragentr�
parsed_urlr4s     r�	can_fetchzRobotFileParser.can_fetch�s���K������>�>��
� � ���\�\�*�*�6�<�<�+?�+?��+D�E�
��l�l�%�%�r�"�Z�_�_����j�.�.�
�0C�0C�
'E
�
F
���l�l� � ��%����C��\�\�E����	�*����s�+�+�"�����%�%�/�/��4�4�rc���|j
�s
y|jD]!}|j|
�
s
�|jc
S|jr|jjSyr)rrrWrIr	�rrYr4s   r�crawl_delayzRobotFileParser.crawl_delay�sT���z�z�|���\�\�E����	�*��{�{�"�"�����%�%�+�+�+�rc���|j
�s
y|jD]!}|j|
�
s
�|jc
S|jr|jjSyr)rrrWrJr	r]s   r�request_ratezRobotFileParser.request_rate�sT���z�z�|���\�\�E����	�*��~�~�%�"�����%�%�.�.�.�rc

�4�|js
y|jSr)
rrs
 r�	site_mapszRobotFileParser.site_maps�s���}�}���}�}�rc
��|j}
|j�|
|jg
z}
d
jtt|
��
S)Nz

)rr	�join�map�str)rrs  r�__str__zRobotFileParser.__str__�s@���,�,�����)���!3�!3� 4�4�G��{�{�3�s�G�,�-�-rN)
rQ)�__name__�
__module__�__qualname__rrrrr&r5rr[r^r`rbrg�rrrrsE���
�!�(�?�
9�'�G
#�R�:���
.rc
� �eZ
dZ	d
�Zd�Zd�Zy)rFc���|
d
k(r|sd}tjjtjj|
�
�
}
tjj	|
�
|_||_y)NrQT)rrrRr rVr"rX)rr"rXs   rrzRuleLine.__init__�sP���2�:�i��I��|�|�&�&�v�|�|�'<�'<�T�'B�C���L�L�&�&�t�,��	�"��rc�Z�|jd
k(x
s
|
j|j�
Sr0)r"�
startswith)r�filenames  rrWzRuleLine.applies_to�s%���y�y�C��A�8�#6�#6�t�y�y�#A�Arc
�B�|jrd
n
ddz|jzS)N�Allow�Disallowz: )rXr"rs
 rrgzRuleLine.__str__�s���>�>��z�T�A�D�I�I�M�MrN)rhrirjrrWrgrkrrrFrF�s��
1�#�
B
�
N
rrFc
�&�eZ
dZ	d
�Zd�Zd�Zd�Zy)r>c
�<�g|_g|_
d|_d|_yr)r2rErIrJrs
 rrzEntry.__init__�s����������
���
rc
�
�g}
|jD]}|
jd
|���

�|j�|
jd|j���

|j�7|j}|
jd|j�d|j
���

|
j
tt|j��

dj|
�
S)NzUser-agent: z
Crawl-delay: zRequest-rate: r<�

)r2r3rIrJ�requests�seconds�extendrerfrErd)r�ret�agent�rates    rrgz
Entry.__str__�s������_�_�E��J�J��e�W�-�.�%��:�:�!��J�J��t�z�z�l�3�4��=�=�$��=�=�D��J�J���
�
��a����~�F�G��
�
�3�s�D�N�N�+�,��y�y��~�rc��	|
j
d
�
dj�}
|jD]}|dk(r
y|j�}||
vs
�
yy)Nr<rr1TF)rArCr2)rrYr|s   rrWzEntry.applies_to�sT��@��O�O�C�(��+�1�1�3�	��_�_�E���|���K�K�M�E��	�!��
%�rc�f�	|jD]!}|j|
�
s
�|jc
Sy
)NT)rErWrX)rrprMs   rrXzEntry.allowance

s2��	%��N�N�D����x�(��~�~�%�#�rN)rhrirjrrgrWrXrkrrr>r>�s��I��
��rr>)
�collections�urllib.parser�urllib.request�__all__�
namedtuplerrrFr>rkrr�<module>r�
sU�


�

�
����
��$�k�$�$�]�4F�G��~
.�~
.�B
N
�
N
�$(
�(
r