o
    ˷e`%                     @   s   d dl Z d dlZd dlZd dlZd dlZddlmZmZmZm	Z	 d dl
Z
d dlmZ ddlmZ d dlmZmZmZmZmZmZmZmZ d dl mZ erRddlmZ G d	d
 d
eZdS )    N   )Tokenmatch_tokenis_non_coding_tokenpatched_generate_tokens)xrange)LineNumbers)CallableIteratorListOptionalTupleAnycastTYPE_CHECKING)Module)AstNodec                   @   s   e Zd ZdZd'ddZdd Zd	d
 Zedd Zedd Z	edd Z
edd Zdd Zdd Zdd Zd(ddZd(ddZd)ddZ	d(dd Zd(d!d"Zd#d$ Zd%d& ZdS )*	ASTTokensa  
  ASTTokens maintains the text of Python code in several forms: as a string, as line numbers, and
  as tokens, and is used to mark and access token and position information.

  ``source_text`` must be a unicode or UTF8-encoded string. If you pass in UTF8 bytes, remember
  that all offsets you'll get are to the unicode text, which is available as the ``.text``
  property.

  If ``parse`` is set, the ``source_text`` will be parsed with ``ast.parse()``, and the resulting
  tree marked with token info and made available as the ``.tree`` property.

  If ``tree`` is given, it will be marked and made available as the ``.tree`` property. In
  addition to the trees produced by the ``ast`` module, ASTTokens will also mark trees produced
  using ``astroid`` library <https://www.astroid.org>.

  If only ``source_text`` is given, you may use ``.mark_tokens(tree)`` to mark the nodes of an AST
  tree created separately.
  FN	<unknown>c                 C   sr   || _ |rt||n|| _t|}|| _t|| _t	| 
|| _dd | jD | _| jr7| | j d S d S )Nc                 S   s   g | ]}|j qS  )startpos).0tokr   r   J/var/www/ideatree/venv/lib/python3.10/site-packages/asttokens/asttokens.py
<listcomp>F   s    z&ASTTokens.__init__.<locals>.<listcomp>)	_filenameastparse_treesixensure_text_textr   _line_numberslist_generate_tokens_tokens_token_offsetsmark_tokens)selfsource_textr   treefilenamer   r   r   __init__2   s   

zASTTokens.__init__c                 C   s   ddl m} || | dS )ap  
    Given the root of the AST or Astroid tree produced from source_text, visits all nodes marking
    them with token and position information by adding ``.first_token`` and
    ``.last_token``attributes. This is done automatically in the constructor when ``parse`` or
    ``tree`` arguments are set, but may be used manually with a separate AST or Astroid tree.
    r   )
MarkTokensN)r'   r-   
visit_tree)r(   	root_noder-   r   r   r   r'   L   s   	zASTTokens.mark_tokensc           
      c   s    t ttg tf t|j}tt	|D ])\}}|\}}}}}	t
|||||	|| j|d |d | j|d |d V  qdS )z.
    Generates tokens for the given code.
    r   r   N)tokenizegenerate_tokensr   r	   strioStringIOreadline	enumerater   r   r"   line_to_offset)
r(   textoriginal_tokensindexr   tok_typetok_strstartendliner   r   r   r$   Y   s    zASTTokens._generate_tokensc                 C      | j S )z,The source code passed into the constructor.)r!   r(   r   r   r   r8   h      zASTTokens.textc                 C   r@   )zIThe list of tokens corresponding to the source code from the constructor.)r%   rA   r   r   r   tokensn   rB   zASTTokens.tokensc                 C   r@   )zTThe root of the AST tree passed into the constructor or parsed from the source code.)r   rA   r   r   r   r*   t   rB   zASTTokens.treec                 C   r@   )zThe filename that was parsed)r   rA   r   r   r   r+   z   rB   zASTTokens.filenamec                 C   s   | j t| j|d  S )z
    Returns the token containing the given character offset (0-based position in source text),
    or the preceeding token if the position is between tokens.
    r   )r%   bisectr&   )r(   offsetr   r   r   get_token_from_offset   s   zASTTokens.get_token_from_offsetc                 C   s   |  | j||S )z
    Returns the token containing the given (lineno, col_offset) position, or the preceeding token
    if the position is between tokens.
    )rF   r"   r7   r(   lineno
col_offsetr   r   r   	get_token   s   	zASTTokens.get_tokenc                 C   s   |  || j||S )zd
    Same as get_token(), but interprets col_offset as a UTF8 offset, which is what `ast` uses.
    )rJ   r"   from_utf8_colrG   r   r   r   get_token_from_utf8   s   zASTTokens.get_token_from_utf8c                 C   s@   |j d }|st| j| jr|d7 }t| j| js| j| S )z
    Returns the next token after the given one. If include_extra is True, includes non-coding
    tokens from the tokenize module, such as NL and COMMENT.
    r   r:   r   r%   typer(   r   include_extrair   r   r   
next_token      

zASTTokens.next_tokenc                 C   s@   |j d }|st| j| jr|d8 }t| j| js| j| S )z
    Returns the previous token before the given one. If include_extra is True, includes non-coding
    tokens from the tokenize module, such as NL and COMMENT.
    r   rM   rO   r   r   r   
prev_token   rS   zASTTokens.prev_tokenc                 C   sT   |}|r| j n| j}t|||s(t|js(||dd}t|||s(t|jr|S )z
    Looks for the first token, starting at start_token, that matches tok_type and, if given, the
    token string. Searches backwards if reverse is True. Returns ENDMARKER token if not found (you
    can check it with `token.ISEOF(t.type)`.
    TrP   )rT   rR   r   tokenISEOFrN   )r(   start_tokenr;   r<   reversetadvancer   r   r   
find_token   s   zASTTokens.find_tokenc                 c   s>    t |j|jd D ]}|st| j| js| j| V  q
dS )z
    Yields all tokens in order from first_token through and including last_token. If
    include_extra is True, includes non-coding tokens such as tokenize.NL and .COMMENT.
    r   N)r   r:   r   r%   rN   )r(   first_token
last_tokenrP   rQ   r   r   r   token_range   s   
zASTTokens.token_rangec                 C   s   | j |j|j|dS )z
    Yields all tokens making up the given node. If include_extra is True, includes non-coding
    tokens such as tokenize.NL and .COMMENT.
    rU   )r_   r]   r^   )r(   noderP   r   r   r   
get_tokens   s   zASTTokens.get_tokensc                 C   sN   t |dsdS |jj}tdd | |D r!| jdd|d }||jjfS )z
    After mark_tokens() has been called, returns the (startpos, endpos) positions in source text
    corresponding to the given node. Returns (0, 0) for nodes (like `Load`) that don't correspond
    to any particular text.
    r]   )r   r   c                 s   s    | ]	}t |tjV  qd S )N)r   rV   NEWLINE)r   rZ   r   r   r   	<genexpr>   s    z+ASTTokens.get_text_range.<locals>.<genexpr>
r   r   )	hasattrr]   r   anyra   r!   rfindr^   endpos)r(   r`   r=   r   r   r   get_text_range   s   
zASTTokens.get_text_rangec                 C   s   |  |\}}| j|| S )z
    After mark_tokens() has been called, returns the text corresponding to the given node. Returns
    '' for nodes (like `Load`) that don't correspond to any particular text.
    )ri   r!   )r(   r`   r=   r>   r   r   r   get_text   s   zASTTokens.get_text)FNr   )F)NF)__name__
__module____qualname____doc__r,   r'   r$   propertyr8   rC   r*   r+   rF   rJ   rL   rR   rT   r\   r_   ra   ri   rj   r   r   r   r   r      s0    









r   )r   rD   r3   rV   r0   utilr   r   r   r   r   	six.movesr   line_numbersr   typingr	   r
   r   r   r   r   r   r   r   r   objectr   r   r   r   r   <module>   s   (