3
\=m @ s8 d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
m Z m Z ddl
mZ ddlmZmZ ddlmZ d%d
d ZG d d
d
eZdd Zdd Zdd Zdd Zejd&k rddl Z e j Z dd Z dd Z n e Z eZ eZ dd Z!G d d d eZ"G d d d e#Z$G d!d" d"eZ%G d#d$ d$e$Z&dS )'z#Guido van Rossum N)chain )drivertokenizetoken) find_root)pytreepygram) btm_matcherTc C st t | g g dg}tjj|j}g }xLttj|D ]:}|jdr2|jdr2|rZ|dd }|j |d d q2W |S )N*fix_z.py )
__import__ospathdirname__file__sortedlistdir
startswithendswithappend)Z fixer_pkgZ
remove_prefixZpkgZ fixer_dirZ fix_namesname r (/usr/lib64/python3.6/lib2to3/refactor.pyget_all_fix_names s r c @ s e Zd ZdS )
_EveryNodeN)__name__
__module__ __qualname__r r r r r + s r c C s t | tjtjfr(| jd kr t| jhS t | tjrH| jrDt| jS tt | tj rt
}x*| jD ] }x|D ]}|j t| qlW qbW |S t d| d S )Nz$Oh no! I don't understand pattern %s)
isinstancer Z NodePatternZ LeafPatterntyper ZNegatedPatternZcontent_get_head_typesZWildcardPatternsetupdate Exception)Zpatrpxr r r r$ / s
r$ c C s t jt}g }x| D ]|}|jrjyt|j}W n tk
rJ |j| Y qX xB|D ]}|| j| qRW q|jd k r||j j| q|j| qW x,tt j
j j t j
j
D ]}|| j| qW t|S )N) collections defaultdictlistpatternr$ r r Z _accept_typer r python_grammarZ
symbol2numbervaluestokensextenddict)Z
fixer_listZ
head_nodesZeveryfixerZheadsZ node_typer r r _get_headnode_dictK s"
r5 c s fddt dD S )Nc s g | ]} d | qS ).r ).0fix_name)pkg_namer r
h s z+get_fixers_from_package..F)r )r9 r )r9 r get_fixers_from_packaged s
r; c C s | S )Nr )objr r r _identityk s r= r c C s | j ddS )Nz
)replace)inputr r r _from_system_newlinesr s rA c C s t jdkr| jdt jS | S d S )Nr> )r linesepr? )r@ r r r _to_system_newlinest s
rC c sT d}t jtj| j fdd}ttjt jtj h}t
}yx| \}}||krVq@q@|tj krl|rfP d}q@|tj ko||dkr,| \}}|tj ks|dkrP | \}}|tj ks|dkrP | \}}|tj
kr|dkr| \}}xJ|tj kr(|j| | \}}|tj
ks|d kr P | \}}qW q@P q@W W n tk
rJ Y nX t|S )
NFc s t } | d | d fS )Nr r )next)tok)genr r advance s z(_detect_future_features..advanceTfromZ
__future__import(,)r generate_tokensioStringIOreadline frozensetr NEWLINENLCOMMENTr% STRINGNAMEOPadd
StopIteration)sourceZhave_docstringrG ignorefeaturestpvaluer )rF r _detect_future_features sD
r^ c @ s e Zd ZdS )
FixerErrorN)r r r! r r r r r_ s r_ c @ s e Zd ZdddZdZdZd4ddZdd Zd
d Zd d
Z dd Z
dd Z d5ddZ d6ddZ
dd Zd7ddZdd Zd8d d Zd d Zd d! Zd9d"d#Zd:d$d%Zd&Zd'Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Z d2d3 Z dS );RefactoringToolF)print_functionwrite_unchanged_filesZFixr Nc C s2 || _ |p g | _| jj | _|d k r0| jj| | jd rDtj| _ntj | _| jj
d| _ g | _ t
jd| _g | _d| _tj| jtj| jd| _| j \| _| _g | _tj | _ g | _ g | _ xXt | j| jD ]F}|j r| j j!| q|| jkr| j j"| q|| jkr| j j"| qW t#| j | _$t#| j | _%d S )Nra rb r` F)convertlogger)&fixersexplicit_default_optionscopyoptionsr& r !python_grammar_no_print_statementgrammarr/ getrb errorsloggingZ getLoggerrd fixer_logwroter ZDriverr rc
get_fixers pre_order
post_orderfilesbmZ
BottomMatcherBMZ
bmi_pre_orderZbmi_post_orderr Z
BM_compatibleZ add_fixerr r5 bmi_pre_order_headsbmi_post_order_heads)selfZ fixer_namesri rf r4 r r r __init__ s<
zRefactoringTool.__init__c C s\ g }g }x&| j D ]}t|i i dg}|jddd }|j| jrV|t| jd }|jd}| jdjdd |D }yt ||}W n$ t
k
r t d||f Y nX || j | j
} | jr| jd k r|| jkr| jd
| q| jd | | jd kr|j| q| jd
kr |j| qt d| j qW tjd}
|j|
d |j|
d ||fS )Nr r6 r _ c S s g | ] }|j qS r )title)r7 r) r r r r: s z.RefactoringTool.get_fixers..zCan't find %s.%sTzSkipping optional fixer: %szAdding transformation: %sZpreZpostzIllegal fixer order: %rZ run_order)key)re r rsplitr FILE_PREFIXlensplit CLASS_PREFIXjoingetattrAttributeErrorr_ ri ro rf log_message log_debugorderr operator
attrgettersort) ry Zpre_order_fixersZpost_order_fixersZ fix_mod_pathmodr8 parts
class_nameZ fix_classr4 Zkey_funcr r r rq s8
zRefactoringTool.get_fixersc O s d S )Nr )ry msgargskwdsr r r log_error s zRefactoringTool.log_errorc G s |r || }| j j| d S )N)rd info)ry r r r r r r s zRefactoringTool.log_messagec G s |r || }| j j| d S )N)rd debug)ry r r r r r r s zRefactoringTool.log_debugc C s d S )Nr )ry old_textnew_textfilenameequalr r r print_output s z RefactoringTool.print_outputc C s< x6|D ].}t jj|r&| j||| q| j||| qW d S )N)r r isdir refactor_dir
refactor_file)ry itemswrite
doctests_onlyZ dir_or_filer r r refactor# s
zRefactoringTool.refactorc
C s t jd }xt j|D ]\}}}| jd| |j |j xH|D ]@}|jd rBt jj|d |krBt jj||} | j | || qBW dd |D |d d < qW d S )NpyzDescending into %sr6 r c S s g | ]}|j d s|qS )r6 )r )r7 Zdnr r r r: > s z0RefactoringTool.refactor_dir..)
r extsepwalkr r r r splitextr r )
ry Zdir_namer r Zpy_extdirpathZdirnames filenamesr fullnamer r r r , s
z RefactoringTool.refactor_dirc C s yt |d}W n. tk
r< } z| jd|| dS d }~X nX ztj|jd }W d |j X t|d|d}t|j |fS Q R X d S )NrbzCan't open %s: %sr r( )encoding)NN)
openOSErrorr r detect_encodingrO close_open_with_encodingrA read)ry r ferrr r r r _read_python_source@ s
z#RefactoringTool._read_python_sourcec C s | j |\}}|d krd S |d7 }|rn| jd| | j||}| jsL||kr`| j||||| q| jd| nH| j||}| js|r|jr| jt|d d |||d n | jd| d S )Nr> zRefactoring doctests in %szNo doctest changes in %sr )r r zNo changes in %sr )r r refactor_docstringrb processed_filerefactor_string was_changedstr)ry r r r r@ r outputtreer r r r P s z RefactoringTool.refactor_filec C s t |}d|krtj| j_zJy| jj|}W n4 tk
r` } z| jd||jj | d S d }~X nX W d | j| j_X ||_
| j d| | j || |S )Nra zCan't parse %s: %s: %szRefactoring %s)
r^ r rj r rk Z parse_stringr' r __class__r future_featuresr
refactor_tree)ry datar r[ r r r r r r g s
z RefactoringTool.refactor_stringc C s t jj }|rN| jd | j|d}| js2||krB| j|d| q| jd n:| j|d}| jsj|r~|jr~| jt |d| n
| jd d S )Nz Refactoring doctests in stdinzzNo doctest changes in stdinzNo changes in stdin)
sysstdinr r r rb r r r r )ry r r@ r r r r r refactor_stdin s
z RefactoringTool.refactor_stdinc
C s x"t | j| jD ]}|j|| qW | j| j|j | j| j|j | jj|j }xvt
|j rАx`| jj D ]R}||ko|| rv|| j
tjjdd |jr|| j
tjjd x
t|| D ]}||| kr|| j| y t| W n tk
r wY nX |jr(||jkr(q|j|}|r|j||}|d k r|j| x,|j D ] }|jspg |_|jj| q^W | jj|j }x2|D ]*} | |krg || < || j || qW qW qvW q\W x$t | j| jD ]}|j || qW |j S )NT)r~ reverse)r~ ) r rr rs Z
start_tree traverse_byrw rx rv ZrunZleavesanyr0 re r r ZBaseZdepthZkeep_line_orderZ
get_linenor- remover
ValueErrorZfixers_appliedmatch transformr? r r2 Z finish_treer )
ry r r r4 Z match_setnoderesultsnewZ new_matchesZfxrr r r r sJ
$z RefactoringTool.refactor_treec C s^ |sd S xP|D ]H}xB||j D ]4}|j|}|r |j||}|d k r |j| |}q W qW d S )N)r# r r r? )ry re Z traversalr r4 r r r r r r s
zRefactoringTool.traverse_byc C s | j j| |d kr.| j|d }|d kr.d S ||k}| j|||| |r`| jd| | js`d S |rv| j|||| n | jd| d S )Nr zNo changes to %szNot writing changes to %s)rt r r r r rb
write_file)ry r r r r r r r r r r s z RefactoringTool.processed_filec % C s yt |d|d}W n. tk
r@ } z| jd|| d S d }~X nX zHy|jt| W n0 tk
r } z| jd|| W Y d d }~X nX W d |j X | jd| d| _d S )Nw)r zCan't create %s: %szCan't write %s: %szWrote changes to %sT)r r r r rC r r rp )ry r r r r r r r r r r
s $
zRefactoringTool.write_filez>>> z... c
C s g }d }d }d }d}x|j ddD ]}|d7 }|j j| jr|d k r\|j| j|||| |}|g}|j| j} |d | }q"|d k r|j|| j s||| jj d kr|j | q"|d k r|j| j|||| d }d }|j | q"W |d k r|j| j|||| dj
|S )Nr T)keependsr r> r| )
splitlineslstripr PS1r2 refactor_doctestfindPS2rstripr r )
ry r@ r resultblockZ block_linenoindentlinenolineir r r r % s:
z"RefactoringTool.refactor_docstringc
s yj || }W nd tk
rv } zHjjtjrRx |D ]}jd|jd q8W jd|||j j
| |S d }~X nX j ||r
t |j
dd}|d |d ||d d } }|d
jds|d d7 < j |jd g}|r
| fdd |D 7 }|S ) Nz
Source: %sr> z+Can't parse docstring in %s line %s: %s: %sT)r r r c s g | ]} j | qS r )r )r7 r )r ry r r r: j s z4RefactoringTool.refactor_doctest..r r ) parse_blockr' rd Z isEnabledForrn DEBUGr r r r r r r r r r pop)
ry r r r r r r r r Zclippedr )r ry r r P s$
"z RefactoringTool.refactor_doctestc C s | j r d}nd}| js$| jd| n&| jd| x| jD ]}| j| q8W | jrt| jd x| jD ]}| j| qbW | jrt| jdkr| jd n| jdt| j x&| jD ] \}}}| j|f|| qW d S ) Nwerez
need to bezNo files %s modified.zFiles that %s modified:z$Warnings/messages while refactoring:r zThere was 1 error:zThere were %d errors:)rp rt r ro rm r )ry r filemessager r r r r r summarizem s$
zRefactoringTool.summarizec C s" | j j| j|||}t |_|S )N)r Z parse_tokens wrap_toksrP r )ry r r r r r r r r s zRefactoringTool.parse_blockc c sh t j| j||j}xN|D ]F\}}\}}\} }
} ||d 7 }| |d 7 } ||||f| |
f| fV qW d S )Nr )r rL gen_lines__next__) ry r r r r1 r# r] Zline0Zcol0Zline1Zcol1Z line_textr r r r s
zRefactoringTool.wrap_toksc c s || j }|| j }|}xV|D ]N}|j|r@|t|d V n(||j d krXdV ntd||f |}q W x
dV qrW d S )Nr> zline=%r, prefix=%rr| )r r r r r AssertionError)ry r r prefix1Zprefix2prefixr r r r r s
zRefactoringTool.gen_lines)NN)FF)FF)FF)F)NFN)N) r r r! rg r r rz rq r r r r r r r r r r r r r r r r r r r r r r r r r r r` s:
4(
O
+
r` c @ s e Zd ZdS )MultiprocessingUnsupportedN)r r r! r r r r r s r c sB e Zd Z fddZd fdd Z fddZ fd d
Z ZS ) MultiprocessRefactoringToolc s" t t| j|| d | _d | _d S )N)superr rz queue output_lock)ry r kwargs)r r r rz s z$MultiprocessRefactoringTool.__init__Fr c s |dkr t tj|||S y dd l W n tk
r@ tY nX jd k rTtd j _ j _
fddt |D }z.x|D ] }|j qW t tj||| W d jj
x t |D ]}jjd qW x |D ]}|j r|j
qW d _X d S )Nr r z already doing multiple processesc s g | ]} j jd qS ))target)ZProcess_child)r7 r )multiprocessingry r r r: s z8MultiprocessRefactoringTool.refactor..)r r r r ImportErrorr r RuntimeErrorZ
JoinableQueueZLockr rangestartr putZis_alive)ry r r r Z
num_processesZ processesr) r )r )r ry r r s2
z$MultiprocessRefactoringTool.refactorc
sR | j j }xB|d k rL|\}}ztt| j|| W d | j j X | j j }q W d S )N)r rl r r r Z task_done)ry Ztaskr r )r r r r s
z"MultiprocessRefactoringTool._childc s2 | j d k r | j j||f ntt| j||S d S )N)r r r r r )ry r r )r r r r s
z)MultiprocessRefactoringTool.refactor_file)FFr )r r r! rz r r r
__classcell__r r )r r r s
r )T)r r )'
__author__r r rn r r+ rM itertoolsr Zpgen2r r r Z
fixer_utilr r| r r r
ru r r' r r$ r5 r; r= version_infocodecsr r rA rC r^ r_ objectr` r r r r r r sD
(