3 ^=mc@s<dZdZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z ddlmZddlmZmZdd lmZd&d d ZGd ddeZddZddZddZddZejd'krddlZejZddZ ddZ!n eZeZ eZ!ddZ"GdddeZ#Gd d!d!e$Z%Gd"d#d#eZ&Gd$d%d%e%Z'dS)(zRefactoring framework. Used as a main program, this can refactor any number of files and/or recursively descend down directories. Imported as a module, this provides infrastructure to write your own refactoring tool. z#Guido van Rossum N)chain)drivertokenizetoken) find_root)pytreepygram) btm_matcherTcCstt|ggdg}tjj|j}g}xLttj|D]:}|jdr2|jdr2|rZ|dd}|j |ddq2W|S)zEReturn a sorted list of all available fix names in the given package.*fix_z.pyiNi) __import__ospathZdirnameZ__file__ZsortedZlistdir startswithendswithappend)Z fixer_pkgZ remove_prefixZpkgZ fixer_dirZ fix_namesnamer(/usr/lib64/python3.6/lib2to3/refactor.pyget_all_fix_namess rc@ eZdZdS) _EveryNodeN__name__ __module__ __qualname__rrrrr+rcCst|tjtjfr(|jdkr t|jhSt|tjrH|jrDt|jStt|tj rt }x*|jD] }x|D]}|j t|qlWqbW|St d|dS)zf Accepts a pytree Pattern Node and returns a set of the pattern types which will match first. Nz$Oh no! I don't understand pattern %s) Z isinstancerZ NodePatternZ LeafPatterntyperZNegatedPatternZcontent_get_head_typesZWildcardPatternsetupdate Exception)ZpatrpZxrrrr/s      rc Cstjt}g}x|D]|}|jrjyt|j}Wntk rJ|j|YqXxB|D]}||j|qRWq|jdk r||jj|q|j|qWx,tt j j j t j j D]}||j|qWt|S)z^ Accepts a list of fixers and returns a dictionary of head node type --> fixer list. N) collectionsZ defaultdictlistZpatternrrrZ _accept_typerrpython_grammarZ symbol2numbervaluestokensextendZdict)Z fixer_listZ head_nodesZeveryfixerZheadsZ node_typerrr_get_headnode_dictKs"    r,csfddtdDS)zN Return the fully qualified names for fixers in the package pkg_name. csg|]}d|qS.r).0fix_nameZpkg_namerr hz+get_fixers_from_package..F)rr1rr1rget_fixers_from_packageds r4cCs|SNr)Zobjrrr _identitykr6r cCs |jddS)Nz  )replaceinputrrr_from_system_newlinesrr7r<cCs tjdkr|jdtjS|SdS)Nr8)rZlinesepr9r:rrr_to_system_newlinests r=c sTd}tjtj|jfdd}ttjtjtj h}t }yx|\}}||krVq@q@|tj krl|rfPd}q@|tj ko||dkr,|\}}|tj ks|dkrP|\}}|tj ks|dkrP|\}}|tj kr|dkr|\}}xJ|tj kr(|j||\}}|tj ks|d krP|\}}qWq@Pq@WWntk rJYnXt|S) NFcst}|d|dfS)Nrr)Znext)ZtokZgenrradvancesz(_detect_future_features..advanceTZfromZ __future__Zimportz(z,)rgenerate_tokensioZStringIOreadline frozensetrZNEWLINEZNLZCOMMENTr ZSTRINGZNAMEZOPZaddZ StopIteration)ZsourceZhave_docstringr?ZignorefeaturesZtpvaluerr>r_detect_future_featuressD          rFc@seZdZdZdS) FixerErrorzA fixer could not be loaded.N)rrr__doc__rrrrrGrrGc@seZdZdddZdZdZd4ddZdd Zd d Zd d Z ddZ ddZ d5ddZ d6ddZ ddZd7ddZddZd8ddZddZd d!Zd9d"d#Zd:d$d%Zd&Zd'Zd(d)Zd*d+Zd,d-Zd.d/Zd0d1Zd2d3ZdS);RefactoringToolF)print_functionwrite_unchanged_filesZFixr NcCs2||_|p g|_|jj|_|dk r0|jj||jdrDtj|_ntj |_|jj d|_ g|_ t jd|_g|_d|_tj|jtj|jd|_|j\|_|_g|_tj|_g|_g|_xXt|j|jD]F}|j r|jj!|q||jkr|jj"|q||jkr|jj"|qWt#|j|_$t#|j|_%dS)zInitializer. Args: fixer_names: a list of fixers to import options: a dict with configuration. explicit: a list of fixers to run even if they are explicit. NrJrKrIF)convertlogger)&fixersexplicit_default_optionsZcopyoptionsr!r!python_grammar_no_print_statementgrammarr'getrKerrorsloggingZ getLoggerrM fixer_logwroterZDriverrrL get_fixers pre_order post_orderfilesbmZ BottomMatcherBMZ bmi_pre_orderZbmi_post_orderrZ BM_compatibleZ add_fixerrr,bmi_pre_order_headsbmi_post_order_heads)selfZ fixer_namesrQrOr+rrr__init__s<           zRefactoringTool.__init__c Cs\g}g}x&|jD]}t|iidg}|jddd}|j|jrV|t|jd}|jd}|jdjdd|D}yt ||}Wn$t k rt d ||fYnX||j |j } | jr|jd k r||jkr|jd |q|jd || jd kr|j| q| jdkr |j| qt d| jqWtjd} |j| d|j| d||fS)aInspects the options to load the requested patterns and handlers. Returns: (pre_order, post_order), where pre_order is the list of fixers that want a pre-order AST traversal, and post_order is the list that want post-order traversal. r r.rNZ_cSsg|] }|jqSr)Ztitle)r/r$rrrr2z.RefactoringTool.get_fixers..zCan't find %s.%sTzSkipping optional fixer: %szAdding transformation: %sZpreZpostzIllegal fixer order: %rZ run_orderkey)rNr Zrsplitr FILE_PREFIXlenZsplit CLASS_PREFIXjoinZgetattrZAttributeErrorrGrQrWrO log_message log_debugZorderroperatorZ attrgettersort) raZpre_order_fixersZpost_order_fixersZ fix_mod_pathZmodr0ZpartsZ class_nameZ fix_classr+Zkey_funcrrrrYs8            zRefactoringTool.get_fixerscOsdS)zCalled when an error occurs.Nr)ramsgargskwdsrrr log_errorszRefactoringTool.log_errorcGs|r ||}|jj|dS)zHook to log a message.N)rMZinforarprqrrrrlszRefactoringTool.log_messagecGs|r ||}|jj|dSr5)rMZdebugrtrrrrmszRefactoringTool.log_debugcCsdS)zTCalled with the old version, new version, and filename of a refactored file.Nr)raold_textnew_textfilenameequalrrr print_outputszRefactoringTool.print_outputcCs<x6|D].}tjj|r&|j|||q|j|||qWdS)z)Refactor a list of files and directories.N)rrZisdir refactor_dir refactor_file)raitemswrite doctests_onlyZ dir_or_filerrrrefactor#s  zRefactoringTool.refactorc Cstjd}xtj|D]\}}}|jd||j|jxH|D]@}|jd rBtjj|d|krBtjj||} |j | ||qBWdd|D|dd<qWdS)zDescends down a directory and refactor every Python file found. Python files are assumed to have a .py extension. Files and subdirectories starting with '.' are skipped. ZpyzDescending into %sr.rcSsg|]}|jds|qSr-)r)r/Zdnrrrr2>rdz0RefactoringTool.refactor_dir..N) rZextsepZwalkrmrorrZsplitextrkr{) raZdir_namer}r~Zpy_extZdirpathZdirnamesZ filenamesrZfullnamerrrrz,s    zRefactoringTool.refactor_dircCsyt|d}Wn.tk r<}z|jd||dSd}~XnXztj|jd}Wd|jXt|d|d}t|j |fSQRXdS)zG Do our best to decode a Python source file correctly. ZrbzCan't open %s: %sNrr#encodingNN) openOSErrorrsrZdetect_encodingrBclose_open_with_encodingr<read)rarwferrrrrr_read_python_source@s z#RefactoringTool._read_python_sourcecCs|j|\}}|dkrdS|d7}|rn|jd||j||}|jsL||kr`|j|||||q|jd|nH|j||}|js|r|jr|jt|dd|||dn |jd|dS) zRefactors a file.Nr8zRefactoring doctests in %szNo doctest changes in %sr)r}rzNo changes in %srg)rrmrefactor_docstringrKprocessed_filerefactor_string was_changedstr)rarwr}r~r;routputtreerrrr{Ps    zRefactoringTool.refactor_filecCst|}d|krtj|j_zJy|jj|}Wn4tk r`}z|jd||jj |dSd}~XnXWd|j|j_X||_ |j d||j |||S)aFRefactor a given input string. Args: data: a string holding the code to be refactored. name: a human-readable name for use in error/log messages. Returns: An AST corresponding to the refactored input stream; None if there were errors during the parse. rJzCan't parse %s: %s: %sNzRefactoring %s) rFrrRrrSZ parse_stringr"rs __class__rfuture_featuresrm refactor_tree)raZdatarrDrrrrrrgs     zRefactoringTool.refactor_stringcCstjj}|rN|jd|j|d}|js2||krB|j|d|q|jdn:|j|d}|jsj|r~|jr~|jt |d|n |jddS)NzRefactoring doctests in stdinzzNo doctest changes in stdinzNo changes in stdin) sysZstdinrrmrrKrrrr)rar~r;rrrrrrefactor_stdins     zRefactoringTool.refactor_stdinc Csx"t|j|jD]}|j||qW|j|j|j|j|j|j|jj|j }xvt |j rАx`|jj D]R}||ko||rv||j tjjdd|jr||j tjjdx t||D]}|||kr||j|y t|Wntk rwYnX|jr(||jkr(q|j|}|r|j||}|dk r|j|x,|jD] }|jspg|_|jj|q^W|jj|j }x2|D]*} | |krg|| <|| j|| qWqWqvWq\Wx$t|j|jD]}|j||qW|jS)aRefactors a parse tree (modifying the tree in place). For compatible patterns the bottom matcher module is used. Otherwise the tree is traversed node-to-node for matches. Args: tree: a pytree.Node instance representing the root of the tree to be refactored. name: a human-readable name for this tree. Returns: True if the tree was modified, False otherwise. T)rfZreversereN)rrZr[Z start_tree traverse_byr_r`r^ZrunZleavesZanyr(rNrorZBaseZdepthZkeep_line_orderZ get_linenor&ZremoverZ ValueErrorZfixers_appliedmatch transformr9rr*Z finish_treer) rarrr+Z match_setnoderesultsnewZ new_matchesZfxrrrrrsJ       $zRefactoringTool.refactor_treecCs^|sdSxP|D]H}xB||jD]4}|j|}|r|j||}|dk r|j||}qWqWdS)aTraverse an AST, applying a set of fixers to each node. This is a helper method for refactor_tree(). Args: fixers: a list of fixer instances. traversal: a generator that yields AST nodes. Returns: None N)rrrr9)rarNZ traversalrr+rrrrrrs     zRefactoringTool.traverse_bycCs|jj||dkr.|j|d}|dkr.dS||k}|j|||||r`|jd||js`dS|rv|j||||n |jd|dS)zR Called when a file has been refactored and there may be changes. NrzNo changes to %szNot writing changes to %s)r\rrryrmrK write_file)rarvrwrur}rrxrrrrs  zRefactoringTool.processed_filec%Csyt|d|d}Wn.tk r@}z|jd||dSd}~XnXzHy|jt|Wn0tk r}z|jd||WYdd}~XnXWd|jX|jd|d|_dS)zWrites a string to a file. It first shows a unified diff between the old text and the new text, and then rewrites the file; the latter is only done if the write option is set. ZwrzCan't create %s: %sNzCan't write %s: %szWrote changes to %sT)rrrsr}r=rrmrX)rarvrwrurrrrrrr s$  zRefactoringTool.write_filez>>> z... c Csg}d}d}d}d}x|jddD]}|d7}|jj|jr|dk r\|j|j|||||}|g}|j|j} |d| }q"|dk r|j||js|||jjdkr|j |q"|dk r|j|j||||d}d}|j |q"W|dk r|j|j||||dj |S)aRefactors a docstring, looking for doctests. This returns a modified version of the input string. It looks for doctests, which start with a ">>>" prompt, and may be continued with "..." prompts, as long as the "..." is indented the same as the ">>>". (Unfortunately we can't use the doctest module's parser, since, like most parsers, it is not geared towards preserving the original source.) NrTZkeependsrr8rc) splitlinesZlstriprPS1r*refactor_doctestZfindPS2rstriprrk) rar;rwZresultblockZ block_linenoindentlinenolineirrrr%s:          z"RefactoringTool.refactor_docstringc syj||}Wndtk rv}zHjjtjrRx|D]}jd|jdq8Wjd|||j j ||Sd}~XnXj ||r t |j dd}|d|d||dd} }|d jds|d d7<j|jdg}|r |fd d |D7}|S) zRefactors one doctest. A doctest is given as a block of lines, the first of which starts with ">>>" (possibly indented), while the remaining lines start with "..." (identically indented). z Source: %sr8z+Can't parse docstring in %s line %s: %s: %sNTrrrcsg|]}j|qSr)r)r/rrrarrr2jrdz4RefactoringTool.refactor_doctest..rgrg) parse_blockr"rMZ isEnabledForrVZDEBUGrmrrsrrrrrrrZpop) rarrrrwrrrrZclippedrrrrPs$ "z RefactoringTool.refactor_doctestcCs|jr d}nd}|js$|jd|n&|jd|x|jD]}|j|q8W|jrt|jdx|jD]}|j|qbW|jrt|jdkr|jdn|jdt|jx&|jD]\}}}|j|f||qWdS) Nwerez need to bezNo files %s modified.zFiles that %s modified:z$Warnings/messages while refactoring:rzThere was 1 error:zThere were %d errors:)rXr\rlrWrUri)rarZfileZmessagerprqrrrrr summarizems$     zRefactoringTool.summarizecCs"|jj|j|||}t|_|S)zParses a block into a tree. This is necessary to get correct line number / offset information in the parser diagnostics and embedded into the parse tree. )rZ parse_tokens wrap_toksrCr)rarrrrrrrrszRefactoringTool.parse_blockc cshtj|j||j}xN|D]F\}}\}}\} } } ||d7}| |d7} ||||f| | f| fVqWdS)z;Wraps a tokenize stream to systematically modify start/end.rN)rr@ gen_linesZ__next__) rarrrr)rrEZline0Zcol0Zline1Zcol1Z line_textrrrrs   zRefactoringTool.wrap_toksccs||j}||j}|}xV|D]N}|j|r@|t|dVn(||jdkrXdVntd||f|}qWx dVqrWdS)zGenerates lines as expected by tokenize from a list of lines. This strips the first len(indent + self.PS1) characters off each line. Nr8zline=%r, prefix=%rrc)rrrrirZAssertionError)rarrZprefix1Zprefix2Zprefixrrrrrs    zRefactoringTool.gen_linesrFFrr)F)NFNr5)rrrrPrjrhrbrYrsrlrmryrrzrr{rrrrrrrrrrrrrrrrrrrIs: 4(   O  + rIc@r)MultiprocessingUnsupportedNrrrrrrrrcsBeZdZfddZd fdd ZfddZfd d ZZS) MultiprocessRefactoringToolcs"tt|j||d|_d|_dSr5)superrrbqueue output_lockrarqkwargsrrrrbsz$MultiprocessRefactoringTool.__init__Frcs|dkrttj|||Sy ddlWntk r@tYnXjdk rTtdj_j _ fddt |D}z.x|D] }|j qWttj|||Wdjj xt |D]}jjdqWx|D]}|jr|j qWd_XdS)Nrrz already doing multiple processescsg|]}jjdqS))Ztarget)ZProcess_child)r/rmultiprocessingrarrr2r3z8MultiprocessRefactoringTool.refactor..)rrrrZ ImportErrorrrZ RuntimeErrorZ JoinableQueueZLockrZrangeZstartrkputZis_alive)rar|r}r~Z num_processesZ processesr$rrrrrs2               z$MultiprocessRefactoringTool.refactorc sR|jj}xB|dk rL|\}}ztt|j||Wd|jjX|jj}q WdSr5)rrTrrr{Z task_done)raZtaskrqrrrrrs     z"MultiprocessRefactoringTool._childcs2|jdk r|jj||fntt|j||SdSr5)rrrrr{rrrrr{s  z)MultiprocessRefactoringTool.refactor_file)FFr)rrrrbrrr{Z __classcell__rrrrrs   r)T)r r)(rHZ __author__rrrVrnr%rAZ itertoolsrZpgen2rrrZ fixer_utilrrcrrr r]rr"rrr,r4r6Z version_infoZcodecsrrr<r=rFrGZobjectrIrrrrrrZ sF      (