Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 58f23ff

Browse files
gpsheadgpshead
committed
Issue #13930: Adds ability for 2to3 to write its output to a different
directory tree instead of overwriting the input files. Adds three command line options: -o/--output-dir, -W/--write-unchanged-files and --add-suffix. Feature backports into stable release branches for 2to3 are allowed by a special exemption: http://mail.python.org/pipermail/python-dev/2011-December/115089.html
1 parent 51d04d1 commit 58f23ff

6 files changed

Lines changed: 270 additions & 14 deletions

File tree

‎Doc/library/2to3.rst‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,38 @@ change can also be enabled manually with the :option:`-p` flag. Use
9494
:option:`-p` to run fixers on code that already has had its print statements
9595
converted.
9696

97+
The :option:`-o` or :option:`--output-dir` option allows specification of an
98+
alternate directory for processed output files to be written to. The
99+
:option:`-n` flag is required when using this as backup files do not make sense
100+
when not overwriting the input files.
101+
102+
.. versionadded:: 3.2.3
103+
The :option:`-o` option was added.
104+
105+
The :option:`-W` or :option:`--write-unchanged-files` flag tells 2to3 to always
106+
write output files even if no changes were required to the file. This is most
107+
useful with :option:`-o` so that an entire Python source tree is copied with
108+
translation from one directory to another.
109+
This option implies the :option:`-w` flag as it would not make sense otherwise.
110+
111+
.. versionadded:: 3.2.3
112+
The :option:`-W` flag was added.
113+
114+
The :option:`--add-suffix` option specifies a string to append to all output
115+
filenames. The :option:`-n` flag is required when specifying this as backups
116+
are not necessary when writing to different filenames. Example::
117+
118+
$ 2to3 -n -W --add-suffix=3 example.py
119+
120+
Will cause a converted file named ``example.py3`` to be written.
121+
122+
.. versionadded:: 3.2.3
123+
The :option:`--add-suffix` option was added.
124+
125+
To translate an entire project from one directory tree to another use::
126+
127+
$ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode
128+
97129

98130
.. _2to3-fixers:
99131

‎Lib/lib2to3/main.py‎

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,65 @@ def diff_texts(a, b, filename):
2525

2626
class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
2727
"""
28+
A refactoring tool that can avoid overwriting its input files.
2829
Prints output to stdout.
30+
31+
Output files can optionally be written to a different directory and or
32+
have an extra file suffix appended to their name for use in situations
33+
where you do not want to replace the input files.
2934
"""
3035

31-
def __init__(self, fixers, options, explicit, nobackups, show_diffs):
36+
def __init__(self, fixers, options, explicit, nobackups, show_diffs,
37+
input_base_dir='', output_dir='', append_suffix=''):
38+
"""
39+
Args:
40+
fixers: A list of fixers to import.
41+
options: A dict with RefactoringTool configuration.
42+
explicit: A list of fixers to run even if they are explicit.
43+
nobackups: If true no backup '.bak' files will be created for those
44+
files that are being refactored.
45+
show_diffs: Should diffs of the refactoring be printed to stdout?
46+
input_base_dir: The base directory for all input files. This class
47+
will strip this path prefix off of filenames before substituting
48+
it with output_dir. Only meaningful if output_dir is supplied.
49+
All files processed by refactor() must start with this path.
50+
output_dir: If supplied, all converted files will be written into
51+
this directory tree instead of input_base_dir.
52+
append_suffix: If supplied, all files output by this tool will have
53+
this appended to their filename. Useful for changing .py to
54+
.py3 for example by passing append_suffix='3'.
55+
"""
3256
self.nobackups = nobackups
3357
self.show_diffs = show_diffs
58+
if input_base_dir and not input_base_dir.endswith(os.sep):
59+
input_base_dir += os.sep
60+
self._input_base_dir = input_base_dir
61+
self._output_dir = output_dir
62+
self._append_suffix = append_suffix
3463
super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
3564

3665
def log_error(self, msg, *args, **kwargs):
3766
self.errors.append((msg, args, kwargs))
3867
self.logger.error(msg, *args, **kwargs)
3968

4069
def write_file(self, new_text, filename, old_text, encoding):
70+
orig_filename = filename
71+
if self._output_dir:
72+
if filename.startswith(self._input_base_dir):
73+
filename = os.path.join(self._output_dir,
74+
filename[len(self._input_base_dir):])
75+
else:
76+
raise ValueError('filename %s does not start with the '
77+
'input_base_dir %s' % (
78+
filename, self._input_base_dir))
79+
if self._append_suffix:
80+
filename += self._append_suffix
81+
if orig_filename != filename:
82+
output_dir = os.path.dirname(filename)
83+
if not os.path.isdir(output_dir):
84+
os.makedirs(output_dir)
85+
self.log_message('Writing converted %s to %s.', orig_filename,
86+
filename)
4187
if not self.nobackups:
4288
# Make backup
4389
backup = filename + ".bak"
@@ -55,6 +101,9 @@ def write_file(self, new_text, filename, old_text, encoding):
55101
write(new_text, filename, old_text, encoding)
56102
if not self.nobackups:
57103
shutil.copymode(backup, filename)
104+
if orig_filename != filename:
105+
# Preserve the file mode in the new output directory.
106+
shutil.copymode(orig_filename, filename)
58107

59108
def print_output(self, old, new, filename, equal):
60109
if equal:
@@ -113,11 +162,33 @@ def main(fixer_pkg, args=None):
113162
help="Write back modified files")
114163
parser.add_option("-n", "--nobackups", action="store_true", default=False,
115164
help="Don't write backups for modified files")
165+
parser.add_option("-o", "--output-dir", action="store", type="str",
166+
default="", help="Put output files in this directory "
167+
"instead of overwriting the input files. Requires -n.")
168+
parser.add_option("-W", "--write-unchanged-files", action="store_true",
169+
help="Also write files even if no changes were required"
170+
" (useful with --output-dir); implies -w.")
171+
parser.add_option("--add-suffix", action="store", type="str", default="",
172+
help="Append this string to all output filenames."
173+
" Requires -n if non-empty. "
174+
"ex: --add-suffix='3' will generate .py3 files.")
116175

117176
# Parse command line arguments
118177
refactor_stdin = False
119178
flags = {}
120179
options, args = parser.parse_args(args)
180+
if options.write_unchanged_files:
181+
flags["write_unchanged_files"] = True
182+
if not options.write:
183+
warn("--write-unchanged-files/-W implies -w.")
184+
options.write = True
185+
# If we allowed these, the original files would be renamed to backup names
186+
# but not replaced.
187+
if options.output_dir and not options.nobackups:
188+
parser.error("Can't use --output-dir/-o without -n.")
189+
if options.add_suffix and not options.nobackups:
190+
parser.error("Can't use --add-suffix without -n.")
191+
121192
if not options.write and options.no_diffs:
122193
warn("not writing files and not printing diffs; that's not very useful")
123194
if not options.write and options.nobackups:
@@ -143,6 +214,7 @@ def main(fixer_pkg, args=None):
143214
# Set up logging handler
144215
level = logging.DEBUG if options.verbose else logging.INFO
145216
logging.basicConfig(format='%(name)s: %(message)s', level=level)
217+
logger = logging.getLogger('lib2to3.main')
146218

147219
# Initialize the refactoring tool
148220
avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
@@ -159,8 +231,23 @@ def main(fixer_pkg, args=None):
159231
else:
160232
requested = avail_fixes.union(explicit)
161233
fixer_names = requested.difference(unwanted_fixes)
162-
rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),
163-
options.nobackups, not options.no_diffs)
234+
input_base_dir = os.path.commonprefix(args)
235+
if (input_base_dir and not input_base_dir.endswith(os.sep)
236+
and not os.path.isdir(input_base_dir)):
237+
# One or more similar names were passed, their directory is the base.
238+
# os.path.commonprefix() is ignorant of path elements, this corrects
239+
# for that weird API.
240+
input_base_dir = os.path.dirname(input_base_dir)
241+
if options.output_dir:
242+
input_base_dir = input_base_dir.rstrip(os.sep)
243+
logger.info('Output in %r will mirror the input directory %r layout.',
244+
options.output_dir, input_base_dir)
245+
rt = StdoutRefactoringTool(
246+
sorted(fixer_names), flags, sorted(explicit),
247+
options.nobackups, not options.no_diffs,
248+
input_base_dir=input_base_dir,
249+
output_dir=options.output_dir,
250+
append_suffix=options.add_suffix)
164251

165252
# Refactor all files and directories passed as arguments
166253
if not rt.errors:

‎Lib/lib2to3/refactor.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ class FixerError(Exception):
173173

174174
class RefactoringTool(object):
175175

176-
_default_options = {"print_function" : False}
176+
_default_options = {"print_function" : False,
177+
"write_unchanged_files" : False}
177178

178179
CLASS_PREFIX = "Fix" # The prefix for fixer classes
179180
FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -195,6 +196,10 @@ def __init__(self, fixer_names, options=None, explicit=None):
195196
self.grammar = pygram.python_grammar_no_print_statement
196197
else:
197198
self.grammar = pygram.python_grammar
199+
# When this is True, the refactor*() methods will call write_file() for
200+
# files processed even if they were not changed during refactoring. If
201+
# and only if the refactor method's write parameter was True.
202+
self.write_unchanged_files = self.options.get("write_unchanged_files")
198203
self.errors = []
199204
self.logger = logging.getLogger("RefactoringTool")
200205
self.fixer_log = []
@@ -341,13 +346,13 @@ def refactor_file(self, filename, write=False, doctests_only=False):
341346
if doctests_only:
342347
self.log_debug("Refactoring doctests in %s", filename)
343348
output = self.refactor_docstring(input, filename)
344-
if output != input:
349+
if self.write_unchanged_files or output != input:
345350
self.processed_file(output, filename, input, write, encoding)
346351
else:
347352
self.log_debug("No doctest changes in %s", filename)
348353
else:
349354
tree = self.refactor_string(input, filename)
350-
if tree and tree.was_changed:
355+
if self.write_unchanged_files or (tree and tree.was_changed):
351356
# The [:-1] is to take off the \n we added earlier
352357
self.processed_file(str(tree)[:-1], filename,
353358
write=write, encoding=encoding)
@@ -386,13 +391,13 @@ def refactor_stdin(self, doctests_only=False):
386391
if doctests_only:
387392
self.log_debug("Refactoring doctests in stdin")
388393
output = self.refactor_docstring(input, "<stdin>")
389-
if output != input:
394+
if self.write_unchanged_files or output != input:
390395
self.processed_file(output, "<stdin>", input)
391396
else:
392397
self.log_debug("No doctest changes in stdin")
393398
else:
394399
tree = self.refactor_string(input, "<stdin>")
395-
if tree and tree.was_changed:
400+
if self.write_unchanged_files or (tree and tree.was_changed):
396401
self.processed_file(str(tree), "<stdin>", input)
397402
else:
398403
self.log_debug("No changes in stdin")
@@ -502,7 +507,7 @@ def traverse_by(self, fixers, traversal):
502507
def processed_file(self, new_text, filename, old_text=None, write=False,
503508
encoding=None):
504509
"""
505-
Called when a file has been refactored, and there are changes.
510+
Called when a file has been refactored and there may be changes.
506511
"""
507512
self.files.append(filename)
508513
if old_text is None:
@@ -513,7 +518,8 @@ def processed_file(self, new_text, filename, old_text=None, write=False,
513518
self.print_output(old_text, new_text, filename, equal)
514519
if equal:
515520
self.log_debug("No changes to %s", filename)
516-
return
521+
if not self.write_unchanged_files:
522+
return
517523
if write:
518524
self.write_file(new_text, filename, old_text, encoding)
519525
else:

‎Lib/lib2to3/tests/test_main.py‎

Lines changed: 96 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,30 @@
11
# -*- coding: utf-8 -*-
2-
import sys
32
import codecs
4-
import logging
53
import io
4+
import logging
5+
import os
6+
import shutil
7+
import sys
8+
import tempfile
69
import unittest
710

811
from lib2to3 import main
912

1013

14+
TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
15+
PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py")
16+
17+
1118
class TestMain(unittest.TestCase):
1219

20+
def setUp(self):
21+
self.temp_dir = None # tearDown() will rmtree this directory if set.
22+
1323
def tearDown(self):
1424
# Clean up logging configuration down by main.
1525
del logging.root.handlers[:]
26+
if self.temp_dir:
27+
shutil.rmtree(self.temp_dir)
1628

1729
def run_2to3_capture(self, args, in_capture, out_capture, err_capture):
1830
save_stdin = sys.stdin
@@ -39,3 +51,85 @@ def test_unencodable_diff(self):
3951
self.assertTrue("-print 'nothing'" in output)
4052
self.assertTrue("WARNING: couldn't encode <stdin>'s diff for "
4153
"your terminal" in err.getvalue())
54+
55+
def setup_test_source_trees(self):
56+
"""Setup a test source tree and output destination tree."""
57+
self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up.
58+
self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
59+
self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
60+
os.mkdir(self.py2_src_dir)
61+
os.mkdir(self.py3_dest_dir)
62+
# Turn it into a package with a few files.
63+
self.setup_files = []
64+
open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
65+
self.setup_files.append("__init__.py")
66+
shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
67+
self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
68+
self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
69+
self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
70+
with open(self.trivial_py2_file, "w") as trivial:
71+
trivial.write("print 'I need a simple conversion.'")
72+
self.setup_files.append("trivial.py")
73+
74+
def test_filename_changing_on_output_single_dir(self):
75+
"""2to3 a single directory with a new output dir and suffix."""
76+
self.setup_test_source_trees()
77+
out = io.StringIO()
78+
err = io.StringIO()
79+
suffix = "TEST"
80+
ret = self.run_2to3_capture(
81+
["-n", "--add-suffix", suffix, "--write-unchanged-files",
82+
"--no-diffs", "--output-dir",
83+
self.py3_dest_dir, self.py2_src_dir],
84+
io.StringIO(""), out, err)
85+
self.assertEqual(ret, 0)
86+
stderr = err.getvalue()
87+
self.assertIn(" implies -w.", stderr)
88+
self.assertIn(
89+
"Output in %r will mirror the input directory %r layout" % (
90+
self.py3_dest_dir, self.py2_src_dir), stderr)
91+
self.assertEqual(set(name+suffix for name in self.setup_files),
92+
set(os.listdir(self.py3_dest_dir)))
93+
for name in self.setup_files:
94+
self.assertIn("Writing converted %s to %s" % (
95+
os.path.join(self.py2_src_dir, name),
96+
os.path.join(self.py3_dest_dir, name+suffix)), stderr)
97+
self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py")
98+
self.assertNotRegex(stderr, r"No changes to .*/trivial\.py")
99+
100+
def test_filename_changing_on_output_two_files(self):
101+
"""2to3 two files in one directory with a new output dir."""
102+
self.setup_test_source_trees()
103+
err = io.StringIO()
104+
py2_files = [self.trivial_py2_file, self.init_py2_file]
105+
expected_files = set(os.path.basename(name) for name in py2_files)
106+
ret = self.run_2to3_capture(
107+
["-n", "-w", "--write-unchanged-files",
108+
"--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,
109+
io.StringIO(""), io.StringIO(), err)
110+
self.assertEqual(ret, 0)
111+
stderr = err.getvalue()
112+
self.assertIn(
113+
"Output in %r will mirror the input directory %r layout" % (
114+
self.py3_dest_dir, self.py2_src_dir), stderr)
115+
self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))
116+
117+
def test_filename_changing_on_output_single_file(self):
118+
"""2to3 a single file with a new output dir."""
119+
self.setup_test_source_trees()
120+
err = io.StringIO()
121+
ret = self.run_2to3_capture(
122+
["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,
123+
self.trivial_py2_file],
124+
io.StringIO(""), io.StringIO(), err)
125+
self.assertEqual(ret, 0)
126+
stderr = err.getvalue()
127+
self.assertIn(
128+
"Output in %r will mirror the input directory %r layout" % (
129+
self.py3_dest_dir, self.py2_src_dir), stderr)
130+
self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),
131+
set(os.listdir(self.py3_dest_dir)))
132+
133+
134+
if __name__ == '__main__':
135+
unittest.main()

0 commit comments

Comments
 (0)