Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 2e0a0e1

Browse files
merwokmerwok
committed
Fix long-standing bugs with MANIFEST.in parsing on Windows (#6884).
These regex changes fix a number of issues for distutils on Windows: - #6884: impossible to include a file starting with 'build' - #9691 and #14004: sdist includes too many files - #13193: test_filelist failures This commit replaces the incorrect changes done in 0a94e2f807c7 and 90b30d62caf2 to fix #13193; we were too eager to fix the test failures and I did not study the code enough before greenlighting patches. This time we have unit tests from the problems reported by users to be sure we have the right fix. Thanks to Nadeem Vawda for his help.
1 parent 3d4809f commit 2e0a0e1

4 files changed

Lines changed: 134 additions & 48 deletions

File tree

‎Lib/distutils/filelist.py‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,7 @@ def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0):
201201
202202
Return True if files are found, False otherwise.
203203
"""
204+
# XXX docstring lying about what the special chars are?
204205
files_found = False
205206
pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
206207
self.debug_print("include_pattern: applying regex r'%s'" %
@@ -284,11 +285,14 @@ def glob_to_re(pattern):
284285
# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
285286
# and by extension they shouldn't match such "special characters" under
286287
# any OS. So change all non-escaped dots in the RE to match any
287-
# character except the special characters.
288-
# XXX currently the "special characters" are just slash -- i.e. this is
289-
# Unix-only.
290-
pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re)
291-
288+
# character except the special characters (currently: just os.sep).
289+
sep = os.sep
290+
if os.sep == '\\':
291+
# we're using a regex to manipulate a regex, so we need
292+
# to escape the backslash twice
293+
sep = r'\\\\'
294+
escaped = r'\1[^%s]' % sep
295+
pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re)
292296
return pattern_re
293297

294298

@@ -312,9 +316,11 @@ def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0):
312316
if prefix is not None:
313317
# ditch end of pattern character
314318
empty_pattern = glob_to_re('')
315-
prefix_re = (glob_to_re(prefix))[:-len(empty_pattern)]
316-
# paths should always use / in manifest templates
317-
pattern_re = "^%s/.*%s" % (prefix_re, pattern_re)
319+
prefix_re = glob_to_re(prefix)[:-len(empty_pattern)]
320+
sep = os.sep
321+
if os.sep == '\\':
322+
sep = r'\\'
323+
pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re))
318324
else: # no prefix -- respect anchor flag
319325
if anchor:
320326
pattern_re = "^" + pattern_re

‎Lib/distutils/tests/test_filelist.py‎

Lines changed: 100 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
"""Tests for distutils.filelist."""
2+
import os
23
import re
34
import unittest
45
from distutils import debug
@@ -9,6 +10,26 @@
910
from test.support import captured_stdout, run_unittest
1011
from distutils.tests import support
1112

13+
MANIFEST_IN = """\
14+
include ok
15+
include xo
16+
exclude xo
17+
include foo.tmp
18+
include buildout.cfg
19+
global-include *.x
20+
global-include *.txt
21+
global-exclude *.tmp
22+
recursive-include f *.oo
23+
recursive-exclude global *.x
24+
graft dir
25+
prune dir3
26+
"""
27+
28+
29+
def make_local_path(s):
30+
"""Converts '/' in a string to os.sep"""
31+
return s.replace('/', os.sep)
32+
1233

1334
class FileListTestCase(support.LoggingSilencer,
1435
unittest.TestCase):
@@ -22,16 +43,62 @@ def assertWarnings(self):
2243
self.clear_logs()
2344

2445
def test_glob_to_re(self):
25-
# simple cases
26-
self.assertEqual(glob_to_re('foo*'), 'foo[^/]*\\Z(?ms)')
27-
self.assertEqual(glob_to_re('foo?'), 'foo[^/]\\Z(?ms)')
28-
self.assertEqual(glob_to_re('foo??'), 'foo[^/][^/]\\Z(?ms)')
29-
30-
# special cases
31-
self.assertEqual(glob_to_re(r'foo\\*'), r'foo\\\\[^/]*\Z(?ms)')
32-
self.assertEqual(glob_to_re(r'foo\\\*'), r'foo\\\\\\[^/]*\Z(?ms)')
33-
self.assertEqual(glob_to_re('foo????'), r'foo[^/][^/][^/][^/]\Z(?ms)')
34-
self.assertEqual(glob_to_re(r'foo\\??'), r'foo\\\\[^/][^/]\Z(?ms)')
46+
sep = os.sep
47+
if os.sep == '\\':
48+
sep = re.escape(os.sep)
49+
50+
for glob, regex in (
51+
# simple cases
52+
('foo*', r'foo[^%(sep)s]*\Z(?ms)'),
53+
('foo?', r'foo[^%(sep)s]\Z(?ms)'),
54+
('foo??', r'foo[^%(sep)s][^%(sep)s]\Z(?ms)'),
55+
# special cases
56+
(r'foo\\*', r'foo\\\\[^%(sep)s]*\Z(?ms)'),
57+
(r'foo\\\*', r'foo\\\\\\[^%(sep)s]*\Z(?ms)'),
58+
('foo????', r'foo[^%(sep)s][^%(sep)s][^%(sep)s][^%(sep)s]\Z(?ms)'),
59+
(r'foo\\??', r'foo\\\\[^%(sep)s][^%(sep)s]\Z(?ms)')):
60+
regex = regex % {'sep': sep}
61+
self.assertEqual(glob_to_re(glob), regex)
62+
63+
def test_process_template_line(self):
64+
# testing all MANIFEST.in template patterns
65+
file_list = FileList()
66+
l = make_local_path
67+
68+
# simulated file list
69+
file_list.allfiles = ['foo.tmp', 'ok', 'xo', 'four.txt',
70+
'buildout.cfg',
71+
# filelist does not filter out VCS directories,
72+
# it's sdist that does
73+
l('.hg/last-message.txt'),
74+
l('global/one.txt'),
75+
l('global/two.txt'),
76+
l('global/files.x'),
77+
l('global/here.tmp'),
78+
l('f/o/f.oo'),
79+
l('dir/graft-one'),
80+
l('dir/dir2/graft2'),
81+
l('dir3/ok'),
82+
l('dir3/sub/ok.txt'),
83+
]
84+
85+
for line in MANIFEST_IN.split('\n'):
86+
if line.strip() == '':
87+
continue
88+
file_list.process_template_line(line)
89+
90+
wanted = ['ok',
91+
'buildout.cfg',
92+
'four.txt',
93+
l('.hg/last-message.txt'),
94+
l('global/one.txt'),
95+
l('global/two.txt'),
96+
l('f/o/f.oo'),
97+
l('dir/graft-one'),
98+
l('dir/dir2/graft2'),
99+
]
100+
101+
self.assertEqual(file_list.files, wanted)
35102

36103
def test_debug_print(self):
37104
file_list = FileList()
@@ -117,6 +184,7 @@ def test_include_pattern(self):
117184
self.assertEqual(file_list.allfiles, ['a.py', 'b.txt'])
118185

119186
def test_process_template(self):
187+
l = make_local_path
120188
# invalid lines
121189
file_list = FileList()
122190
for action in ('include', 'exclude', 'global-include',
@@ -127,7 +195,7 @@ def test_process_template(self):
127195

128196
# include
129197
file_list = FileList()
130-
file_list.set_allfiles(['a.py', 'b.txt', 'd/c.py'])
198+
file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')])
131199

132200
file_list.process_template_line('include *.py')
133201
self.assertEqual(file_list.files, ['a.py'])
@@ -139,31 +207,31 @@ def test_process_template(self):
139207

140208
# exclude
141209
file_list = FileList()
142-
file_list.files = ['a.py', 'b.txt', 'd/c.py']
210+
file_list.files = ['a.py', 'b.txt', l('d/c.py')]
143211

144212
file_list.process_template_line('exclude *.py')
145-
self.assertEqual(file_list.files, ['b.txt', 'd/c.py'])
213+
self.assertEqual(file_list.files, ['b.txt', l('d/c.py')])
146214
self.assertNoWarnings()
147215

148216
file_list.process_template_line('exclude *.rb')
149-
self.assertEqual(file_list.files, ['b.txt', 'd/c.py'])
217+
self.assertEqual(file_list.files, ['b.txt', l('d/c.py')])
150218
self.assertWarnings()
151219

152220
# global-include
153221
file_list = FileList()
154-
file_list.set_allfiles(['a.py', 'b.txt', 'd/c.py'])
222+
file_list.set_allfiles(['a.py', 'b.txt', l('d/c.py')])
155223

156224
file_list.process_template_line('global-include *.py')
157-
self.assertEqual(file_list.files, ['a.py', 'd/c.py'])
225+
self.assertEqual(file_list.files, ['a.py', l('d/c.py')])
158226
self.assertNoWarnings()
159227

160228
file_list.process_template_line('global-include *.rb')
161-
self.assertEqual(file_list.files, ['a.py', 'd/c.py'])
229+
self.assertEqual(file_list.files, ['a.py', l('d/c.py')])
162230
self.assertWarnings()
163231

164232
# global-exclude
165233
file_list = FileList()
166-
file_list.files = ['a.py', 'b.txt', 'd/c.py']
234+
file_list.files = ['a.py', 'b.txt', l('d/c.py')]
167235

168236
file_list.process_template_line('global-exclude *.py')
169237
self.assertEqual(file_list.files, ['b.txt'])
@@ -175,50 +243,52 @@ def test_process_template(self):
175243

176244
# recursive-include
177245
file_list = FileList()
178-
file_list.set_allfiles(['a.py', 'd/b.py', 'd/c.txt', 'd/d/e.py'])
246+
file_list.set_allfiles(['a.py', l('d/b.py'), l('d/c.txt'),
247+
l('d/d/e.py')])
179248

180249
file_list.process_template_line('recursive-include d *.py')
181-
self.assertEqual(file_list.files, ['d/b.py', 'd/d/e.py'])
250+
self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
182251
self.assertNoWarnings()
183252

184253
file_list.process_template_line('recursive-include e *.py')
185-
self.assertEqual(file_list.files, ['d/b.py', 'd/d/e.py'])
254+
self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
186255
self.assertWarnings()
187256

188257
# recursive-exclude
189258
file_list = FileList()
190-
file_list.files = ['a.py', 'd/b.py', 'd/c.txt', 'd/d/e.py']
259+
file_list.files = ['a.py', l('d/b.py'), l('d/c.txt'), l('d/d/e.py')]
191260

192261
file_list.process_template_line('recursive-exclude d *.py')
193-
self.assertEqual(file_list.files, ['a.py', 'd/c.txt'])
262+
self.assertEqual(file_list.files, ['a.py', l('d/c.txt')])
194263
self.assertNoWarnings()
195264

196265
file_list.process_template_line('recursive-exclude e *.py')
197-
self.assertEqual(file_list.files, ['a.py', 'd/c.txt'])
266+
self.assertEqual(file_list.files, ['a.py', l('d/c.txt')])
198267
self.assertWarnings()
199268

200269
# graft
201270
file_list = FileList()
202-
file_list.set_allfiles(['a.py', 'd/b.py', 'd/d/e.py', 'f/f.py'])
271+
file_list.set_allfiles(['a.py', l('d/b.py'), l('d/d/e.py'),
272+
l('f/f.py')])
203273

204274
file_list.process_template_line('graft d')
205-
self.assertEqual(file_list.files, ['d/b.py', 'd/d/e.py'])
275+
self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
206276
self.assertNoWarnings()
207277

208278
file_list.process_template_line('graft e')
209-
self.assertEqual(file_list.files, ['d/b.py', 'd/d/e.py'])
279+
self.assertEqual(file_list.files, [l('d/b.py'), l('d/d/e.py')])
210280
self.assertWarnings()
211281

212282
# prune
213283
file_list = FileList()
214-
file_list.files = ['a.py', 'd/b.py', 'd/d/e.py', 'f/f.py']
284+
file_list.files = ['a.py', l('d/b.py'), l('d/d/e.py'), l('f/f.py')]
215285

216286
file_list.process_template_line('prune d')
217-
self.assertEqual(file_list.files, ['a.py', 'f/f.py'])
287+
self.assertEqual(file_list.files, ['a.py', l('f/f.py')])
218288
self.assertNoWarnings()
219289

220290
file_list.process_template_line('prune e')
221-
self.assertEqual(file_list.files, ['a.py', 'f/f.py'])
291+
self.assertEqual(file_list.files, ['a.py', l('f/f.py')])
222292
self.assertWarnings()
223293

224294

‎Lib/distutils/tests/test_sdist.py‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,12 @@
77
from os.path import join
88
from textwrap import dedent
99

10+
try:
11+
import zlib
12+
ZLIB_SUPPORT = True
13+
except ImportError:
14+
ZLIB_SUPPORT = False
15+
1016
from test.support import captured_stdout, check_warnings, run_unittest
1117

1218
from distutils.command.sdist import sdist, show_formats
@@ -28,6 +34,7 @@
2834
MANIFEST = """\
2935
# file GENERATED by distutils, do NOT edit
3036
README
37+
buildout.cfg
3138
inroot.txt
3239
setup.py
3340
data%(sep)sdata.dt
@@ -39,13 +46,6 @@
3946
somecode%(sep)sdoc.txt
4047
"""
4148

42-
try:
43-
import zlib
44-
ZLIB_SUPPORT = True
45-
except ImportError:
46-
ZLIB_SUPPORT = False
47-
48-
4949
class SDistTestCase(PyPIRCCommandTestCase):
5050

5151
def setUp(self):
@@ -143,7 +143,7 @@ def test_make_distribution(self):
143143
dist_folder = join(self.tmp_dir, 'dist')
144144
result = os.listdir(dist_folder)
145145
result.sort()
146-
self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'] )
146+
self.assertEqual(result, ['fake-1.0.tar', 'fake-1.0.tar.gz'])
147147

148148
os.remove(join(dist_folder, 'fake-1.0.tar'))
149149
os.remove(join(dist_folder, 'fake-1.0.tar.gz'))
@@ -180,11 +180,18 @@ def test_add_defaults(self):
180180
self.write_file((data_dir, 'data.dt'), '#')
181181
some_dir = join(self.tmp_dir, 'some')
182182
os.mkdir(some_dir)
183+
# make sure VCS directories are pruned (#14004)
184+
hg_dir = join(self.tmp_dir, '.hg')
185+
os.mkdir(hg_dir)
186+
self.write_file((hg_dir, 'last-message.txt'), '#')
187+
# a buggy regex used to prevent this from working on windows (#6884)
188+
self.write_file((self.tmp_dir, 'buildout.cfg'), '#')
183189
self.write_file((self.tmp_dir, 'inroot.txt'), '#')
184190
self.write_file((some_dir, 'file.txt'), '#')
185191
self.write_file((some_dir, 'other_file.txt'), '#')
186192

187193
dist.data_files = [('data', ['data/data.dt',
194+
'buildout.cfg',
188195
'inroot.txt',
189196
'notexisting']),
190197
'some/file.txt',
@@ -214,15 +221,15 @@ def test_add_defaults(self):
214221
zip_file.close()
215222

216223
# making sure everything was added
217-
self.assertEqual(len(content), 11)
224+
self.assertEqual(len(content), 12)
218225

219226
# checking the MANIFEST
220227
f = open(join(self.tmp_dir, 'MANIFEST'))
221228
try:
222229
manifest = f.read()
223-
self.assertEqual(manifest, MANIFEST % {'sep': os.sep})
224230
finally:
225231
f.close()
232+
self.assertEqual(manifest, MANIFEST % {'sep': os.sep})
226233

227234
@unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
228235
def test_metadata_check_option(self):

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@ Core and Builtins
124124
Library
125125
-------
126126

127+
- Issue #6884: Fix long-standing bugs with MANIFEST.in parsing in distutils
128+
on Windows.
129+
127130
- Issue #8033: sqlite3: Fix 64-bit integer handling in user functions
128131
on 32-bit architectures. Initial patch by Philippe Devalkeneer.
129132

0 commit comments

Comments
 (0)