Patch #1276356: Implement new resource "urlfetch" for regrtest.

hyeshik · hyeshik · commit aaa2f1dea706 · 2005-12-10T17:44:27.000Z
This enables even impatient people to run tests that require remote
files such as test_normalization and test_codecmaps_*.
diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
@@ -97,6 +97,8 @@
 
     subprocess  Run all tests for the subprocess module.
 
+    urlfetch -  It is okay to download files required on testing.
+
 To enable all resources except one, use '-uall,-<resource>'.  For
 example, to run all the tests except for the bsddb tests, give the
 option '-uall,-bsddb'.
@@ -140,7 +142,7 @@
 from test import test_support
 
 RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',
-                  'decimal', 'compiler', 'subprocess')
+                  'decimal', 'compiler', 'subprocess', 'urlfetch')
 
 
 def usage(code, msg=''):
@@ -671,20 +673,12 @@ def printlist(x, width=70, indent=4):
 #     test_pep277
 #         The _ExpectedSkips constructor adds this to the set of expected
 #         skips if not os.path.supports_unicode_filenames.
-#     test_normalization
-#         Whether a skip is expected here depends on whether a large test
-#         input file has been downloaded.  test_normalization.skip_expected
-#         controls that.
 #     test_socket_ssl
 #         Controlled by test_socket_ssl.skip_expected.  Requires the network
 #         resource, and a socket module with ssl support.
 #     test_timeout
 #         Controlled by test_timeout.skip_expected.  Requires the network
 #         resource and a socket module.
-#     test_codecmaps_*
-#         Whether a skip is expected here depends on whether a large test
-#         input file has been downloaded.  test_codecmaps_*.skip_expected
-#         controls that.
 
 _expectations = {
     'win32':
@@ -1056,7 +1050,6 @@ def printlist(x, width=70, indent=4):
         test_macfs
         test_macostools
         test_nis
-        test_normalization
         test_ossaudiodev
         test_pep277
         test_plistlib
@@ -1108,12 +1101,8 @@ def printlist(x, width=70, indent=4):
 class _ExpectedSkips:
     def __init__(self):
         import os.path
-        from test import test_normalization
         from test import test_socket_ssl
         from test import test_timeout
-        from test import test_codecmaps_cn, test_codecmaps_jp
-        from test import test_codecmaps_kr, test_codecmaps_tw
-        from test import test_codecmaps_hk
 
         self.valid = False
         if sys.platform in _expectations:
@@ -1126,19 +1115,12 @@ def __init__(self):
             if not os.path.supports_unicode_filenames:
                 self.expected.add('test_pep277')
 
-            if test_normalization.skip_expected:
-                self.expected.add('test_normalization')
-
             if test_socket_ssl.skip_expected:
                 self.expected.add('test_socket_ssl')
 
             if test_timeout.skip_expected:
                 self.expected.add('test_timeout')
 
-            for cc in ('cn', 'jp', 'kr', 'tw', 'hk'):
-                if eval('test_codecmaps_' + cc).skip_expected:
-                    self.expected.add('test_codecmaps_' + cc)
-
             if sys.maxint == 9223372036854775807L:
                 self.expected.add('test_rgbimg')
                 self.expected.add('test_imageop')
diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py
@@ -12,13 +12,11 @@
 class TestGB2312Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'gb2312'
-    mapfilename = 'EUC-CN.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-CN.TXT'
 
 class TestGBKMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'gbk'
-    mapfilename = 'CP936.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/' \
                  'MICSFT/WINDOWS/CP936.TXT'
 
@@ -28,6 +26,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestGBKMap))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestGB2312Map, TestGBKMap)
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py
@@ -12,14 +12,12 @@
 class TestBig5HKSCSMap(test_multibytecodec_support.TestBase_Mapping,
                        unittest.TestCase):
     encoding = 'big5hkscs'
-    mapfilename = 'BIG5HKSCS.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/BIG5HKSCS.TXT'
 
 def test_main():
     suite = unittest.TestSuite()
     suite.addTest(unittest.makeSuite(TestBig5HKSCSMap))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestBig5HKSCSMap)
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py
@@ -12,7 +12,6 @@
 class TestCP932Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp932'
-    mapfilename = 'CP932.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/' \
                  'WINDOWS/CP932.TXT'
     supmaps = [
@@ -71,9 +70,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestSJISX0213Map))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestCP932Map,
-    TestEUCJPCOMPATMap, TestSJISCOMPATMap, TestEUCJISX0213Map,
-    TestSJISX0213Map)
-
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py
@@ -12,22 +12,19 @@
 class TestCP949Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp949'
-    mapfilename = 'CP949.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT' \
                  '/WINDOWS/CP949.TXT'
 
 
 class TestEUCKRMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'euc_kr'
-    mapfilename = 'EUC-KR.TXT'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-KR.TXT'
 
 
 class TestJOHABMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'johab'
-    mapfilename = 'JOHAB.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/' \
                  'KSC/JOHAB.TXT'
     # KS X 1001 standard assigned 0x5c as WON SIGN.
@@ -44,7 +41,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestJOHABMap))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestCP949Map,
-    TestEUCKRMap, TestJOHABMap)
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py
@@ -12,14 +12,12 @@
 class TestBIG5Map(test_multibytecodec_support.TestBase_Mapping,
                   unittest.TestCase):
     encoding = 'big5'
-    mapfilename = 'BIG5.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE/' \
                  'EASTASIA/OTHER/BIG5.TXT'
 
 class TestCP950Map(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
     encoding = 'cp950'
-    mapfilename = 'CP950.TXT'
     mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/' \
                  'WINDOWS/CP950.TXT'
     pass_enctest = [
@@ -33,6 +31,5 @@ def test_main():
     suite.addTest(unittest.makeSuite(TestCP950Map))
     test_support.run_suite(suite)
 
-test_multibytecodec_support.register_skip_expected(TestBIG5Map, TestCP950Map)
 if __name__ == "__main__":
     test_main()
diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/test_multibytecodec_support.py
@@ -163,15 +163,16 @@ class TestBase_Mapping(unittest.TestCase):
 
     def __init__(self, *args, **kw):
         unittest.TestCase.__init__(self, *args, **kw)
-        if not os.path.exists(self.mapfilename):
-            raise test_support.TestSkipped('%s not found, download from %s' %
-                    (self.mapfilename, self.mapfileurl))
+        self.open_mapping_file() # test it to report the error early
+
+    def open_mapping_file(self):
+        return test_support.open_urlresource(self.mapfileurl)
 
     def test_mapping_file(self):
         unichrs = lambda s: u''.join(map(unichr, map(eval, s.split('+'))))
         urt_wa = {}
 
-        for line in open(self.mapfilename):
+        for line in self.open_mapping_file():
             if not line:
                 break
             data = line.split('#')[0].strip().split()
@@ -217,16 +218,3 @@ def load_teststring(encoding):
     else:
         from test import cjkencodings_test
         return cjkencodings_test.teststring[encoding]
-
-def register_skip_expected(*cases):
-    for case in cases: # len(cases) must be 1 at least.
-        for path in [os.path.curdir, os.path.pardir]:
-            fn = os.path.join(path, case.mapfilename)
-            if os.path.exists(fn):
-                case.mapfilename = fn
-                break
-        else:
-            sys.modules[case.__module__].skip_expected = True
-            break
-    else:
-        sys.modules[case.__module__].skip_expected = False
diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py
@@ -1,21 +1,11 @@
-from test.test_support import verbose, TestFailed, TestSkipped, verify
+from test.test_support import (verbose, TestFailed, TestSkipped, verify,
+                               open_urlresource)
 import sys
 import os
 from unicodedata import normalize
 
 TESTDATAFILE = "NormalizationTest-3.2.0" + os.extsep + "txt"
-
-# This search allows using a build directory just inside the source
-# directory, and saving just one copy of the test data in the source
-# tree, rather than having a copy in each build directory.
-# There might be a better way to do this.
-
-for path in [os.path.curdir, os.path.pardir]:
-    fn = os.path.join(path, TESTDATAFILE)
-    skip_expected = not os.path.exists(fn)
-    if not skip_expected:
-        TESTDATAFILE = fn
-        break
+TESTDATAURL = "http://www.unicode.org/Public/3.2-Update/" + TESTDATAFILE
 
 class RangeError:
     pass
@@ -40,12 +30,8 @@ def unistr(data):
     return u"".join([unichr(x) for x in data])
 
 def test_main():
-    if skip_expected:
-        raise TestSkipped(TESTDATAFILE + " not found, download from " +
-                    "http://www.unicode.org/Public/3.2-Update/" + TESTDATAFILE)
-
     part1_data = {}
-    for line in open(TESTDATAFILE):
+    for line in open_urlresource(TESTDATAURL):
         if '#' in line:
             line = line.split('#')[0]
         line = line.strip()
diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
@@ -237,7 +237,21 @@ def check_syntax(statement):
     else:
         print 'Missing SyntaxError: "%s"' % statement
 
+def open_urlresource(url):
+    import urllib, urlparse
+    import os.path
 
+    filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's URL!
+
+    for path in [os.path.curdir, os.path.pardir]:
+        fn = os.path.join(path, filename)
+        if os.path.exists(fn):
+            return open(fn)
+
+    requires('urlfetch')
+    print >> get_original_stdout(), '\tfetching %s ...' % url
+    fn, _ = urllib.urlretrieve(url, filename)
+    return open(fn)
 
 #=======================================================================
 # Preliminary PyUNIT integration.
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -633,6 +633,9 @@ Tests
 
 - In test_os, st_?time is now truncated before comparing it with ST_?TIME.
 
+- Patch #1276356: New resource "urlfetch" is implemented.  This enables
+  even impatient people to run tests that require remote files.
+
 
 Documentation
 -------------
