Add some infrastructure for benchmark tests that don't run by default

msullivan · msullivan · commit 8dd648446854 · 2018-05-25T13:13:30.000-07:00
And add in one simple one
diff --git a/conftest.py b/conftest.py
@@ -1,3 +1,8 @@
 pytest_plugins = [
     'mypy.test.data',
 ]
+# This function name is special to pytest.  See
+# http://doc.pytest.org/en/latest/writing_plugins.html#initialization-command-line-and-configuration-hooks
+def pytest_addoption(parser)-> None:
+    parser.addoption('--bench', action='store_true', default=False,
+                     help='Enable the benchmark test runs')
diff --git a/mypyc/test/test_bench.py b/mypyc/test/test_bench.py
@@ -0,0 +1,13 @@
+"""Benchmark run test cases that don't run by default
+
+All of the real code for this lives in test_run.py.
+"""
+
+# We can't "import TestRun from ..." because that will cause pytest
+# to collect the non-caching tests when running this file.
+import mypyc.test.test_run
+
+
+class TestBench(mypyc.test.test_run.TestRun):
+    benchmark = True
+    files = ['run-bench.test']
diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py
@@ -17,18 +17,22 @@
     ICODE_GEN_BUILTINS, use_custom_builtins, MypycDataSuite, assert_test_output,
 )
 
+import pytest  # type: ignore  # no pytest in typeshed
 
 files = ['run.test',
          'run-classes.test']
 
-
 class TestRun(MypycDataSuite):
     """Test cases that build a C extension and run code."""
     files = files
     base_path = test_temp_dir
     optional_out = True
+    benchmark = False
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
+        if self.benchmark and not testcase.config.getoption('--bench', False):
+            pytest.skip()
+
         with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
             text = '\n'.join(testcase.input)
 
@@ -42,6 +46,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             source_path = 'tmp/py/native.py'
             with open(source_path, 'w') as f:
                 f.write(text)
+            with open('tmp/interpreted.py', 'w') as f:
+                f.write(text)
 
             source = build.BuildSource(source_path, 'native', text)
 
@@ -90,7 +96,11 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                 print('*** Exit status: %d' % proc.returncode)
 
             # Verify output.
-            assert_test_output(testcase, outlines, 'Invalid output')
+            if self.benchmark:
+                print('Test output:')
+                print(output)
+            else:
+                assert_test_output(testcase, outlines, 'Invalid output')
 
             assert proc.returncode == 0
 
diff --git a/test-data/run-bench.test b/test-data/run-bench.test
@@ -0,0 +1,55 @@
+[case testTree]
+from typing import Optional
+# This is bullshit
+class Node:
+    def __init__(self) -> None:
+        self.value = 0
+        self.left = None  # type: Optional[Node]
+        self.right = None  # type: Optional[Node]
+    def sum(self) -> int:
+        left = 0
+        if self.left is not None:
+            left = self.left.sum()
+        right = 0
+        if self.right is not None:
+            right = self.right.sum()
+        return self.value + left + right
+def node(v: int) -> Node:
+    x = Node()
+    x.value = v
+    x.left = None
+    x.right = None
+    return x
+def sum_tree(x: Optional[Node]) -> int:
+    if x is None:
+        return 0
+    return x.value + sum_tree(x.left) + sum_tree(x.right)
+def lol(n: int) -> Optional[Node]:
+    if n == 0:
+        return None
+    x = node(n)
+    x.left = lol(n - 1)
+    x.right = x.left
+    return x
+[file driver.py]
+from typing import Optional
+import native
+import interpreted
+from timeit import timeit
+
+def test(m):
+    tree = m.lol(5)
+    assert(m.sum_tree(tree) == 57)
+    assert(tree.sum() == 57)
+
+    g = {**globals(), **locals()}
+    sum = timeit('m.sum_tree(tree)', globals=g)
+    sum2 = timeit('tree.sum()', globals=g)
+    build = timeit('m.lol(5)', globals=g)
+    return (sum, sum2, build)
+
+nsum, nsum2, nbuild = test(native)
+isum, isum2, ibuild = test(interpreted)
+print("Sum speedup:", isum/nsum)
+print("Sum method speedup:", isum2/nsum2)
+print("Build speedup:", ibuild/nbuild)

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,8 @@`
`1`	`1`	`pytest_plugins = [`
`2`	`2`	`'mypy.test.data',`
`3`	`3`	`]`
	`4`	`+# This function name is special to pytest. See`
	`5`	`+# http://doc.pytest.org/en/latest/writing_plugins.html#initialization-command-line-and-configuration-hooks`
	`6`	`+def pytest_addoption(parser)-> None:`
	`7`	`+ parser.addoption('--bench', action='store_true', default=False,`
	`8`	`+ help='Enable the benchmark test runs')`