Categorize some stubgen tests as slow tests (#6340)

JukkaL · web-flow · commit 615c2072f604 · 2019-02-05T17:26:55.000Z
This allows pytest to parallelize tests more effectively and
seems to make the test suite faster by ~5 seconds or so on my
Linux desktop.

These can use typeshed so they are not fast tests.

Some of the StubgenPython tests are fast but others aren't. I'm
classifying them as slow since it looks like this is generally
better for scheduling, but it might be even better to split the
suite into fast and slow parts.
diff --git a/runtests.py b/runtests.py
@@ -21,8 +21,17 @@
 PEP561 = 'TestPEP561'
 EVALUATION = 'PythonEvaluation'
 DAEMON = 'testdaemon'
+STUBGEN_CMD = 'StubgenCmdLine'
+STUBGEN_PY = 'StubgenPythonSuite'
 
-ALL_NON_FAST = [CMDLINE, SAMPLES, TYPESHED, PEP561, EVALUATION, DAEMON]
+ALL_NON_FAST = [CMDLINE,
+                SAMPLES,
+                TYPESHED,
+                PEP561,
+                EVALUATION,
+                DAEMON,
+                STUBGEN_CMD,
+                STUBGEN_PY]
 
 # We split the pytest run into three parts to improve test
 # parallelization. Each run should have tests that each take a roughly similiar
@@ -35,7 +44,10 @@
     # Fast test cases only (this is the bulk of the test suite)
     'pytest-fast': 'pytest -k "not (%s)"' % ' or '.join(ALL_NON_FAST),
     # Test cases that invoke mypy (with small inputs)
-    'pytest-cmdline': 'pytest -k "%s"' % ' or '.join([CMDLINE, EVALUATION]),
+    'pytest-cmdline': 'pytest -k "%s"' % ' or '.join([CMDLINE,
+                                                      EVALUATION,
+                                                      STUBGEN_CMD,
+                                                      STUBGEN_PY]),
     # Test cases that may take seconds to run each
     'pytest-slow': 'pytest -k "%s"' % ' or '.join([SAMPLES, TYPESHED, PEP561, DAEMON]),
 }
