Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 020b6b8

Browse files
msullivanmsullivan
authored
Make the runtime tests be invoked via an extension module (mypyc/mypyc#615)
Currently the runtime tests are built as a standalone binary linked against libpython by using `python-config` to produce cflags and ldflags. This turns out to be shockingly fragile and error-prone, since python-config location and behavior doesn't really seem to be standardized. For example, `python3-config`, which we default to using, doesn't exist in virtualenvs, though a `python-config` does. So when run from a virtualenv, we actually run the runtime tests against the system default Python 3, not the virtualenv's Python. But we can't just switch to using `python-config` because that will often point to Python 2. Worse than all that, for reasons I have yet to fully understand I was never able to get the python-config for dev versions on Travis to work. Using the python3-config in the root python 3.8-dev install directory failed because it didn't include the library path (??) and using the python-config in the virtualenv failed because it didn't include `-lpython` (???) (or maybe vice versa). Meanwhile building C extension modules inside of virtualenvs works pretty reliably, so just switch to driving the runtime tests from a C extension module instead of from a standalone binary. This means that the RT tests will run against the correct python version on developer machines and we can turn on 3.8-dev tests in Travis.
1 parent 6791e82 commit 020b6b8

7 files changed

Lines changed: 87 additions & 78 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,5 @@ test_capi
1414
/.dmypy.json
1515
/.mypyc-flake8-cache.json
1616
/build
17+
/mypyc/lib-rt/build/
18+
/mypyc/lib-rt/*.so

‎external/googletest/make/Makefile‎

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -16,20 +16,13 @@
1616
# Remember to tweak this if you move this file.
1717
GTEST_DIR = ..
1818

19-
# Where to find user code.
20-
USER_DIR = ../samples
21-
2219
# Flags passed to the preprocessor.
2320
# Set Google Test's header directory as a system directory, such that
2421
# the compiler doesn't generate warnings in Google Test headers.
2522
CPPFLAGS += -isystem $(GTEST_DIR)/include
2623

2724
# Flags passed to the C++ compiler.
28-
CXXFLAGS += -g -Wall -Wextra -pthread
29-
30-
# All tests produced by this Makefile. Remember to add new tests you
31-
# created to the list.
32-
TESTS = sample1_unittest
25+
CXXFLAGS += -g -Wall -Wextra -pthread -fPIC
3326

3427
# All Google Test headers. Usually you shouldn't change this
3528
# definition.
@@ -38,12 +31,12 @@ GTEST_HEADERS = $(GTEST_DIR)/include/gtest/*.h \
3831

3932
# House-keeping build targets.
4033

41-
all : $(TESTS)
34+
all : libgtest.a libgtest_main.a
4235

4336
clean :
44-
rm -f $(TESTS) gtest.a gtest_main.a *.o
37+
rm -f libgtest.a libgtest_main.a *.o
4538

46-
# Builds gtest.a and gtest_main.a.
39+
# Builds libgtest.a and libgtest_main.a.
4740

4841
# Usually you shouldn't tweak such internal variables, indicated by a
4942
# trailing _.
@@ -61,22 +54,8 @@ gtest_main.o : $(GTEST_SRCS_)
6154
$(CXX) $(CPPFLAGS) -I$(GTEST_DIR) $(CXXFLAGS) -c \
6255
$(GTEST_DIR)/src/gtest_main.cc
6356

64-
gtest.a : gtest-all.o
57+
libgtest.a : gtest-all.o
6558
$(AR) $(ARFLAGS) $@ $^
6659

67-
gtest_main.a : gtest-all.o gtest_main.o
60+
libgtest_main.a : gtest-all.o gtest_main.o
6861
$(AR) $(ARFLAGS) $@ $^
69-
70-
# Builds a sample test. A test should link with either gtest.a or
71-
# gtest_main.a, depending on whether it defines its own main()
72-
# function.
73-
74-
sample1.o : $(USER_DIR)/sample1.cc $(USER_DIR)/sample1.h $(GTEST_HEADERS)
75-
$(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $(USER_DIR)/sample1.cc
76-
77-
sample1_unittest.o : $(USER_DIR)/sample1_unittest.cc \
78-
$(USER_DIR)/sample1.h $(GTEST_HEADERS)
79-
$(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $(USER_DIR)/sample1_unittest.cc
80-
81-
sample1_unittest : sample1.o sample1_unittest.o gtest_main.a
82-
$(CXX) $(CPPFLAGS) $(CXXFLAGS) -lpthread $^ -o $@

‎mypyc/lib-rt/CPy.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
CPy.c

‎mypyc/lib-rt/Makefile‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎mypyc/lib-rt/setup.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
from distutils.core import setup, Extension
2+
3+
setup(name='test_capi',
4+
version='0.1',
5+
ext_modules=[Extension(
6+
'test_capi',
7+
['test_capi.cc', 'CPy.cc'],
8+
depends=['CPy.h', 'mypyc_util.h', 'pythonsupport.h'],
9+
extra_compile_args=['--std=c++11', '-Wno-unused-function', '-Wno-sign-compare'],
10+
library_dirs=['../../external/googletest/make'],
11+
libraries=['gtest'],
12+
include_dirs=['../../external/googletest', '../../external/googletest/include'],
13+
)])

‎mypyc/lib-rt/test_capi.cc‎

Lines changed: 48 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
#include <Python.h>
55
#include "CPy.h"
66

7-
static bool is_initialized = false;
87
static PyObject *moduleDict;
98

109
static PyObject *int_from_str(const char *str) {
@@ -84,21 +83,9 @@ class CAPITest : public ::testing::Test {
8483
Py_ssize_t c_max_neg_long;
8584

8685
virtual void SetUp() {
87-
if (!is_initialized) {
88-
wchar_t *program = Py_DecodeLocale("test_capi", 0);
89-
Py_SetProgramName(program);
90-
Py_Initialize();
91-
PyObject *module = PyModule_New("test");
92-
if (module == 0) {
93-
fail("Could not create module 'test'");
94-
}
95-
moduleDict = PyModule_GetDict(module);
96-
if (module == 0) {
97-
fail("Could not fine module dictionary");
98-
}
99-
is_initialized = true;
86+
if (!moduleDict) {
87+
fail("Could not find module dictionary");
10088
}
101-
// TODO: Call Py_Finalize() and PyMem_RawFree(program) at the end somehow.
10289

10390
max_short = int_from_str("4611686018427387903"); // 2**62-1
10491
min_pos_long = int_from_str("4611686018427387904"); // 2**62
@@ -540,9 +527,52 @@ TEST_F(CAPITest, test_tagged_as_long_long) {
540527
EXPECT_FALSE(PyErr_Occurred());
541528
EXPECT_TRUE(CPyTagged_AsSsize_t(l) == -1);
542529
EXPECT_TRUE(PyErr_Occurred());
530+
PyErr_Clear();
531+
}
532+
533+
534+
////
535+
// Python module glue to drive the C-API tests.
536+
//
537+
// The reason we have this as an extension module instead of a
538+
// standalone binary is because building an extension module is pretty
539+
// well behaved (just specify it with distutils/setuptools and it will
540+
// get compiled and linked against the running python) while linking a
541+
// library against libpython is a huge non-standard
542+
// PITA: python-config locations are janky and it behaves in weird
543+
// ways that I don't understand, while this works very cleanly.
544+
545+
static PyObject *run_tests(PyObject *dummy, PyObject *should_be_null) {
546+
// Fake command line arguments. We could arrange to actually pass
547+
// in command line arguments (either real ones or ones given as
548+
// arguments) but have not bothered.
549+
int argc = 1;
550+
char asdf[] = "test_capi"; // InitGoogleTest wants char** which means it can't be const...
551+
char *argv[] = {asdf, NULL};
552+
::testing::InitGoogleTest(&argc, argv);
553+
return PyLong_FromLong(RUN_ALL_TESTS());
543554
}
544555

545-
int main(int argc, char **argv) {
546-
::testing::InitGoogleTest(&argc, argv);
547-
return RUN_ALL_TESTS();
556+
557+
static PyMethodDef test_methods[] = {
558+
{"run_tests", run_tests, METH_NOARGS, "Run the C API tests"},
559+
{NULL, NULL, 0, NULL}
560+
};
561+
562+
static struct PyModuleDef test_module = {
563+
PyModuleDef_HEAD_INIT,
564+
"test_capi",
565+
NULL,
566+
-1,
567+
test_methods
568+
};
569+
570+
PyMODINIT_FUNC
571+
PyInit_test_capi(void)
572+
{
573+
PyObject *module = PyModule_Create(&test_module);
574+
if (module) {
575+
moduleDict = PyModule_GetDict(module);
576+
}
577+
return module;
548578
}

‎mypyc/test/test_external.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
"""Test cases that run tests as subprocesses."""
22

3+
from typing import List
4+
35
import os
46
import subprocess
57
import sys
@@ -17,18 +19,27 @@ def test_c_unit_test(self) -> None:
1719
"""Run C unit tests in a subprocess."""
1820
# Build Google Test, the C++ framework we use for testing C code.
1921
# The source code for Google Test is copied to this repository.
22+
cppflags = [] # type: List[str]
23+
env = os.environ.copy()
2024
if sys.platform == 'darwin':
21-
env = {'CPPFLAGS': '-mmacosx-version-min=10.10'}
22-
else:
23-
env = os.environ.copy()
24-
subprocess.check_call(['make', 'gtest_main.a'],
25+
cppflags += ['-mmacosx-version-min=10.10', '-stdlib=libc++']
26+
env['CPPFLAGS'] = ' '.join(cppflags)
27+
subprocess.check_call(['make', 'libgtest.a'],
2528
env=env,
2629
cwd=os.path.join(base_dir, 'external', 'googletest', 'make'))
27-
# Build and run C unit tests.
30+
# Build Python wrapper for C unit tests.
31+
env = os.environ.copy()
32+
env['CPPFLAGS'] = ' '.join(cppflags)
33+
status = subprocess.check_call(
34+
[sys.executable, 'setup.py', 'build_ext', '--inplace'],
35+
env=env,
36+
cwd=os.path.join(base_dir, 'mypyc', 'lib-rt'))
37+
# Run C unit tests.
2838
env = os.environ.copy()
2939
if 'GTEST_COLOR' not in os.environ:
3040
env['GTEST_COLOR'] = 'yes' # Use fancy colors
31-
status = subprocess.call(['make', 'test'],
41+
status = subprocess.call([sys.executable, '-c',
42+
'import sys, test_capi; sys.exit(test_capi.run_tests())'],
3243
env=env,
3344
cwd=os.path.join(base_dir, 'mypyc', 'lib-rt'))
3445
if status != 0:

0 commit comments

Comments
 (0)