Implement custom loading of pybindings to replace imp.load_dynamic

calibration/src/python.cxx

@@ -2,7 +2,7 @@
 
 namespace bp = boost::python;
 
-BOOST_PYTHON_MODULE(calibration)
+SPT3G_PYTHON_MODULE(calibration)
 {
 	// Python bindings dependencies
 	bp::import("spt3g.core");

core/CMakeLists.txt

@@ -1,3 +1,14 @@
+# Build a minimal library with a constant name which can be loaded as a python
+# module with a plain `import`, which can then load other modules which are
+# native libraries with normal names.
+add_library(dload SHARED src/dload.c)
+set_target_properties(dload PROPERTIES PREFIX "")
+set_target_properties(dload PROPERTIES SUFFIX ".so")
+install(TARGETS dload DESTINATION ${PYTHON_MODULE_DIR}/spt3g)
+MESSAGE("Setting dload target_include_directories to ${Python_INCLUDE_DIRS}")
+target_include_directories(dload PRIVATE ${Python_INCLUDE_DIRS})
+target_link_libraries(dload PUBLIC ${Python_LIBRARIES})
+
 # OS X has a broken implementatin of pthreads.
 if(APPLE)
 	set(CORE_EXTRA_SRCS src/ApplePthreadBarrier.cxx)

core/include/core/pybindings.h

@@ -4,6 +4,9 @@
 #include <G3.h>
 #include <G3Frame.h>
 
+#include <boost/preprocessor/cat.hpp>
+#include <boost/preprocessor/facilities/overload.hpp>
+#include <boost/preprocessor/stringize.hpp>
 #include <boost/python.hpp>
 #include <boost/python/suite/indexing/vector_indexing_suite.hpp>
 #include <boost/python/suite/indexing/container_utils.hpp>
@@ -351,3 +354,33 @@ class G3ModuleRegistrator {
 
 #endif
 
+// Declare a python module with a name and the name of its enclosing package scope.
+// name should be be a bare token, while pkg should be a string literal, e.g.:
+//     SPT3G_PYTHON_MODULE_2(foo, "spt3g.bar")
+// for a package whose fully qualified name will be spt3g.bar.foo
+#define SPT3G_PYTHON_MODULE_2(name, pkg) \
+BOOST_PYTHON_MODULE(name) { \
+	namespace bp = boost::python; \
+	auto mod = bp::scope(); \
+	std::string package_prefix = pkg; \
+	std::string full_name = package_prefix + "." + bp::extract<std::string>(mod.attr("__name__"))(); \
+	mod.attr("__name__") = full_name; \
+	mod.attr("__package__") = package_prefix; \
+	void BOOST_PP_CAT(spt3g_init_module_, name)(); \
+	BOOST_PP_CAT(spt3g_init_module_, name)(); \
+	if(PY_MAJOR_VERSION < 3){ \
+		Py_INCREF(mod.ptr()); \
+		PyDict_SetItemString(PyImport_GetModuleDict(),full_name.c_str(),mod.ptr()); \
+	} \
+} \
+void BOOST_PP_CAT(spt3g_init_module_, name)()
+
+// Declare a python module with the given name, assuming that the enclosing package 
+// is the default "spt3g".
+#define SPT3G_PYTHON_MODULE_1(name) SPT3G_PYTHON_MODULE_2(name, "spt3g")
+
+// Declare a python module with a name and optionally the name of its enclosing package scope.
+// name should be be a bare token, while if provided the enclosing package name should be a
+// string literal.
+// If the enclosing package name is not specified, it will default to "spt3g".
+#define SPT3G_PYTHON_MODULE(...) BOOST_PP_OVERLOAD(SPT3G_PYTHON_MODULE_,__VA_ARGS__)(__VA_ARGS__)

core/python/load_pybindings.py

@@ -18,13 +18,13 @@
     lib_suffix = ".so"
 
 def load_pybindings(name, path):
-	import imp, os, sys
+	import os
 	mod = sys.modules[name]
 	p = os.path.split(path[0])
-	m = imp.load_dynamic(name, p[0] + "/" + lib_prefix + p[1] + lib_suffix)
+	from spt3g import dload
+	m = dload.load_dynamic(name, p[1], p[0] + "/" + lib_prefix + p[1] + lib_suffix)
 	sys.modules[name] = mod # Don't override Python mod with C++
 
 	for (k,v) in m.__dict__.items():
 		if not k.startswith("_"):
 			mod.__dict__[k] = v
-

core/src/dload.c

@@ -0,0 +1,122 @@
+#include <dlfcn.h>
+
+#include <Python.h>
+
+static PyObject* load_dynamic_library(PyObject* mod, PyObject* args, PyObject* kwds){
+	static const char* kwlist[] = {"full_name", "name", "path", NULL};
+	char* full_name=NULL;
+	char* name=NULL;
+	char* path=NULL;
+	if (!PyArg_ParseTupleAndKeywords(args, kwds, "sss", (char**)kwlist, &full_name, &name, &path))
+		return Py_None;
+
+	void* h = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
+    char* errmsg = dlerror();
+
+    if(h == NULL || errmsg != NULL){
+    	char err_buf[1024];
+    	snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': %s", name, path, 
+    	         (errmsg == NULL ? "dlopen: unknown error" : errmsg));
+    	PyErr_SetString(PyExc_RuntimeError, err_buf);
+		return Py_None;
+    }
+    PyObject* (*init)(void);
+    char initName[256];
+    int print_res = 0;
+#if PY_MAJOR_VERSION >= 3
+	print_res = snprintf(initName, 256, "PyInit_%s", name);
+#else
+	print_res = snprintf(initName, 256, "init%s", name);
+#endif
+	if(print_res<0){
+		char err_buf[1024];
+    	snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': "
+    	         "module init function name too long", name, path);
+    	PyErr_SetString(PyExc_RuntimeError, err_buf);
+		return Py_None;
+	}
+
+    *(void **) (&init) = dlsym(h,initName);
+    errmsg = dlerror();
+    if(errmsg != NULL){
+    	char err_buf[1024];
+    	snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': %s", name, path,
+    	         (errmsg == NULL ? "dlsym: unknown error" : errmsg));
+    	printf("%s\n", err_buf);
+    	PyErr_SetString(PyExc_RuntimeError, err_buf);
+		return Py_None;
+    }
+
+#if PY_MAJOR_VERSION >= 3
+    PyObject* module = (*init)();
+    PyObject_SetAttrString(module, "__file__", PyUnicode_FromString(path));
+#else
+	(*init)();
+	PyObject* module = PyDict_GetItemString(PyImport_GetModuleDict(), full_name);
+	if(nmod==NULL){
+		char err_buf[1024];
+		snprintf(err_buf, 1024, "dynamic loading error: %s not in global module dict", full_name);
+		PyErr_SetString(PyExc_RuntimeError, err_buf);
+		return Py_None;
+	}
+	PyObject_SetAttrString(module, "__file__", PyString_FromString(path));
+#endif
+	return module;
+}
+
+static PyMethodDef spt3g_dload_methods[] = {
+	{"load_dynamic", (PyCFunction)&load_dynamic_library, METH_VARARGS | METH_KEYWORDS, 
+	 "\n"
+	 "Load a compiled extension module. No restriction is placed on the relationship\n"
+	 "between the name of the module and the name of the dynamic library file which\n"
+	 "implements it; e.g. module `foo` need not reside in `foo.so`.\n"
+	 "\n"
+	 "Arguments\n"
+	 "---------\n"
+	 "full_name : str\n"
+	 "    The module's fully qualified name\n"
+	 "name : str\n"
+	 "    The module's name\n"
+	 "path : str\n"
+	 "    The path to the dynamic library which implements the module\n"
+	 "\n"
+	 "Returns\n"
+	 "-------\n"
+	 "The loaded module\n"
+	},
+	{NULL, NULL, 0, NULL}
+};
+
+#if PY_MAJOR_VERSION >= 3
+static struct PyModuleDef moduledef = {
+	PyModuleDef_HEAD_INIT,
+	"spt3g_dload",
+	"A beach-head module for loading other compiled spt3g modules",
+	0,
+	spt3g_dload_methods,
+	NULL,
+	NULL,
+	NULL,
+	NULL,
+};
+#endif
+
+PyMODINIT_FUNC
+#if PY_MAJOR_VERSION >= 3
+PyInit_dload(void){
+#else
+initdload(void){
+#endif
+	PyObject* module;
+
+#if PY_MAJOR_VERSION >= 3
+	module = PyModule_Create(&moduledef);
+	PyObject_SetAttrString(module, "__version__", PyUnicode_FromString("1.0.0"));
+	return module;
+#else
+	module = Py_InitModule3("spt3g_dload", spt3g_dload_methods,
+	                        "A beach-head module for loading other compiled spt3g modules");
+	PyObject_SetAttrString(module, "__version__", PyString_FromString("1.0.0"));
+	PyDict_SetItemString(PyImport_GetModuleDict(),_Py_PackageContext,module);
+#endif
+}

core/src/python.cxx

@@ -368,7 +368,7 @@ numpy_container_from_object(boost::python::object v)
 numpy_vector_infrastructure(std::complex<double>, cxdouble, "Zd");
 numpy_vector_infrastructure(std::complex<float>, cxfloat, "Zf");
 
-BOOST_PYTHON_MODULE(core)
+SPT3G_PYTHON_MODULE(core)
 {
 	bp::docstring_options docopts(true, true, false);
 

dfmux/src/python.cxx

@@ -2,7 +2,7 @@
 
 namespace bp = boost::python;
 
-BOOST_PYTHON_MODULE(dfmux)
+SPT3G_PYTHON_MODULE(dfmux)
 {
 	// Python bindings dependencies
 	bp::import("spt3g.core");

doc/buildsystem.rst

@@ -60,7 +60,7 @@ Every C++ library must contain a file declaring the library to Python. This file
 	#include <pybindings.h>
 	#include <boost/python.hpp>
 
-	BOOST_PYTHON_MODULE(newthing)
+	SPT3G_PYTHON_MODULE(newthing)
 	{
 		boost::python::import("spt3g.core"); // Import core python bindings
 

gcp/src/python.cxx

@@ -7,7 +7,7 @@
 
 namespace bp = boost::python;
 
-BOOST_PYTHON_MODULE(gcp)
+SPT3G_PYTHON_MODULE(gcp)
 {
 	bp::import("spt3g.core");
 	bp::docstring_options docopts(true, true, false);

maps/src/python.cxx

@@ -2,7 +2,7 @@
 
 namespace bp = boost::python;
 
-BOOST_PYTHON_MODULE(maps)
+SPT3G_PYTHON_MODULE(maps)
 {
 	bp::import("spt3g.core");
 	bp::docstring_options docopts(true, true, false);