Skip to content

Commit

Permalink
Implement custom loading of pybindings to replace imp.load_dynamic (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
cnweaver authored Feb 2, 2024
1 parent 7fd257c commit a9d32d5
Show file tree
Hide file tree
Showing 10 changed files with 175 additions and 9 deletions.
2 changes: 1 addition & 1 deletion calibration/src/python.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace bp = boost::python;

BOOST_PYTHON_MODULE(calibration)
SPT3G_PYTHON_MODULE(calibration)
{
// Python bindings dependencies
bp::import("spt3g.core");
Expand Down
11 changes: 11 additions & 0 deletions core/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,14 @@
# Build a minimal library with a constant name which can be loaded as a python
# module with a plain `import`, which can then load other modules which are
# native libraries with normal names.
add_library(dload SHARED src/dload.c)
set_target_properties(dload PROPERTIES PREFIX "")
set_target_properties(dload PROPERTIES SUFFIX ".so")
install(TARGETS dload DESTINATION ${PYTHON_MODULE_DIR}/spt3g)
MESSAGE("Setting dload target_include_directories to ${Python_INCLUDE_DIRS}")
target_include_directories(dload PRIVATE ${Python_INCLUDE_DIRS})
target_link_libraries(dload PUBLIC ${Python_LIBRARIES})

# OS X has a broken implementatin of pthreads.
if(APPLE)
set(CORE_EXTRA_SRCS src/ApplePthreadBarrier.cxx)
Expand Down
33 changes: 33 additions & 0 deletions core/include/core/pybindings.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
#include <G3.h>
#include <G3Frame.h>

#include <boost/preprocessor/cat.hpp>
#include <boost/preprocessor/facilities/overload.hpp>
#include <boost/preprocessor/stringize.hpp>
#include <boost/python.hpp>
#include <boost/python/suite/indexing/vector_indexing_suite.hpp>
#include <boost/python/suite/indexing/container_utils.hpp>
Expand Down Expand Up @@ -351,3 +354,33 @@ class G3ModuleRegistrator {

#endif

// Declare a python module with a name and the name of its enclosing package scope.
// name should be be a bare token, while pkg should be a string literal, e.g.:
// SPT3G_PYTHON_MODULE_2(foo, "spt3g.bar")
// for a package whose fully qualified name will be spt3g.bar.foo
#define SPT3G_PYTHON_MODULE_2(name, pkg) \
BOOST_PYTHON_MODULE(name) { \
namespace bp = boost::python; \
auto mod = bp::scope(); \
std::string package_prefix = pkg; \
std::string full_name = package_prefix + "." + bp::extract<std::string>(mod.attr("__name__"))(); \
mod.attr("__name__") = full_name; \
mod.attr("__package__") = package_prefix; \
void BOOST_PP_CAT(spt3g_init_module_, name)(); \
BOOST_PP_CAT(spt3g_init_module_, name)(); \
if(PY_MAJOR_VERSION < 3){ \
Py_INCREF(mod.ptr()); \
PyDict_SetItemString(PyImport_GetModuleDict(),full_name.c_str(),mod.ptr()); \
} \
} \
void BOOST_PP_CAT(spt3g_init_module_, name)()

// Declare a python module with the given name, assuming that the enclosing package
// is the default "spt3g".
#define SPT3G_PYTHON_MODULE_1(name) SPT3G_PYTHON_MODULE_2(name, "spt3g")

// Declare a python module with a name and optionally the name of its enclosing package scope.
// name should be be a bare token, while if provided the enclosing package name should be a
// string literal.
// If the enclosing package name is not specified, it will default to "spt3g".
#define SPT3G_PYTHON_MODULE(...) BOOST_PP_OVERLOAD(SPT3G_PYTHON_MODULE_,__VA_ARGS__)(__VA_ARGS__)
6 changes: 3 additions & 3 deletions core/python/load_pybindings.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
lib_suffix = ".so"

def load_pybindings(name, path):
import imp, os, sys
import os
mod = sys.modules[name]
p = os.path.split(path[0])
m = imp.load_dynamic(name, p[0] + "/" + lib_prefix + p[1] + lib_suffix)
from spt3g import dload
m = dload.load_dynamic(name, p[1], p[0] + "/" + lib_prefix + p[1] + lib_suffix)
sys.modules[name] = mod # Don't override Python mod with C++

for (k,v) in m.__dict__.items():
if not k.startswith("_"):
mod.__dict__[k] = v

122 changes: 122 additions & 0 deletions core/src/dload.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
#include <dlfcn.h>

#include <Python.h>

static PyObject* load_dynamic_library(PyObject* mod, PyObject* args, PyObject* kwds){
static const char* kwlist[] = {"full_name", "name", "path", NULL};
char* full_name=NULL;
char* name=NULL;
char* path=NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "sss", (char**)kwlist, &full_name, &name, &path))
return Py_None;

void* h = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
char* errmsg = dlerror();

if(h == NULL || errmsg != NULL){
char err_buf[1024];
snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': %s", name, path,
(errmsg == NULL ? "dlopen: unknown error" : errmsg));
PyErr_SetString(PyExc_RuntimeError, err_buf);
return Py_None;
}
PyObject* (*init)(void);
char initName[256];
int print_res = 0;
#if PY_MAJOR_VERSION >= 3
print_res = snprintf(initName, 256, "PyInit_%s", name);
#else
print_res = snprintf(initName, 256, "init%s", name);
#endif
if(print_res<0){
char err_buf[1024];
snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': "
"module init function name too long", name, path);
PyErr_SetString(PyExc_RuntimeError, err_buf);
return Py_None;
}

*(void **) (&init) = dlsym(h,initName);
errmsg = dlerror();
if(errmsg != NULL){
char err_buf[1024];
snprintf(err_buf, 1024, "dynamic loading error: loading '%s' from '%s': %s", name, path,
(errmsg == NULL ? "dlsym: unknown error" : errmsg));
printf("%s\n", err_buf);
PyErr_SetString(PyExc_RuntimeError, err_buf);
return Py_None;
}

#if PY_MAJOR_VERSION >= 3
PyObject* module = (*init)();
PyObject_SetAttrString(module, "__file__", PyUnicode_FromString(path));
#else
(*init)();
PyObject* module = PyDict_GetItemString(PyImport_GetModuleDict(), full_name);
if(nmod==NULL){
char err_buf[1024];
snprintf(err_buf, 1024, "dynamic loading error: %s not in global module dict", full_name);
PyErr_SetString(PyExc_RuntimeError, err_buf);
return Py_None;
}
PyObject_SetAttrString(module, "__file__", PyString_FromString(path));
#endif
return module;
}

static PyMethodDef spt3g_dload_methods[] = {
{"load_dynamic", (PyCFunction)&load_dynamic_library, METH_VARARGS | METH_KEYWORDS,
"\n"
"Load a compiled extension module. No restriction is placed on the relationship\n"
"between the name of the module and the name of the dynamic library file which\n"
"implements it; e.g. module `foo` need not reside in `foo.so`.\n"
"\n"
"Arguments\n"
"---------\n"
"full_name : str\n"
" The module's fully qualified name\n"
"name : str\n"
" The module's name\n"
"path : str\n"
" The path to the dynamic library which implements the module\n"
"\n"
"Returns\n"
"-------\n"
"The loaded module\n"
},
{NULL, NULL, 0, NULL}
};

#if PY_MAJOR_VERSION >= 3
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"spt3g_dload",
"A beach-head module for loading other compiled spt3g modules",
0,
spt3g_dload_methods,
NULL,
NULL,
NULL,
NULL,
};
#endif

PyMODINIT_FUNC
#if PY_MAJOR_VERSION >= 3
PyInit_dload(void){
#else
initdload(void){
#endif
PyObject* module;

#if PY_MAJOR_VERSION >= 3
module = PyModule_Create(&moduledef);
PyObject_SetAttrString(module, "__version__", PyUnicode_FromString("1.0.0"));
return module;
#else
module = Py_InitModule3("spt3g_dload", spt3g_dload_methods,
"A beach-head module for loading other compiled spt3g modules");
PyObject_SetAttrString(module, "__version__", PyString_FromString("1.0.0"));
PyDict_SetItemString(PyImport_GetModuleDict(),_Py_PackageContext,module);
#endif
}
2 changes: 1 addition & 1 deletion core/src/python.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -368,7 +368,7 @@ numpy_container_from_object(boost::python::object v)
numpy_vector_infrastructure(std::complex<double>, cxdouble, "Zd");
numpy_vector_infrastructure(std::complex<float>, cxfloat, "Zf");

BOOST_PYTHON_MODULE(core)
SPT3G_PYTHON_MODULE(core)
{
bp::docstring_options docopts(true, true, false);

Expand Down
2 changes: 1 addition & 1 deletion dfmux/src/python.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace bp = boost::python;

BOOST_PYTHON_MODULE(dfmux)
SPT3G_PYTHON_MODULE(dfmux)
{
// Python bindings dependencies
bp::import("spt3g.core");
Expand Down
2 changes: 1 addition & 1 deletion doc/buildsystem.rst
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ Every C++ library must contain a file declaring the library to Python. This file
#include <pybindings.h>
#include <boost/python.hpp>

BOOST_PYTHON_MODULE(newthing)
SPT3G_PYTHON_MODULE(newthing)
{
boost::python::import("spt3g.core"); // Import core python bindings

Expand Down
2 changes: 1 addition & 1 deletion gcp/src/python.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

namespace bp = boost::python;

BOOST_PYTHON_MODULE(gcp)
SPT3G_PYTHON_MODULE(gcp)
{
bp::import("spt3g.core");
bp::docstring_options docopts(true, true, false);
Expand Down
2 changes: 1 addition & 1 deletion maps/src/python.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

namespace bp = boost::python;

BOOST_PYTHON_MODULE(maps)
SPT3G_PYTHON_MODULE(maps)
{
bp::import("spt3g.core");
bp::docstring_options docopts(true, true, false);
Expand Down

0 comments on commit a9d32d5

Please sign in to comment.