boost python shared_ptr + virtual + overriding in python - boost-python

I would appreciate help making boost-python use a wrapper class I defined for an abstract base class to wrap derived classes.
I have an abstract C++ base class baz, and derived classes foo (not exposed) and bar (exposed to python). There is a factory function returning boost::shared_ptr<baz> pointing to instances of foo, and here I am hoping to get the boost::shared_ptr<baz> from the factory to be wrapped in baz_callback. I would also like to override pure in python, which seems to work for the base class baz (overridden in the mumble python class), but not for the derived class bar (overridden in jungle).
For illustration, I extended an example from the python wiki, see below. All the printouts should have a 1 as second digit, coming from baz_callback::pure, but only the mumble version gets that right. For jungle, the python verion of pure is ignored, and the C++ version used instead.
File override_shared.cc:
#include <boost/python/class.hpp>
#include <boost/python/module_init.hpp>
#include <boost/python/def.hpp>
#include <boost/python/call_method.hpp>
#include <boost/ref.hpp>
#include <boost/utility.hpp>
#include <boost/python/register_ptr_to_python.hpp>
#include <boost/python/pure_virtual.hpp>
using namespace boost::python;
struct baz {
virtual int pure(int) = 0;
int calls_pure(int x) { return pure(x) + 1000; }
};
struct foo : baz {
int pure(int i) { return i+10; };
};
struct bar : baz {
int pure(int i) { return i+30; };
};
struct baz_callback : baz {
baz_callback(PyObject *p) : self(p) {}
int pure(int x) { return 100 + call_method<int>(self, "pure", x); }
PyObject *self;
};
boost::shared_ptr<baz> make_foo()
{
return boost::shared_ptr<foo>(new foo());
}
boost::shared_ptr<baz> make_bar()
{
return boost::shared_ptr<bar>(new bar());
}
BOOST_PYTHON_MODULE_INIT(liboverride_shared)
{
class_<baz, boost::shared_ptr<baz_callback>, boost::noncopyable>("baz")
.def("pure", pure_virtual(&baz::pure))
.def("calls_pure", &baz::calls_pure);
class_<bar, bases<baz> >("bar");
def("make_foo", make_foo);
def("make_bar", make_bar);
// I suspect the problem lies here:
register_ptr_to_python< boost::shared_ptr<baz> >();
}
File override_shared.py:
from liboverride_shared import *
f = make_foo()
class mumble(baz):
def pure(self, x): return x + 20
m = mumble()
class jungle(bar):
def pure(self, x): return x + 40
j = jungle()
def check(name, bar, i, expected):
actual = bar.calls_pure(i)
msg = "ok" if actual == expected else ("bad, expected %d" % expected)
print (name, actual, "--", msg)
check("foo", f, 1, 1111)
check("mumble", m, 2, 1122)
check("jungle", j, 4, 1144)
File CMakeLists.txt:
CMAKE_MINIMUM_REQUIRED(VERSION 3.0)
FIND_PACKAGE(PythonInterp 3 REQUIRED)
FIND_PACKAGE(PythonLibs "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}" REQUIRED)
FIND_PACKAGE(Boost 1.58.0 REQUIRED COMPONENTS "python-py${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}")
INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIRS})
ADD_LIBRARY(override_shared SHARED
override_shared.cc
)
TARGET_LINK_LIBRARIES(override_shared ${Boost_LIBRARIES} ${PYTHON_LIBRARIES})

Related

Fully-transparent exposure of Eigen::Vector/Matrix types using pybind11

I have a simple class definition:
class State {
private:
Eigen::Vector3f m_data;
public:
State(const Eigen::Vector3f& state) : m_data(state) { }
Eigen::Vector3f get() const { return m_data; }
void set(const Eigen::Vector3f& _state) { m_data = _state; }
std::string repr() const {
return "state data: [x=" + std::to_string(m_data[0]) + ", y=" + std::to_string(m_data[1]) + ", theta=" + std::to_string(m_data[2]) + "]";
}
};
I then expose the above in python with pybind11:
namespace py = pybind11;
PYBIND11_MODULE(bound_state, m) {
m.doc() = "python bindings for State";
py::class_<State>(m, "State")
.def(py::init<Eigen::Vector3f>())
.def("get", &_State::get)
.def("set", &_State::set)
.def("__repr__", &_State::repr);
}
And everything works fine; I am able to import this module into python and construct a State instance with a numpy array. This isn't exactly what I want though. I want to be able to access this object as if it were a numpy array; I want to be able to do something like the following in python:
import bound_state as bs
arr = np.array([1, 2, 3])
a = bs.State(arr)
print(a[0])
(the above throws a TypeError: 'bound_state.State' object does not support indexing)
In the past, I've used boost::python to expose lists by using add_property and this allowed indexing of the underlying data in C++. does pybind11 have something similar that can work with Eigen? Could someone provide an example showing how to expose a State instance that is indexable?
Per the API Docs, this can be done easily with the def_property method.
Turn this bit:
namespace py = pybind11;
PYBIND11_MODULE(bound_state, m) {
m.doc() = "python bindings for State";
py::class_<State>(m, "State")
.def(py::init<Eigen::Vector3f>())
.def("get", &State::get)
.def("set", &State::set)
.def("__repr__", &State::repr);
}
Into this:
namespace py = pybind11;
PYBIND11_MODULE(bound_state, m) {
m.doc() = "python bindings for State";
py::class_<State>(m, "State")
.def(py::init<Eigen::Vector3f>())
.def_property("m_data", &State::get, &State::set)
.def("__repr__", &State::repr);
}
Now, from the python-side, I can do:
import bound_state as bs
arr = np.array([1, 2, 3])
a = bs.State(arr)
print(a.m_data[0])
This is not exactly what I want, but is a step in the right direction.

How to display the result of function called using object reference in c++

#include "pch.h"
#include <iostream>
#include <cstdlib>
#include <ctime>
#include <stdio.h>
using namespace std;
class LetterDistribution
{
public: char district, trace;
public: int random_num;
public : LetterDistribution(){}
public: LetterDistribution(char dis)
{
district = dis;
trace = 'Z';
}
public: string LetterNumbers()
{
random_num = rand();
string letter_no ( district + " " + random_num);
return letter_no;
}
};
int main()
{
srand(time(0));
cout << "Enter district\n"<<endl;
char dis ;
cin >> dis;
LetterDistribution ld(dis);
cout << ld.LetterNumbers();
return 0;}
I am getting error in second last line inside main "cout << ld.LetterNumbers();". I am new to c++ , I have been working on C# earlier. I shall be thankful if someone could help me .
You have 2 issues in LetterNumbers function:
You can't add to string a number, you should convert the number to string first. you can do so by std::to_string(random_num)
You can't start concatenate string with a character, since character is like number in c++, and adding anything to number is a number. You should start from a string, even an empty one.
So the whole function can be something like:
string LetterNumbers()
{
random_num = rand();
string letter_no ( std::string("") + district + " " + std::to_string(random_num));
return letter_no;
}
Another issues: (but not errors!)
in c++ you can specify public: once, and everything after it is still public, until you change it. same thing for private and protected.
instead of <stdio.h> you should use <cstdio> which is the c++ wrapper for the c header.

pybind with array as class attribute

I want to wrap the following C++ code into python using pybind
class Galaxy {
public:
double x[3];
double v[3];
};
class GalaxyCatalogue {
public:
long n_tracers;
Galaxy *object;
GalaxyCatalogue(long n_tracers);
~GalaxyCatalogue();
};
GalaxyCatalogue::GalaxyCatalogue(long n_tracers) : n_tracers(n_tracers) {
std::cout << "from galaxies " << n_tracers << std::endl;
object = new Galaxy[n_tracers];
std::cout << "has been allocated " << std::endl;
}
GalaxyCatalogue::~GalaxyCatalogue() {
delete[] object;
}
The first problem I have is that Galaxy doesn't have a constructor, so I'm not sure what to do with that. Even if I declare an empty constructor I don't know how to treat the array in a way that I don't get an error when compiling. This is what I've tried:
#include <pybind11/pybind11.h>
#include <iostream>
namespace py = pybind11;
class Galaxy {
public:
Galaxy();
double x[3];
};
PYBIND11_MODULE(example, m){
py::class_<Galaxy>(m, "Galaxy")
.def(py::init<>())
.def_readwrite("x", &Galaxy::x);
}
This is how I compile it:
c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` gal.cpp -o example`python3-config --extension-suffix`
and this is the error I get:
In file included from gal.cpp:1:
/home/florpi/.conda/envs/virtualito/include/python3.5m/pybind11/pybind11.h: In instantiation of ‘pybind11::class_<type_, options>& pybind11::class_<type_, options>::def_readwrite(const char*, D C::*, const Extra& ...) [with C = Galaxy; D = double [3]; Extra = {}; type_ = Galaxy; options = {}]’:
gal.cpp:19:33: required from here
/home/florpi/.conda/envs/virtualito/include/python3.5m/pybind11/pybind11.h:1163:65: error: invalid array assignment
fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this));
~~~~~~^~~~~~~
In file included from gal.cpp:1:
/home/florpi/.conda/envs/virtualito/include/python3.5m/pybind11/pybind11.h:64:5: error: ‘pybind11::cpp_function::cpp_function(Func&&, const Extra& ...) [with Func = pybind11::class_<type_, options>::def_readwrite(const char*, D C::*, const Extra& ...) [with C = Galaxy; D = double [3]; Extra = {}; type_ = Galaxy; options = {}]::<lambda(pybind11::class_<Galaxy>::type&, const double (&)[3])>; Extra = {pybind11::is_method}; <template-parameter-1-3> = void]’, declared using local type ‘pybind11::class_<type_, options>::def_readwrite(const char*, D C::*, const Extra& ...) [with C = Galaxy; D = double [3]; Extra = {}; type_ = Galaxy; options = {}]::<lambda(pybind11::class_<Galaxy>::type&, const double (&)[3])>’, is used but never defined [-fpermissive]
cpp_function(Func &&f, const Extra&... extra) {
^~~~~~~~~~~~
Thank you in advance.
In C++, you can't assign directly to an array, which is what pybind11 is trying to do inside its wrapping magic. In general, C++ arrays are not great abstractions for numerical arrays. As you've noticed, you can't even say galaxy.x = other_galaxy.x.
Your best bet is to use a higher-level library for matrices and vectors, which will
a) give you a much better experience writing your C++
b) perform better
c) map more cleanly to Python
Eigen is a good choice. pybind11 automatically knows how to map Eigen matrices and vectors to numpy arrays. Your Galaxy would become:
class Galaxy {
public:
Eigen::Vector3d x;
Eigen::Vector3d v;
};
If you absolutely can't do this, you'll have to supply manual getter/setter functions to the property, where you do your own conversion to and from python types:
https://pybind11.readthedocs.io/en/master/classes.html?highlight=def_property#instance-and-static-fields

Implicit conversion between c++11 clocks/time_points

Is it possible to do implicit/explicit conversion between time_points of two C++11 clocks?
Motivation: chrono::durations provide means of storing time intervals from epoch, conceptually is not equal to a time_point of a custom clock that has an epoch on its own.
Having an implicit conversion between clocks eases up the use Howard Hinnant's date library <date/date.h> which provides means to manipulate and print out time_points of system clocks.
Example:
#include <date/date.h>
using namespace date;
namespace ch = std::chrono;
//
#define EPOCH_OFFSET 100
template<class Duration> using PosixTimePoint =
ch::time_point<ch::system_clock, Duration>;
typedef PosixTimePoint<ch::duration<long,std::micro>> PosixTimePointType;
struct SomeClock{
typedef ch::duration<long,std::micro> duration;
typedef ch::time_point<SomeClock> time_point;
...
static time_point now() noexcept {
using namespace std::chrono;
return time_point (
duration_cast<duration>(
system_clock::now().time_since_epoch()) + date::years(EPOCH_OFFSET) );
}
static PosixTimePoint<duration> to_posix( const time_point& tp ){...}
}
auto tp = SomeClock::now(); //<time_point<SomeClock,ch::duration<long,std::micro>>;
Objective: to convert tp so the std::stream conversions of date.h works and prints out the current time, which in my case is: 2017-12-24 17:02:56.000000
// std::cout << tp; compile error
std::cout << SomeClock::to_posix( tp ); // OK
Explicit cast: this could ease up readability, support conversion feature of the language and facilitate access to date.h routines.
long time_value = static_cast<long>( tp );
auto st = static_cast<PosixTimePointType>( tp );
std::cout << static_cast<PosixTimePointType>( tp );
I recommend mimicking the implementations of either date::utc_clock or date::tai_clock found in tz.h. For example utc_clock implements two functions to convert to and from sys_time:
template<typename Duration>
static
std::chrono::time_point<std::chrono::system_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
to_sys(const std::chrono::time_point<utc_clock, Duration>&);
template<typename Duration>
static
std::chrono::time_point<utc_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
from_sys(const std::chrono::time_point<std::chrono::system_clock, Duration>&);
So you can think of std::chrono::system_clock as a "hub". Any clock that implements these conversions can convert to any other clock which implements these conversions by bouncing off of system_clock under the covers. And to facilitate that bounce, date::clock_cast is introduced.
Additionally, utc_time can be used as the hub, if that is more efficient for your type. For example tai_clock implements:
template<typename Duration>
static
std::chrono::time_point<utc_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
to_utc(const std::chrono::time_point<tai_clock, Duration>&) NOEXCEPT;
template<typename Duration>
static
std::chrono::time_point<tai_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
from_utc(const std::chrono::time_point<utc_clock, Duration>&) NOEXCEPT;
clock_cast is smart enough to deal with this "dual-hub" system, so one can convert a clock that converts to/from utc_time, to another clock that uses sys_time as its hub.
If you also implement to_stream for your clock, then you can directly use format to format your clock::time_point. And clock_cast is likely to be useful in the implementation of your to_stream function.
Also from_stream can be used to hook your clock::time_point up to date::parse.
Search https://howardhinnant.github.io/date/tz.html for "clock_cast" for example uses of it. For your use case the to_sys/from_sys API appears to be the most useful. Just these two functions will allow you to use clock_cast between your SomeClock and any other clock in tz.h (and any other custom clock that meets these requirements).
Full Demo
#include "date/tz.h"
#include <iostream>
#include <sstream>
struct SomeClock
{
using duration = std::chrono::microseconds;
using rep = duration::rep;
using period = duration::period;
using time_point = std::chrono::time_point<SomeClock>;
static constexpr bool is_steady = false;
static time_point now() noexcept
{
return from_sys(date::floor<duration>(std::chrono::system_clock::now()));
}
static constexpr auto offset = date::sys_days{} - date::sys_days{date::year{1870}/1/1};
template<typename Duration>
static
date::sys_time<Duration>
to_sys(const std::chrono::time_point<SomeClock, Duration>& t)
{
return date::sys_time<Duration>{(t - offset).time_since_epoch()};
}
template<typename Duration>
static
std::chrono::time_point<SomeClock, Duration>
from_sys(const date::sys_time<Duration>& t)
{
return std::chrono::time_point<SomeClock, Duration>{(t + offset).time_since_epoch()};
}
};
template <class Duration>
using SomeTime = std::chrono::time_point<SomeClock, Duration>;
constexpr date::days SomeClock::offset;
template <class CharT, class Traits, class Duration>
std::basic_ostream<CharT, Traits>&
to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
const SomeTime<Duration>& t)
{
return date::to_stream(os, fmt, date::clock_cast<std::chrono::system_clock>(t));
}
template <class CharT, class Traits, class Duration>
std::basic_ostream<CharT, Traits>&
operator<<(std::basic_ostream<CharT, Traits>& os, const SomeTime<Duration>& t)
{
const CharT fmt[] = {'%', 'F', ' ', '%', 'T', CharT{}};
return to_stream(os, fmt, t);
}
template <class Duration, class CharT, class Traits, class Alloc = std::allocator<CharT>>
std::basic_istream<CharT, Traits>&
from_stream(std::basic_istream<CharT, Traits>& is, const CharT* fmt,
SomeTime<Duration>& tp, std::basic_string<CharT, Traits, Alloc>* abbrev = nullptr,
std::chrono::minutes* offset = nullptr)
{
using namespace date;
sys_time<Duration> st;
date::from_stream(is, fmt, st, abbrev, offset);
if (!is.fail())
tp = clock_cast<SomeClock>(st);
return is;
}
int
main()
{
std::cout << SomeClock::now() << '\n';
std::cout << date::format("%a, %b %d, %Y\n", SomeClock::now());
std::istringstream in{"2017-12-24 19:52:30"};
SomeClock::time_point t;
in >> date::parse("%F %T", t);
std::cout << t << '\n';
}

How to expose aligned class with boost.python

When trying to expose aligned class like this:
class __declspec(align(16)) foo
{
public:
void foo_method() {}
};
BOOST_PYTHON_MODULE(foo_module)
{
class_<foo>("foo")
.def("foo_method", &foo::foo_method);
}
You end up with error (msvs 2010):
error C2719: 'unnamed-parameter': formal parameter with __declspec(align('16')) won't be aligned,
see reference to class template instantiation 'boost::python::converter::as_to_python_function<T,ToPython>' being compiled
The solution I found so far, is to use smart pointer to store object:
BOOST_PYTHON_MODULE(foo_module)
{
class_<foo, boost::shared_ptr<foo>, boost::noncopyable>("foo")
.def("foo_method", &foo::foo_method);
}
Isn't there a better solution? This is quite annoying, because you should wrap all your functions returning objects by value to return smart pointers instead, and performance also degrades.
I had the same problem and wanted a solution that doesn't involve shared_ptr. It involves specializing some boost::python classes to make sure we get a storage area big enough to be able to align our object within it.
I have written a somewhat long blog post explaining how I arrived at this solution here. Below is the solution I found. I feel it is quite a hack, so maybe it will break other things. But so far it seems to work and I haven't found anything better.
I was trying to expose an Eigen::Quaternionf (which requires 16 bytes alignment) :
bp::class_<Quaternionf>("Quaternion", bp::init<float, float, float, float>())
.def(bp::init<Matrix3f>())
.add_property("w", get_prop_const(&Quaternionf::w))
.add_property("x", get_prop_const(&Quaternionf::x))
.add_property("y", get_prop_const(&Quaternionf::y))
.add_property("z", get_prop_const(&Quaternionf::z))
.def("matrix", &Quaternionf::matrix)
.def("rotvec", &quaternion_to_rotvec);
The solution involves specializing 3 classes :
boost::python::objects::instance to request 16 bytes more than what our type requires to ensure we can align
...
union
{
align_t align;
char bytes[sizeof(Data) + 16];
} storage;
...
boost::python::objects::make_instance_impl to correctly set the Py_SIZE of our instance
...
Holder* holder = Derived::construct(
&instance->storage, (PyObject*)instance, x);
holder->install(raw_result);
// Note the position of the internally-stored Holder,
// for the sake of destruction
// Since the holder not necessarily allocated at the start of
// storage (to respect alignment), we have to add the holder
// offset relative to storage
size_t holder_offset = reinterpret_cast<size_t>(holder)
- reinterpret_cast<size_t>(&instance->storage)
+ offsetof(instance_t, storage);
Py_SIZE(instance) = holder_offset;
...
boost::python::objects::make_instance so that the construct method will align the holder in the storage
...
static inline QuaternionfHolder* construct(void* storage, PyObject* instance, reference_wrapper<Quaternionf const> x)
{
// From the specialized make_instance_impl above, we are guaranteed to
// be able to align our storage
void* aligned_storage = reinterpret_cast<void*>(
(reinterpret_cast<size_t>(storage) & ~(size_t(15))) + 16);
QuaternionfHolder* new_holder = new (aligned_storage)
QuaternionfHolder(instance, x);
return new_holder;
}
...
The full code is below :
typedef bp::objects::value_holder<Eigen::Quaternionf> QuaternionfHolder;
namespace boost { namespace python { namespace objects {
using namespace Eigen;
//template <class Data = char>
template<>
struct instance<QuaternionfHolder>
{
typedef QuaternionfHolder Data;
PyObject_VAR_HEAD
PyObject* dict;
PyObject* weakrefs;
instance_holder* objects;
typedef typename type_with_alignment<
::boost::alignment_of<Data>::value
>::type align_t;
union
{
align_t align;
char bytes[sizeof(Data) + 16];
} storage;
};
// Adapted from boost/python/object/make_instance.hpp
//template <class T, class Holder, class Derived>
template<class Derived>
struct make_instance_impl<Quaternionf, QuaternionfHolder, Derived>
{
typedef Quaternionf T;
typedef QuaternionfHolder Holder;
typedef objects::instance<Holder> instance_t;
template <class Arg>
static inline PyObject* execute(Arg& x)
{
BOOST_MPL_ASSERT((mpl::or_<is_class<T>, is_union<T> >));
PyTypeObject* type = Derived::get_class_object(x);
if (type == 0)
return python::detail::none();
PyObject* raw_result = type->tp_alloc(
type, objects::additional_instance_size<Holder>::value);
if (raw_result != 0)
{
python::detail::decref_guard protect(raw_result);
instance_t* instance = (instance_t*)raw_result;
// construct the new C++ object and install the pointer
// in the Python object.
//Derived::construct(&instance->storage, (PyObject*)instance, x)->install(raw_result);
Holder* holder = Derived::construct(
&instance->storage, (PyObject*)instance, x);
holder->install(raw_result);
// Note the position of the internally-stored Holder,
// for the sake of destruction
// Since the holder not necessarily allocated at the start of
// storage (to respect alignment), we have to add the holder
// offset relative to storage
size_t holder_offset = reinterpret_cast<size_t>(holder)
- reinterpret_cast<size_t>(&instance->storage)
+ offsetof(instance_t, storage);
Py_SIZE(instance) = holder_offset;
// Release ownership of the python object
protect.cancel();
}
return raw_result;
}
};
//template <class T, class Holder>
template<>
struct make_instance<Quaternionf, QuaternionfHolder>
: make_instance_impl<Quaternionf, QuaternionfHolder, make_instance<Quaternionf,QuaternionfHolder> >
{
template <class U>
static inline PyTypeObject* get_class_object(U&)
{
return converter::registered<Quaternionf>::converters.get_class_object();
}
static inline QuaternionfHolder* construct(void* storage, PyObject* instance, reference_wrapper<Quaternionf const> x)
{
LOG(INFO) << "Into make_instance";
LOG(INFO) << "storage : " << storage;
LOG(INFO) << "&x : " << x.get_pointer();
LOG(INFO) << "&x alignment (0 = aligned): " << (reinterpret_cast<size_t>(x.get_pointer()) & 0xf);
// From the specialized make_instance_impl above, we are guaranteed to
// be able to align our storage
void* aligned_storage = reinterpret_cast<void*>(
(reinterpret_cast<size_t>(storage) & ~(size_t(15))) + 16);
QuaternionfHolder* new_holder = new (aligned_storage) QuaternionfHolder(instance, x);
LOG(INFO) << "&new_holder : " << new_holder;
return new_holder;
//return new (storage) QuaternionfHolder(instance, x);
}
};
}}} // namespace boost::python::objects