postgresql timestamp to std::chrono value - postgresql

What is the appropriate way to work with the the postgresql datatype "timestamp without timezone" from c++ (libpqxx)? I haven't been able to find a way to do this yet.
I am restricted to the "timestamp without timezone" datatype in the postgresql and the environment is running utc time. I was hoping to find a mapping to a std::chrono::system_clock::time_point member but I can't find a such in libpqxx.
//s has a time_point var and r is a pqxx::result, r[0] is sensible
s.creationtime = r[0]["creationtime"].as<std::chrono::system_clock::time_point>();

With help from boost:
s.creationtime = makeTimePoint(r[0]["creationtime"].as<string>());
makeTimePoint:
std::chrono::system_clock::time_point makeTimePoint(const std::string& s)
{
using namespace boost::posix_time;
using namespace std::chrono;
const ptime ts = time_from_string(s);
auto seconds = to_time_t(ts);
time_duration td = ts.time_of_day();
auto microseconds = td.fractional_seconds();
auto d = std::chrono::seconds{seconds} + std::chrono::microseconds{microseconds};
system_clock::time_point tp{duration_cast<system_clock::duration>(d)};
return tp;
}

The C++20 spec introduces a family of chrono::time_points called local_time:
// [time.clock.local], local time
struct local_t {};
template<class Duration>
using local_time = time_point<local_t, Duration>;
using local_seconds = local_time<seconds>;
using local_days = local_time<days>;
These time_points represent a "timestamp without a timezone".
There exists a free, open-source preview of this C++20 library here:
https://github.com/HowardHinnant/date
which is currently in use by other projects around the globe. This library has a few minor changes from the C++20 spec, such as putting everything in namespace date instead of namespace std::chrono.
Example program using this library:
#include "date/date.h"
#include <iostream>
int
main()
{
using namespace date;
using namespace std::chrono;
int y = 2019;
int m = 8;
int d = 28;
int H = 14;
int M = 42;
int S = 16;
int US = 500'000;
local_time<microseconds> lt = local_days{year{y}/m/d} + hours{H} +
minutes{M} + seconds{S} + microseconds{US};
std::cout << lt << '\n';
}
Output:
2019-08-28 14:42:16.500000

Related

Bad address error when comparing Strings within BPF

I have an example program I am running here to see if the substring matches the string and then print them out. So far, I am having trouble running the program due to a bad address. I am wondering if there is a way to fix this problem? I have attached the entire code but my problem is mostly related to isSubstring.
#include <uapi/linux/bpf.h>
#define ARRAYSIZE 64
struct data_t {
char buf[ARRAYSIZE];
};
BPF_ARRAY(lookupTable, struct data_t, ARRAYSIZE);
//char name[20];
//find substring in a string
static bool isSubstring(struct data_t stringVal)
{
char substring[] = "New York";
int M = sizeof(substring);
int N = sizeof(stringVal.buf) - 1;
/* A loop to slide pat[] one by one */
for (int i = 0; i <= N - M; i++) {
int j;
/* For current index i, check for
pattern match */
for (j = 0; j < M; j++)
if (stringVal.buf[i + j] != substring[j])
break;
if (j == M)
return true;
}
return false;
}
int Test(void *ctx)
{
#pragma clang loop unroll(full)
for (int i = 0; i < ARRAYSIZE; i++) {
int k = i;
struct data_t *line = lookupTable.lookup(&k);
if (line) {
// bpf_trace_printk("%s\n", key->buf);
if (isSubstring(*line)) {
bpf_trace_printk("%s\n", line->buf);
}
}
}
return 0;
}
My python code here:
import ctypes
from bcc import BPF
b = BPF(src_file="hello.c")
lookupTable = b["lookupTable"]
#add hello.csv to the lookupTable array
f = open("hello.csv","r")
contents = f.readlines()
for i in range(0,len(contents)):
string = contents[i].encode('utf-8')
print(len(string))
lookupTable[ctypes.c_int(i)] = ctypes.create_string_buffer(string, len(string))
f.close()
b.attach_kprobe(event=b.get_syscall_fnname("clone"), fn_name="Test")
b.trace_print()
Edit: Forgot to add the error: It's really long and can be found here: https://pastebin.com/a7E9L230
I think the most interesting part of the error is near the bottom where it mentions:
The sequence of 8193 jumps is too complex.
And a little bit farther down mentions: Bad Address.
The verifier checks all branches in your program. Each time it sees a jump instruction, it pushes the new branch to its “stack of branches to check”. This stack has a limit (BPF_COMPLEXITY_LIMIT_JMP_SEQ, currently 8192) that you are hitting, as the verifier tells you. “Bad Address” is just the translation of kernel's errno value which is set to -EFAULT in that case.
Not sure how to fix it though, you could try:
With smaller strings, or
On a 5.3+ kernel (which supports bounded loops): without unrolling the loop with clang (I don't know if it would help).

How to return a jsonb object from a PostgreSQL c extension function?

How can I return a simple jsonb object in a PostgreSQL function written in C?
I don't know enough about postgres server side programming. And below is my attempt to return a simple json/jsonb object based on the C source code for hstore_to_jsonb_loose, which is the closest example I can find. I am trying to return {"hi": -101} from the C function, but got an error:
=> ERROR: unexpected jsonb type as object key
Can anyone help exaplain how to get this right?
My C code is:
PG_FUNCTION_INFO_V1(test_return_jsonb);
Datum
test_return_jsonb( PG_FUNCTION_ARGS) {
JsonbParseState *state = NULL;
JsonbValue *res;
StringInfoData tmp;
initStringInfo(&tmp);
(void) pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL);
JsonbValue key, val;
//key
key.type = jbvString;
key.val.string.len = 2;
key.val.string.val = "hi";
Datum numd;
//value
val.type = jbvNumeric;
numd = DirectFunctionCall3(numeric_in, CStringGetDatum("-101"), //!tmp.data),
ObjectIdGetDatum(InvalidOid), Int32GetDatum(-1));
val.val.numeric = DatumGetNumeric(numd);
(void) pushJsonbValue(&state, WJB_VALUE, &val);
res = pushJsonbValue(&state, WJB_END_OBJECT, NULL);
PG_RETURN_POINTER(JsonbValueToJsonb(res));
}
And the SQL interface code is:
CREATE OR REPLACE FUNCTION test_return_jsonb()
RETURNS jsonb
AS '$libdir/pgtest', 'test_return_jsonb'
LANGUAGE 'c' IMMUTABLE STRICT COST 100; -- Guessed cost
This is with PostgreSQL 12 and Ubuntu 18.04 LTS.
I'm learning too currently and encountered the same issue. I solved it like following (not sure if this is the right way, but it works for now):
// Defined in "/src/backend/utils/adt/numeric.c"
extern Datum int8_numeric(PG_FUNCTION_ARGS);
extern Datum float8_numeric(PG_FUNCTION_ARGS);
extern Datum numeric_int8(PG_FUNCTION_ARGS);
extern Datum numeric_float8(PG_FUNCTION_ARGS);
PG_FUNCTION_INFO_V1(test_return_jsonb);
Datum test_return_jsonb(PG_FUNCTION_ARGS) {
JsonbPair *pair = palloc(sizeof(JsonbPair));
pair->key.type = jbvString;
pair->key.val.string.len = 3;
pair->key.val.string.val = "foo";
pair->value.type = jbvNumeric;
pair->value.val.numeric = DatumGetNumeric(DirectFunctionCall1(int8_numeric, (int64_t)100));
JsonbValue *object = palloc(sizeof(JsonbValue));
object->type = jbvObject;
object->val.object.nPairs = 1;
object->val.object.pairs = pair;
PG_RETURN_POINTER(JsonbValueToJsonb(object));
}

pybind with array as class attribute

I want to wrap the following C++ code into python using pybind
class Galaxy {
public:
double x[3];
double v[3];
};
class GalaxyCatalogue {
public:
long n_tracers;
Galaxy *object;
GalaxyCatalogue(long n_tracers);
~GalaxyCatalogue();
};
GalaxyCatalogue::GalaxyCatalogue(long n_tracers) : n_tracers(n_tracers) {
std::cout << "from galaxies " << n_tracers << std::endl;
object = new Galaxy[n_tracers];
std::cout << "has been allocated " << std::endl;
}
GalaxyCatalogue::~GalaxyCatalogue() {
delete[] object;
}
The first problem I have is that Galaxy doesn't have a constructor, so I'm not sure what to do with that. Even if I declare an empty constructor I don't know how to treat the array in a way that I don't get an error when compiling. This is what I've tried:
#include <pybind11/pybind11.h>
#include <iostream>
namespace py = pybind11;
class Galaxy {
public:
Galaxy();
double x[3];
};
PYBIND11_MODULE(example, m){
py::class_<Galaxy>(m, "Galaxy")
.def(py::init<>())
.def_readwrite("x", &Galaxy::x);
}
This is how I compile it:
c++ -O3 -Wall -shared -std=c++11 -fPIC `python3 -m pybind11 --includes` gal.cpp -o example`python3-config --extension-suffix`
and this is the error I get:
In file included from gal.cpp:1:
/home/florpi/.conda/envs/virtualito/include/python3.5m/pybind11/pybind11.h: In instantiation of ‘pybind11::class_<type_, options>& pybind11::class_<type_, options>::def_readwrite(const char*, D C::*, const Extra& ...) [with C = Galaxy; D = double [3]; Extra = {}; type_ = Galaxy; options = {}]’:
gal.cpp:19:33: required from here
/home/florpi/.conda/envs/virtualito/include/python3.5m/pybind11/pybind11.h:1163:65: error: invalid array assignment
fset([pm](type &c, const D &value) { c.*pm = value; }, is_method(*this));
~~~~~~^~~~~~~
In file included from gal.cpp:1:
/home/florpi/.conda/envs/virtualito/include/python3.5m/pybind11/pybind11.h:64:5: error: ‘pybind11::cpp_function::cpp_function(Func&&, const Extra& ...) [with Func = pybind11::class_<type_, options>::def_readwrite(const char*, D C::*, const Extra& ...) [with C = Galaxy; D = double [3]; Extra = {}; type_ = Galaxy; options = {}]::<lambda(pybind11::class_<Galaxy>::type&, const double (&)[3])>; Extra = {pybind11::is_method}; <template-parameter-1-3> = void]’, declared using local type ‘pybind11::class_<type_, options>::def_readwrite(const char*, D C::*, const Extra& ...) [with C = Galaxy; D = double [3]; Extra = {}; type_ = Galaxy; options = {}]::<lambda(pybind11::class_<Galaxy>::type&, const double (&)[3])>’, is used but never defined [-fpermissive]
cpp_function(Func &&f, const Extra&... extra) {
^~~~~~~~~~~~
Thank you in advance.
In C++, you can't assign directly to an array, which is what pybind11 is trying to do inside its wrapping magic. In general, C++ arrays are not great abstractions for numerical arrays. As you've noticed, you can't even say galaxy.x = other_galaxy.x.
Your best bet is to use a higher-level library for matrices and vectors, which will
a) give you a much better experience writing your C++
b) perform better
c) map more cleanly to Python
Eigen is a good choice. pybind11 automatically knows how to map Eigen matrices and vectors to numpy arrays. Your Galaxy would become:
class Galaxy {
public:
Eigen::Vector3d x;
Eigen::Vector3d v;
};
If you absolutely can't do this, you'll have to supply manual getter/setter functions to the property, where you do your own conversion to and from python types:
https://pybind11.readthedocs.io/en/master/classes.html?highlight=def_property#instance-and-static-fields

Implicit conversion between c++11 clocks/time_points

Is it possible to do implicit/explicit conversion between time_points of two C++11 clocks?
Motivation: chrono::durations provide means of storing time intervals from epoch, conceptually is not equal to a time_point of a custom clock that has an epoch on its own.
Having an implicit conversion between clocks eases up the use Howard Hinnant's date library <date/date.h> which provides means to manipulate and print out time_points of system clocks.
Example:
#include <date/date.h>
using namespace date;
namespace ch = std::chrono;
//
#define EPOCH_OFFSET 100
template<class Duration> using PosixTimePoint =
ch::time_point<ch::system_clock, Duration>;
typedef PosixTimePoint<ch::duration<long,std::micro>> PosixTimePointType;
struct SomeClock{
typedef ch::duration<long,std::micro> duration;
typedef ch::time_point<SomeClock> time_point;
...
static time_point now() noexcept {
using namespace std::chrono;
return time_point (
duration_cast<duration>(
system_clock::now().time_since_epoch()) + date::years(EPOCH_OFFSET) );
}
static PosixTimePoint<duration> to_posix( const time_point& tp ){...}
}
auto tp = SomeClock::now(); //<time_point<SomeClock,ch::duration<long,std::micro>>;
Objective: to convert tp so the std::stream conversions of date.h works and prints out the current time, which in my case is: 2017-12-24 17:02:56.000000
// std::cout << tp; compile error
std::cout << SomeClock::to_posix( tp ); // OK
Explicit cast: this could ease up readability, support conversion feature of the language and facilitate access to date.h routines.
long time_value = static_cast<long>( tp );
auto st = static_cast<PosixTimePointType>( tp );
std::cout << static_cast<PosixTimePointType>( tp );
I recommend mimicking the implementations of either date::utc_clock or date::tai_clock found in tz.h. For example utc_clock implements two functions to convert to and from sys_time:
template<typename Duration>
static
std::chrono::time_point<std::chrono::system_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
to_sys(const std::chrono::time_point<utc_clock, Duration>&);
template<typename Duration>
static
std::chrono::time_point<utc_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
from_sys(const std::chrono::time_point<std::chrono::system_clock, Duration>&);
So you can think of std::chrono::system_clock as a "hub". Any clock that implements these conversions can convert to any other clock which implements these conversions by bouncing off of system_clock under the covers. And to facilitate that bounce, date::clock_cast is introduced.
Additionally, utc_time can be used as the hub, if that is more efficient for your type. For example tai_clock implements:
template<typename Duration>
static
std::chrono::time_point<utc_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
to_utc(const std::chrono::time_point<tai_clock, Duration>&) NOEXCEPT;
template<typename Duration>
static
std::chrono::time_point<tai_clock, typename std::common_type<Duration, std::chrono::seconds>::type>
from_utc(const std::chrono::time_point<utc_clock, Duration>&) NOEXCEPT;
clock_cast is smart enough to deal with this "dual-hub" system, so one can convert a clock that converts to/from utc_time, to another clock that uses sys_time as its hub.
If you also implement to_stream for your clock, then you can directly use format to format your clock::time_point. And clock_cast is likely to be useful in the implementation of your to_stream function.
Also from_stream can be used to hook your clock::time_point up to date::parse.
Search https://howardhinnant.github.io/date/tz.html for "clock_cast" for example uses of it. For your use case the to_sys/from_sys API appears to be the most useful. Just these two functions will allow you to use clock_cast between your SomeClock and any other clock in tz.h (and any other custom clock that meets these requirements).
Full Demo
#include "date/tz.h"
#include <iostream>
#include <sstream>
struct SomeClock
{
using duration = std::chrono::microseconds;
using rep = duration::rep;
using period = duration::period;
using time_point = std::chrono::time_point<SomeClock>;
static constexpr bool is_steady = false;
static time_point now() noexcept
{
return from_sys(date::floor<duration>(std::chrono::system_clock::now()));
}
static constexpr auto offset = date::sys_days{} - date::sys_days{date::year{1870}/1/1};
template<typename Duration>
static
date::sys_time<Duration>
to_sys(const std::chrono::time_point<SomeClock, Duration>& t)
{
return date::sys_time<Duration>{(t - offset).time_since_epoch()};
}
template<typename Duration>
static
std::chrono::time_point<SomeClock, Duration>
from_sys(const date::sys_time<Duration>& t)
{
return std::chrono::time_point<SomeClock, Duration>{(t + offset).time_since_epoch()};
}
};
template <class Duration>
using SomeTime = std::chrono::time_point<SomeClock, Duration>;
constexpr date::days SomeClock::offset;
template <class CharT, class Traits, class Duration>
std::basic_ostream<CharT, Traits>&
to_stream(std::basic_ostream<CharT, Traits>& os, const CharT* fmt,
const SomeTime<Duration>& t)
{
return date::to_stream(os, fmt, date::clock_cast<std::chrono::system_clock>(t));
}
template <class CharT, class Traits, class Duration>
std::basic_ostream<CharT, Traits>&
operator<<(std::basic_ostream<CharT, Traits>& os, const SomeTime<Duration>& t)
{
const CharT fmt[] = {'%', 'F', ' ', '%', 'T', CharT{}};
return to_stream(os, fmt, t);
}
template <class Duration, class CharT, class Traits, class Alloc = std::allocator<CharT>>
std::basic_istream<CharT, Traits>&
from_stream(std::basic_istream<CharT, Traits>& is, const CharT* fmt,
SomeTime<Duration>& tp, std::basic_string<CharT, Traits, Alloc>* abbrev = nullptr,
std::chrono::minutes* offset = nullptr)
{
using namespace date;
sys_time<Duration> st;
date::from_stream(is, fmt, st, abbrev, offset);
if (!is.fail())
tp = clock_cast<SomeClock>(st);
return is;
}
int
main()
{
std::cout << SomeClock::now() << '\n';
std::cout << date::format("%a, %b %d, %Y\n", SomeClock::now());
std::istringstream in{"2017-12-24 19:52:30"};
SomeClock::time_point t;
in >> date::parse("%F %T", t);
std::cout << t << '\n';
}

QCustomPlot using timestamp to set Date along one axis

I am having trouble setting the date properly. Basically I have timestamp, open, close, high, low, volume stored line by line in a text file (downloaded using Yahoo API). My program then reads each line and converts it to a QStringList. It then puts each item in the list into the appropriate QVector (dates[], open[], close[], high[], low[], volume[]) converting each item to a double. Here is where the problem is. It appears that the precision is lost during the conversion. The dates always show as periods back in 1970 when the actually timestamp is in fact a date from a few days ago.
#include "dialog.h"
#include "ui_dialog.h"
#include<QFile>
#include<QTextStream>
#include<string>
#include<iostream>
#include<ctime>
using namespace std;
Dialog::Dialog(QWidget *parent) :
QDialog(parent),
ui(new Ui::Dialog)
{
ui->setupUi(this);
QStringList lines;
QString line;
QVector<double> dates;
QVector<double> high;
QVector<double> low;
QVector<double> open;
QVector<double> close;
QVector<double> volume;
QFile file ("YHOO.cvs");
if(file.open(QIODevice::ReadOnly))
{
QTextStream in(&file);
while (!in.atEnd())
{
line = in.readLine();
lines = line.split(",");
dates.append(lines[0].toDouble());
close.append(lines[1].toDouble());
high.append(lines[2].toDouble());
low.append(lines[3].toDouble());
open.append(lines[4].toDouble());
volume.append(lines[5].toInt());
}
file.close();
}
else{
QMessageBox::information(0,"info",file.errorString());
}
ui->plot->addGraph();
ui->plot->graph(0)->setData(dates, high);
ui->plot->xAxis->setTickLabelType(QCPAxis::ltDateTime);
ui->plot->xAxis->setDateTimeFormat("MM/dd/yyyy");
QPen pen;
pen.setColor(QColor(200,200,200));
ui->plot->graph(0)->setPen(pen);
ui->plot->graph(0)->setLineStyle(QCPGraph::lsLine);
ui->plot->graph(0)->setBrush(QBrush(QColor(160,50,150)));
ui->plot->xAxis->setRange(dates[0], dates[dates.length()-1]);
ui->plot->yAxis->setRange(*std::min_element(high.begin(), high.end()),*std::max_element(high.begin(),high.end()));
}
Dialog::~Dialog()
{
delete ui;
}
YHOO.cvs
20140227,30.1000,30.1600,28.4100,29.7000,2351300
20140228,28.3000,32.0000,27.0000,29.2000,3781000
20140303,28.1900,28.9100,26.8900,27.3000,1664900
20140304,30.0400,30.3800,28.6300,28.8500,2341700
20140305,28.5500,29.5000,28.4900,29.2400,7314100
20140306,27.1700,29.0100,27.1500,28.7600,3007300
20140307,27.2000,28.3200,26.7100,27.8400,2961800
20140310,28.2400,28.5000,27.3500,27.7200,1622100
20140311,27.5300,28.7400,27.1800,28.4400,1745200
20140312,28.5400,28.7400,27.3500,27.4700,2206300
I figured it out. Turns out the first column of data is a date and not a timestamp. I came up with this function to convert the QString to a double value timestamp.
double timeStamp(QString qs){
char tempDate[10];
memcpy(tempDate, qs.toStdString().c_str(), 10);
char date[] = " ";
date[0] = tempDate[0];
date[1] = tempDate[1];
date[2] = tempDate[2];
date[3] = tempDate[3];
date[4] = '\0';
date[5] = tempDate[4];
date[6] = tempDate[5];
date[7] = '\0';
date[8] = tempDate[6];
date[9] = tempDate[7];
struct tm tmdate = {0};
tmdate.tm_year = atoi(&date[0]) - 1900;
tmdate.tm_mon = atoi(&date[5]) - 1;
tmdate.tm_mday = atoi(&date[8]);
time_t t = mktime( &tmdate );
double actual_time_sec = difftime(t,0);
return actual_time_sec;
}