why the diesel new pagination sql did not work - postgresql

Question

I am using diesel diesel = { version = "1.4.8", features = ["postgres","64-column-tables","chrono"] } to do a pagination with rust 1.59.0, this is the key part to do the pagination query in diesel:
use diesel::pg::Pg;
use diesel::query_builder::{AstPass, QueryFragment};
use diesel::QueryResult;
use diesel::sql_types::BigInt;
use crate::common::query::pagination::Paginated;
pub fn handle_table_query<T: QueryFragment<Pg>>(this: &Paginated<T>, mut out: AstPass<Pg>) -> QueryResult<()> {
out.push_sql("SELECT *, COUNT(*) OVER () FROM ");
if this.is_sub_query {
out.push_sql("(");
}
this.query.walk_ast(out.reborrow())?;
if this.is_sub_query {
out.push_sql(")");
}
out.push_sql(" t LIMIT ");
out.push_bind_param::<BigInt, _>(&this.per_page)?;
out.push_sql(" OFFSET ");
let offset = (this.page - 1) * this.per_page;
out.push_bind_param::<BigInt, _>(&offset)?;
Ok(())
}
this code will generate the sql like this:
select
*,
COUNT(*) over ()
from
(
select
"article"."id",
"article"."user_id",
"article"."title",
"article"."author",
"article"."guid",
"article"."created_time",
"article"."updated_time",
"article"."link",
"article"."pub_time",
"article"."sub_source_id",
"article"."cover_image",
"article"."channel_reputation",
"article"."editor_pick"
from
"article"
where
"article"."id" > $1) t
limit $2 offset $3
as you know, this sql have a big problem. when the article table data increase. this sub query will cause sequence scan. Now the article table contains 2000000 rows and each time query takes more than 20s. What I am trying to do is remove the window function and move the limit condition into the sub query, the finally sql will look like this:
select
*,
count_estimate('select * from article')
from
(
select
"article"."id",
"article"."user_id",
"article"."title",
"article"."author",
"article"."guid",
"article"."created_time",
"article"."updated_time",
"article"."link",
"article"."pub_time",
"article"."sub_source_id",
"article"."cover_image",
"article"."channel_reputation",
"article"."editor_pick"
from
"article"
where
"article"."id" > $1 limit $2 offset $3 ) t
this sql only take less than 100ms. This is the rust code I am tweak:
pub fn handle_big_table_query<T: QueryFragment<Pg>>(this: &Paginated<T>, mut out: AstPass<Pg>)-> QueryResult<()>{
out.push_sql("SELECT *, count_estimate('select * from article') FROM ");
if this.is_sub_query {
out.push_sql("(");
}
this.query.walk_ast(out.reborrow())?;
if this.is_sub_query {
out.push_sql(" t LIMIT ");
out.push_bind_param::<BigInt, _>(&this.per_page)?;
out.push_sql(" OFFSET ");
let offset = (this.page - 1) * this.per_page;
out.push_bind_param::<BigInt, _>(&offset)?;
out.push_sql(")");
}
Ok(())
}
to my surprise, this new code generate sql did not return any content. is it possible to see the sql? I check my rust source code but did not figure out where is going wrong. And this is the full pagination code:
use diesel::prelude::*;
use diesel::query_dsl::methods::LoadQuery;
use diesel::query_builder::{QueryFragment, Query, AstPass};
use diesel::pg::Pg;
use diesel::sql_types::BigInt;
use diesel::QueryId;
use serde::{Serialize, Deserialize};
use crate::common::query::page_query_handler::{handle_big_table_query, handle_table_query};
pub trait PaginateForQueryFragment: Sized {
fn paginate(self, page: i64, is_big_table: bool) -> Paginated<Self>;
}
impl<T> PaginateForQueryFragment for T
where T: QueryFragment<Pg>{
fn paginate(self, page: i64, is_big_table: bool) -> Paginated<Self> {
Paginated {
query: self,
per_page: 10,
page,
is_sub_query: true,
is_big_table
}
}
}
#[derive(Debug, Clone, Copy, QueryId, Serialize, Deserialize, Default)]
pub struct Paginated<T> {
pub query: T,
pub page: i64,
pub per_page: i64,
pub is_sub_query: bool,
pub is_big_table: bool
}
impl<T> Paginated<T> {
pub fn per_page(self, per_page: i64) -> Self {
Paginated { per_page, ..self }
}
pub fn load_and_count_pages<U>(self, conn: &PgConnection) -> QueryResult<(Vec<U>, i64)>
where
Self: LoadQuery<PgConnection, (U, i64)>,
{
let per_page = self.per_page;
let results = self.load::<(U, i64)>(conn)?;
let total = results.get(0).map(|x| x.1).unwrap_or(0);
let records = results.into_iter().map(|x| x.0).collect();
let total_pages = (total as f64 / per_page as f64).ceil() as i64;
Ok((records, total_pages))
}
pub fn load_and_count_pages_total<U>(self, conn: &PgConnection) -> QueryResult<(Vec<U>, i64, i64)>
where
Self: LoadQuery<PgConnection, (U, i64)>,
{
let per_page = self.per_page;
let results = self.load::<(U, i64)>(conn)?;
let total = results.get(0).map(|x| x.1).unwrap_or(0);
let records = results.into_iter().map(|x| x.0).collect();
let total_pages = (total as f64 / per_page as f64).ceil() as i64;
Ok((records, total_pages,total))
}
}
impl<T: Query> Query for Paginated<T> {
type SqlType = (T::SqlType, BigInt);
}
impl<T> RunQueryDsl<PgConnection> for Paginated<T> {}
impl<T> QueryFragment<Pg> for Paginated<T>
where
T: QueryFragment<Pg>,
{
fn walk_ast(&self, mut out: AstPass<Pg>) -> QueryResult<()> {
if self.is_big_table {
handle_big_table_query(&self, out);
}else{
handle_table_query(&self,out);
}
Ok(())
}
}
#[derive(Debug, Clone, Copy, QueryId)]
pub struct QuerySourceToQueryFragment<T> {
query_source: T,
}
impl<FC, T> QueryFragment<Pg> for QuerySourceToQueryFragment<T>
where
FC: QueryFragment<Pg>,
T: QuerySource<FromClause=FC>,
{
fn walk_ast(&self, mut out: AstPass<Pg>) -> QueryResult<()> {
self.query_source.from_clause().walk_ast(out.reborrow())?;
Ok(())
}
}
pub trait PaginateForQuerySource: Sized {
fn paginate(self, page: i64, is_big_table: bool) -> Paginated<QuerySourceToQueryFragment<Self>>;
}
impl<T> PaginateForQuerySource for T
where T: QuerySource {
fn paginate(self, page: i64, is_big_table: bool) -> Paginated<QuerySourceToQueryFragment<Self>> {
Paginated {
query: QuerySourceToQueryFragment {query_source: self},
per_page: 10,
page,
is_sub_query: false,
is_big_table
}
}
}

Answer 1

I finally found using this code block generates the SQL command in the wrong format, tweaking the code like this will fix it:
pub fn handle_big_table_query<T: QueryFragment<Pg>>(this: &Paginated<T>, mut out: AstPass<Pg>) -> QueryResult<()> {
out.push_sql("SELECT *, count_estimate('select * from article') FROM ");
if this.is_sub_query {
out.push_sql("(");
}
this.query.walk_ast(out.reborrow())?;
if this.is_sub_query {
out.push_sql(" LIMIT ");
out.push_bind_param::<BigInt, _>(&this.per_page)?;
out.push_sql(" OFFSET ");
let offset = (this.page - 1) * this.per_page;
out.push_bind_param::<BigInt, _>(&offset)?;
out.push_sql(") t");
}
Ok(())
}