Union multiple tables - postgresql

I tried to follow this answer
Dynamic UNION ALL query in Postgres
but I am getting
ERROR: syntax error at or near "record"
DECLARE
rec record;
strSQL text;
BEGIN
FOR
rec in
select table_name
from
information_schema.tables
where
table_name like 'history%'
loop
strSQL : = strSQL || 'Select * from' || rec.table_schema ||'.'|| rec.table_name || ' UNION ';
end loop;
-- remove last ' UNION ' from strSQL
--strSQL := 'Select row_number() over(order by rowid ) as row_num from (' || strSQL || ')';
execute strSQL;
anyone have any ideas?
background:
history table is moved every night to its own table with the date appended.
so history04242018 for each table name, any better way to get the data for multiple days?
edit: tables will always have the same amount of columns so the union should be fine
edit2: I only have read access.
update
with the suggestion of using an anonymous code block I am now using the following:
DO
$$
declare
strSQL text;
begin
select
string_agg(format('select * from %I.%I', table_schema, table_name), E' union\n')
into strSQL
from information_schema.tables
where table_name like 'history%';
execute strSQL ;
end $$;
however I now get the error
Describe Error: Failed to retrieve EXPLAIN plan(s): ERROR: syntax
error at or near "DO" Position: 58
0 record(s) affected

declare, for, loop, execute are the parts of the plpgsql, not plain sql (declare could be used in the plain sql but in different meaning). So you should to wrap your code into anonymous block or into function if you want to return some data from it:
create function get_history(p_day int)
returns table (<structure of history tables here>)
-- or
-- returns setof <history table name>
language plpgsql
as $$
declare
strSQL text;
begin
select
string_agg(format('select * from %I.%I', table_schema, table_name), E' union\n')
into strSQL
from information_schema.tables
where table_name like to_char(p_day, '"history__"FM09%');
return query execute strSQL;
end $$;
Also look at the Table partitioning (choose your PostgreSQL version at the top of article).
Update
However there are several ways to return query data from anonymous plpgsql block without changing DB schema: cursors and prepared statements.
IMO second one is simpler a bit, so:
do $$
declare
strSQL text;
begin
select
string_agg(format('select * from %I.%I', table_schema, table_name), E' union\n')
into strSQL
from information_schema.tables
where table_name like to_char(p_day, '"history__"FM09%');
-- Prepend "prepare", change the "foo" name as you wish
strSQL := 'prepare foo as ' || strSQL;
execute strSQL;
end $$;
-- Usage
execute foo;
-- And deallocate prepared statement when it does not need anymore:
deallocate foo;
Simple working example

Related

Postgresql error: cannot open EXECUTE query as cursor

I have written a function to read certain columns from a table below using a dynamic query:
CREATE OR REPLACE FUNCTION select_cols ()
RETURNS SETOF mytable_name
LANGUAGE plpgsql
AS $$
DECLARE
list_of_columns text;
BEGIN
SELECT
string_agg(trim(cols::text, '()'), ', ') INTO list_of_columns
FROM (
SELECT
'mytable_name.' || column_name
FROM
information_schema.columns
WHERE
table_name = 'mytable_name'
AND column_name LIKE 'rm%_b'
OR column_name LIKE 'rm%_s') AS cols;
RETURN query EXECUTE concat(format('select %s from mytable_name', list_of_columns), ' RETURNING *');
END
$$;
Though when I run
select * from select_cols();
it gives me an error : "cannot open EXECUTE query as cursor".
I appreciate if someone can help with this issue
You are not returning a set, but you aggreagte the result set for only one table. So, for only one table you can use:
CREATE OR REPLACE FUNCTION select_colsx ()
RETURNS text
LANGUAGE plpgsql
AS $$
DECLARE
list_of_columns text;
BEGIN
SELECT
'select '||string_agg(trim(cols::text, '()'), ', ') || ' from pg_class RETURNING *'
INTO list_of_columns
FROM (
SELECT
'pg_class.' || column_name
FROM
information_schema.columns
WHERE
table_name = 'pg_class'
AND column_name LIKE 'oid'
OR column_name LIKE 'relacl') AS cols;
RETURN list_of_columns ;
END
$$;
select select_colsx();
DB Fiddle Example
RETURN QUERY EXECUTE was introduced in PostgreSQL 8.4. Upgrade to a less ancient version.

Dynamic query splicing error,a fuction use falie

I want to use the cursor to get the count(data) in all tables under the schema I need.
But I am unfamiliar with SQL, and I still cannot pass the following code:
CREATE OR REPLACE FUNCTION check_tool.get_nae(v_dbName character varying)
RETURNS numeric
LANGUAGE edbspl
SECURITY DEFINER
AS $function$
numInsert numeric;
numCal numeric;
v_result numeric;
query TEXT DEFAULT '';
cursor c_pj is
select t.table_schema::text as tableSchema,
t.table_name::text as tableName
from information_schema.tables t
where t.table_catalog = v_dbName
and t.table_type = 'BASE TABLE'
and t.table_schema in (select schema_name from check_tool.img_schema where dbName = v_dbName);
BEGIN
v_result := -1;
numInsert := 0;
for r_pj in c_pj loop
query := 'select count(*) from '||tableSchema||'.'||tableName||';'; -- select count(*) from "item"."project";
execute query into numCal;
insert into check_tool.img_result(schema_name,table_name,num) values (r_pj.tableSchema,r_pj.tableName,numCal);
numInsert := numInsert + 1;
if numInsert > 1000 then
numInsert := 0;
commit;
end if;
end loop;
commit;
v_result := 0;
RETURN v_result;
EXCEPTION
WHEN others THEN
RETURN v_result;
END get_nae$function$
;
/
I also tried concat() and quote_ident(), but the result is not ideal.
Dynamic SQL should be constructed using format() to better handle identifiers. In Postgres you can't commit inside a function, only in a procedure. Committing in a loop rarely improves the performance to begin with, so I would just skip that. I also wouldn't hide the real error (by just returning -1 or 0) but simply let any exception reach the caller of the procedure. And language edbspl is nothing I know, but in PL/pgSQL I would write it like this:
CREATE OR REPLACE PROCEDURE check_tool.get_nae(v_dbName character varying)
LANGUAGE plpgsql
AS
$body$
declare --<< required in PL/pgSQL to declare variables
numcal numeric;
query TEXT DEFAULT '';
l_rec record;
BEGIN
for l_rec in select t.table_schema::text as tableschema, t.table_name::text as tablename
from information_schema.tables t
where t.table_catalog = v_dbName
and t.table_type = 'BASE TABLE'
and t.table_schema in (select schema_name from check_tool.img_schema where dbName = v_dbName)
loop
query := format('select count(*) from %I.%I', l_rec.tableschema, l_rec.tablename);
execute query into numcal;
insert into check_tool.img_result(schema_name,table_name,num) values (l_rec.tableschema, l_rec.tablename, numcal);
end loop;
commit;
END;
$body$
;
Note that the condition t.table_catalog = v_dbName is actually useless, because you can't query tables that are not in the current database anyway.
Note that you don't really need a stored procedure to do this. You can use query_to_xml() to do this in a single SQL query by adjusting this answer
insert into check_tool.img_result(schema_name,table_name,num)
select table_schema,
table_name,
(xpath('/row/cnt/text()', xml_count))[1]::text::int as row_count
from (
select table_name, table_schema,
query_to_xml(format('select count(*) as cnt from %I.%I', table_schema, table_name), false, true, '') as xml_count
from information_schema.tables
where t.table_type = 'BASE TABLE'
and t.table_schema in (select schema_name from check_tool.img_schema)
) t;

How to execute query statements generated in pgAdmin4?

I have the following query which generates a list of ALTER TABLE statements in the 'Data Output' field in pgAdmin4. I can copy & paste them back into the query pane and execute them there.
select 'ALTER TABLE ' || table_name || ' OWNER TO myuser;' from information_schema.tables where table_schema = 'public';
How can I execute the generated statements without having to copy & paste them?
You can use the function for that.
CREATE OR REPLACE FUNCTION change_permission_table()
RETURNS VOID AS $$
DECLARE
rec text;
BEGIN
FOR rec IN SELECT 'ALTER TABLE ' || table_name || ' OWNER TO maz;' FROM information_schema.tables WHERE table_schema = 'public'
LOOP
EXECUTE rec;
END LOOP;
END;
$$ LANGUAGE plpgsql;
-- Run the function to change the permission
SELECT change_permission_table()

Getting Results of Dynamic Query As A Table?

My company is going to start generating documents using data from our database and I am designing the function that will spit out the document text. These documents will need to contain data taken from multiple tables, with hundreds of columns and invariably some records will be missing data.
I am trying to make a function that will take null fields and replace them with a little error message that makes it clear to the end user that a piece of data is missing. Since the end user is totally unfamiliar with the backend, I want these messages to reference something intelligible to them.
My solution is pretty simple yet I for the life of me can't get it to work. The record identifier, table name are set as parameters in the function. The function then loops through names for each of the columns in the specified table, building a query that contains a bunch of case statements. Once the loop is complete, the identifier is appended and then the query is executed, returning the results to the calling function.
Despite reading around quite a bit, the best I can is a single column/row containing all the results - not useful to me at all, because I need to be able to easily reference specific pieces of data in the parent query. I am a beginner with Postgres and the documentation is too complex for me to understand, any help would be appreciated.
-- Function: data_handler(text, text)
-- DROP FUNCTION data_handler(text, text);
CREATE OR REPLACE FUNCTION data_handler(target_uri text, target_table TEXT)
RETURNS SETOF record AS
$BODY$
DECLARE
c text;
strSQL text;
site_only text;
result record;
BEGIN
--We need the schema for strSQL but the loop needs just the table name.
site_only = split_part(target_table, '.', 2);
FOR c IN
SELECT column_name
FROM information_schema.columns
WHERE table_name = site_only
LOOP
strSQL = concat(strSQL, chr(10), '(SELECT CASE WHEN ', c::text, '::text IS NULL THEN concat(', chr(39), '<Error:', chr(39), ', (SELECT lkp_value FROM alb_cr.lkp_field_values WHERE column_name = ', chr(39), c::text, chr(39), ')::text, ', chr(39), ' value not found>', chr(39), ')::text ELSE ',
c::text, '::text END AS ', c::text, '_convert) AS ', c::text, ',');
END LOOP;
strSQL = LEFT(strSQL, character_length(strSQL) - 1);
strSQL = concat('SELECT ', strSQL, ' FROM ', target_table, ' WHERE nm_site_id = ', chr(39), target_uri, chr(39));
RETURN QUERY EXECUTE strSQL;
RAISE NOTICE 'strSQL: %', strSQL;
--RETURN strSQL;
--RETURN QUERY EXECUTE format('SELECT ' || strSQL || 'FROM %s WHERE nm_site_id = $1', pg_typeof(target_table)) USING target_uri;
END
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION data_handler(text, text)
OWNER TO inti;
You could create views for that as well, in the following example on a schema nullsbegone:
-- create the schema to hold the views
create schema if not exists nullsbegone;
-- create a function to create the views (any and all that you might need)
create or replace function nullsbegone.f_make_view_of(p_tablename text) returns void as $f$
begin
execute ($$
create or replace view nullsbegone.$$||(select relname from pg_class where oid = $1::regclass)||$$
returns void as
select $$||array_to_string(array(
select case when not attnotnull then 'COALESCE('||quote_ident(attname)||$$::text, (SELECT '<Error:'''||lkp_value||''' value not found>' FROM alb_cr.lkp_field_values
WHERE column_name = $$||quote_literal(attname)||$$)) AS $$
else '' end || quote_ident(attname)
from pg_attribute
where attrelid = $1::regclass and attnum > 0 order by attnum
), E', \n')||$$
from $$||$1);
end;$f$ language plpgsql;
-- create the view based on a given table
select nullsbegone.f_make_view_of('yourschema.yourtable');
-- select from your view as if you were selecting from the actual table
select * from nullsbegone.yourtable
where nm_site_id = 'yoursite';

Loop on tables with PL/pgSQL in Postgres 9.0+

I want to loop through all my tables to count rows in each of them. The following query gets me an error:
DO $$
DECLARE
tables CURSOR FOR
SELECT tablename FROM pg_tables
WHERE tablename NOT LIKE 'pg_%'
ORDER BY tablename;
tablename varchar(100);
nbRow int;
BEGIN
FOR tablename IN tables LOOP
EXECUTE 'SELECT count(*) FROM ' || tablename INTO nbRow;
-- Do something with nbRow
END LOOP;
END$$;
Errors:
ERROR: syntax error at or near ")"
LINE 1: SELECT count(*) FROM (sql_features)
^
QUERY: SELECT count(*) FROM (sql_features)
CONTEXT: PL/pgSQL function inline_code_block line 8 at EXECUTE statement
sql_features is a table's name in my DB. I already tried to use quote_ident() but to no avail.
I can't remember the last time I actually needed to use an explicit cursor for looping in PL/pgSQL.
Use the implicit cursor of a FOR loop, that's much cleaner:
DO
$$
DECLARE
rec record;
nbrow bigint;
BEGIN
FOR rec IN
SELECT *
FROM pg_tables
WHERE tablename NOT LIKE 'pg\_%'
ORDER BY tablename
LOOP
EXECUTE 'SELECT count(*) FROM '
|| quote_ident(rec.schemaname) || '.'
|| quote_ident(rec.tablename)
INTO nbrow;
-- Do something with nbrow
END LOOP;
END
$$;
You need to include the schema name to make this work for all schemas (including those not in your search_path).
Also, you actually need to use quote_ident() or format() with %I or a regclass variable to safeguard against SQL injection. A table name can be almost anything inside double quotes. See:
Table name as a PostgreSQL function parameter
Minor detail: escape the underscore (_) in the LIKE pattern to make it a literal underscore: tablename NOT LIKE 'pg\_%'
How I might do it:
DO
$$
DECLARE
tbl regclass;
nbrow bigint;
BEGIN
FOR tbl IN
SELECT c.oid
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE c.relkind = 'r'
AND n.nspname NOT LIKE 'pg\_%' -- system schema(s)
AND n.nspname <> 'information_schema' -- information schema
ORDER BY n.nspname, c.relname
LOOP
EXECUTE 'SELECT count(*) FROM ' || tbl INTO nbrow;
-- raise notice '%: % rows', tbl, nbrow;
END LOOP;
END
$$;
Query pg_catalog.pg_class instead of tablename, it provides the OID of the table.
The object identifier type regclass is handy to simplify. n particular, table names are double-quoted and schema-qualified where necessary automatically (also prevents SQL injection).
This query also excludes temporary tables (temp schema is named pg_temp% internally).
To only include tables from a given schema:
AND n.nspname = 'public' -- schema name here, case-sensitive
The cursor returns a record, not a scalar value, so "tablename" is not a string variable.
The concatenation turns the record into a string that looks like this (sql_features). If you had selected e.g. the schemaname with the tablename, the text representation of the record would have been (public,sql_features).
So you need to access the column inside the record to create your SQL statement:
DO $$
DECLARE
tables CURSOR FOR
SELECT tablename
FROM pg_tables
WHERE tablename NOT LIKE 'pg_%'
ORDER BY tablename;
nbRow int;
BEGIN
FOR table_record IN tables LOOP
EXECUTE 'SELECT count(*) FROM ' || table_record.tablename INTO nbRow;
-- Do something with nbRow
END LOOP;
END$$;
You might want to use WHERE schemaname = 'public' instead of not like 'pg_%' to exclude the Postgres system tables.