Postgres COPY command escapes new lines and tabs - postgresql

I have a sql function:
CREATE OR REPLACE FUNCTION dump_func()
RETURNS void AS
$BODY$
DECLARE
r record;
loc varchar(100);
BEGIN
for r in (SELECT f.oid as oid, proname, pg_get_functiondef(f.oid) as src
FROM pg_catalog.pg_proc f
INNER JOIN pg_catalog.pg_namespace n ON (f.pronamespace = n.oid)
WHERE n.nspname = 'public') loop
loc = '/Users/raman/Desktop/functions/' || r.proname || '.sql';
raise notice '% - %', r.oid, loc;
raise notice 'func: %', r.src;
EXECUTE format('COPY (
SELECT pg_get_functiondef(f.oid)
FROM pg_catalog.pg_proc f
INNER JOIN pg_catalog.pg_namespace n ON (f.pronamespace = n.oid)
WHERE n.nspname = ''public'' and f.oid=%s
) TO ''%s'' ', r.oid, loc);
end loop;
end
$BODY$
LANGUAGE plpgsql;
which exports each db functions in postgres into a separate file using the COPY command. Everything works except that in the exported sql files new lines and tabs are escaped, for example:
CREATE OR REPLACE FUNCTION public.is_numeric(text)\n RETURNS boolean\n
LANGUAGE plpgsql\n IMMUTABLE\nAS $function$\nDECLARE x
NUMERIC;\nBEGIN\n x = $1::NUMERIC;\n RETURN TRUE;\nEXCEPTION
WHEN others THEN\n RETURN FALSE;\nEND;\n$function$\n
As you can see new lines and tabs are escaped, how to fix that?

Actually, this can be done. Hacky though it is you simply use regexp_split_to_table and split on the \n character. I was just trying to work out the same problem myself. Here's the code I wound up with:
DO $$
DECLARE
v_row record;
v_proname text;
v_sql text;
BEGIN
FOR v_row IN (
SELECT f.proname
FROM pg_catalog.pg_proc AS f
INNER JOIN pg_catalog.pg_namespace AS n ON (f.pronamespace = n.oid)
WHERE n.nspname = 'public' AND f.proisagg = false
) LOOP
SELECT v_row.proname::text INTO v_proname;
EXECUTE FORMAT('COPY (
SELECT regexp_split_to_table(regexp_replace(pg_get_functiondef(oid), ''\t'', '' '', ''g''), ''\n'')
FROM pg_catalog.pg_proc WHERE proname = ''%s'')
TO ''C:\pg\%s.sql'';'
, v_proname, v_proname);
END LOOP;
END $$

Andrew's answer is correct, helped me format it as well.
I came up with a similar function but to print all user defined functions in PSQL. Hope it helps to anyone who needs it:
CREATE OR REPLACE FUNCTION printFunctions(stage VARCHAR, ischema VARCHAR, file_path VARCHAR)
RETURNS VARCHAR AS
$$
DECLARE
func VARCHAR;
file_location TEXT;
BEGIN
FOR func IN SELECT procname FROM listfunctions()
LOOP
SELECT (file_path || '/' || func || '_' || stage || '.sql') INTO file_location;
IF (SELECT regexp_matches(func, '(pldb|plpg)')) IS NULL
THEN
EXECUTE 'COPY ' ||
'(SELECT regexp_split_to_table(regexp_replace(pg_get_functiondef(oid), ''\t'', '' '', ''g''),''\n'') from pg_proc where proname=' ||
'''' || func || '''' ||
' and pronamespace=(select oid from pg_namespace where nspname=' ||
'''' || ischema || ''''
')) TO ' || '''' || file_location || '''';
END IF;
END LOOP;
RETURN stage;
END
$$
LANGUAGE 'plpgsql';
You can call it with:
SELECT * FROM printFunctions('dev', 'public', '/tmp');

Related

Could you explain part of this function that begins with " if not exist...."?

CREATE OR REPLACE FUNCTION event_partition()
RETURNS trigger AS
$BODY$
DECLARE
_tbl text := to_char(NEW.the_date, 'YYYY-MM-DD');
BEGIN
***IF NOT EXISTS (
SELECT 1
FROM pg_catalog.pg_class c
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' -- your schema
AND c.relname = _tbl
AND c.relkind = 'r') THEN***
EXECUTE format('CREATE TABLE %I (CHECK (the_date >= %L AND
the_date < %L)) INHERITS (events)'
, _tbl
, to_char(NEW.the_date, 'YYYY-MM-DD')
, to_char(NEW.the_date + 1, 'YYYY-MM-DD')
);
END IF;
EXECUTE 'INSERT INTO ' || quote_ident(_tbl) || ' VALUES ($1.*)'
USING NEW;
RETURN NULL;
END
$BODY$
LANGUAGE plpgsql VOLATILE
COST 100;
ALTER FUNCTION event_partition() SET search_path=public;
It's querying the database's internal table list (pg_class) to see if the table _tbl already exists in the public schema, and creating it if it doesn't.
You should be able to remove the check, and just use a CREATE TABLE IF NOT EXISTS... statement instead.

Postgresql function return multiple select statements

Can any one of you tell me how to approach this:
CREATE OR REPLACE FUNCTION name()
RETURNS ????? AS
$func$
BEGIN
SELECT * FROM tbl_a a;
SELECT * FROM tbl_b b;
END
$func$ LANGUAGE plpgsql;
Both tables have different structures.
You can use cursors but I can hardly imagine why you need such a function.
CREATE OR REPLACE FUNCTION my_multiselect(refcursor, refcursor) RETURNS VOID AS
$func$
BEGIN
OPEN $1 FOR SELECT * FROM information_schema.routines;
OPEN $2 FOR SELECT * FROM information_schema.sequences;
END
$func$ LANGUAGE plpgsql;
BEGIN;
SELECT my_multiselect('first_cursor_to_routines', 'second_cursor_to_sequences');
FETCH ALL IN first_cursor_to_routines;
FETCH ALL IN second_cursor_to_sequences;
COMMIT;
I'm not really sure what you're doing with this, but it sounds like you just want to return a union of these distinct result sets. You can do this with a dynamic query. I'm using Postgres 9.4.
CREATE OR REPLACE FUNCTION make_query(IN p_tables text[])
RETURNS void AS
$BODY$
DECLARE
v_qry text;
v_cols text;
v_types text;
v_as text;
BEGIN
EXECUTE format('
WITH sub AS (
SELECT
table_name,
column_name,
data_type
FROM
information_schema.columns
WHERE
table_name = ANY(%L)
ORDER BY
table_name,
ordinal_position)
,sub2 AS(
SELECT
DISTINCT ON (column_name, data_type)
column_name || '' '' || data_type AS def
FROM
sub
)
SELECT
string_agg(def, '','')
FROM
sub2;
',
p_tables
) INTO v_types;
v_qry := '
CREATE OR REPLACE FUNCTION name()
RETURNS TABLE(' || v_types || ') AS
$func$';
FOR i IN 1..array_upper(p_tables, 1)
LOOP
v_as := 'tbl' || i;
EXECUTE format('
WITH sub AS (
SELECT
table_name,
column_name,
data_type
FROM
information_schema.columns
WHERE
table_name = ANY(%L)
ORDER BY
table_name,
ordinal_position)
,sub2 AS(
SELECT
DISTINCT ON (column_name, data_type)
CASE WHEN table_name = ''%I''
THEN %L || ''.'' || column_name
ELSE ''NULL::'' || data_type
END AS cols
FROM
sub
)
SELECT
string_agg(cols, '','')
FROM
sub2;
',
p_tables,
p_tables[i],
v_as
) INTO v_cols;
IF i > 1 THEN
v_qry := v_qry || '
UNION ALL';
END IF;
v_qry := v_qry || '
SELECT ' || v_cols || ' FROM ' || p_tables[i] || ' AS ' || v_as;
IF i = array_upper(p_tables, 1) THEN
v_qry := v_qry || ';';
END IF;
END LOOP;
v_qry := v_qry || '
$func$ LANGUAGE sql;
';
EXECUTE v_qry;
END;
$BODY$
LANGUAGE plpgsql VOLATILE;
Sorry it looks ugly here, but this formatting helps the final product look nicer. If you're shy about executing a dynamic query like this off the bat, just replace EXECUTE v_qry; with RAISE INFO 'v_qry: %', v_qry; and it will simply print the dynamic query out in a message without executing it, so you can review what it will do once executed.
Then execute make_query() with a list of tables you want to display like this:
SELECT make_query(ARRAY['tbl_a', 'tbl_b']);
The result is that you will now have a function called name() which you can call in order to see the results of both tables at the same time, with all the union details already sorted out:
SELECT * FROM name();

Postgres Function returning no results but when the same query returns results outside function

I am running a simple postgres function to return the count of rows. I am able to run the same query outside function with the output of raise option , but the function doesn't return any rows. I have tried different ways to produce results but unable to. Please find my function below,
CREATE OR REPLACE FUNCTION my_schema.usp_spellcheck3(SearchORItems_WithPipe varchar, site varchar, lan varchar, rows_display integer)
RETURNS TABLE (docmnt int) AS $BODY$
DECLARE
arrSearchTerms text[];
NewTerm varchar;
i integer;
AltSearch_withComma varchar;
AltSearch_withPipe varchar;
strDidYouMean varchar;
dpDidYouMean double precision;
txtDidYouMean Text;
SearchORItems_withComma varchar;
SearchORItems varchar;
SearchORItem varchar;
ws varchar;
arrSearchORItems_withComma varchar[];
BEGIN
strDidYouMean = 'DidYouMeanRow';
dpDidYouMean = 0.0;
txtDidYouMean = 'DidYouMeanRow';
ws = '''' || '%' || site || '%' || '''' ;
RAISE NOTICE '%', ws;
SearchORItems = REPLACE(SearchORItems_WithPipe, '|', ',');
SELECT regexp_split_to_array(SearchORItems, ',') INTO arrSearchORItems_withComma;
RAISE NOTICE '%', SearchORItems;
FOR i IN 1 .. coalesce(array_upper(arrSearchORItems_withComma, 1), 1) LOOP
IF (i = 1) THEN
SearchORItems_withComma = '''' || arrSearchORItems_withComma[i] || '''';
ELSE
SearchORItems_withComma = SearchORItems_withComma||','||'''' || arrSearchORItems_withComma[i] || '''';
END IF;
END LOOP;
RAISE NOTICE '%',SearchORItems_withComma;
SELECT COUNT(*) INTO res_count
FROM (
SELECT 1 FROM my_schema.features f , my_schema.documents d
WHERE term IN (SearchORItems_withComma)
AND d.docid = f.docid
AND d.url LIKE ws
GROUP BY f.docid, d.url) t;
RAISE NOTICE '%', res_count;
SearchORItem = 'SELECT COUNT(*) INTO res_count
FROM (SELECT 1 FROM my_schema.features f , my_schema.documents d
WHERE term IN ('||SearchORItems_withComma||')
AND d.docid = f.docid AND d.url LIKE ' || ws ||'
GROUP BY f.docid, d.url) t';
RAISE NOTICE '%',SearchORItem;
END;
$BODY$ LANGUAGE SQL VOLATILE;
this is my query output :
NOTICE: '%uni%'
NOTICE: daniel,data
NOTICE: 'daniel','data'
NOTICE: 0
NOTICE: select count(*) into res_count
from ( select 1 from my_schema.features f , my_schema.documents d
where term in ('daniel','data')
and d.docid=f.docid and d.url like '%uni%'
group by f.docid,d.url)t
Total query runtime: 16 ms.
0 rows retrieved.
I dont know where I'm going wrong, any help would be appreciated .. Thanks..
The simple reason that nothing is returned is that you have no RETURN statements in your code. When a function RETURNS TABLE you need to explicitly put one or more RETURN NEXT or RETURN QUERY statements in the body of your code, with a final RETURN statement to indicate the end of the function. See the documentation here: http://www.postgresql.org/docs/9.3/static/plpgsql-control-structures.html#PLPGSQL-STATEMENTS-RETURNING. What exactly you want to return is not clear but likely candidates are res_count and d.docid.
Other than that, your code could use a real clean-up reducing clutter like:
ws := '''%' || site || '%''' ;
instead of:
ws = '''' || '%' || site || '%' || '''' ;
and:
SELECT COUNT(*) INTO res_count
FROM my_schema.features f,
JOIN my_schema.documents d ON d.docid = f.docid
WHERE term IN (SearchORItems_withComma)
AND d.url LIKE ws
GROUP BY f.docid, d.url;
instead of:
SELECT COUNT(*) INTO res_count
FROM (
SELECT 1 FROM my_schema.features f , my_schema.documents d
WHERE term IN (SearchORItems_withComma)
AND d.docid = f.docid
AND d.url LIKE ws
GROUP BY f.docid, d.url) t;
And you should use the assignment operator (:=) instead of the equality operator in any plpgsql statement that is not a SQL statement.

List all tables in Postgres that contain a boolean type column with NULL values?

Is there a way to list all tables within information schema which contain a boolean type column with NULL values? If I have a names of the tables, I can later on use query:
SELECT * FROM table_name WHERE column_name IS NULL
Of course, if there is a way to list all rows from all the tables with single query, that would be even faster.
Doing this step by step would be:
SELECT * FROM table1 WHERE column_name IS NULL
SELECT * FROM table2 WHERE column_name IS NULL
SELECT * FROM table3 WHERE column_name IS NULL
...
Tables are already populated, for new database, such columns should have NOT NULL constraint.
Try:
SELECT table_catalog, table_schema, table_name, column_name, ordinal_position
FROM information_schema.columns
WHERE table_schema <> 'pg_catalog' AND data_type = 'boolean';
and:
DO $$
DECLARE
r record;
s record;
BEGIN
FOR r IN SELECT table_catalog, table_schema, table_name, column_name, ordinal_position
FROM information_schema.columns
WHERE table_schema <> 'pg_catalog' AND data_type = 'boolean'
LOOP
FOR s IN EXECUTE 'SELECT ' || quote_literal(r.table_schema) || ', ' || quote_literal(r.table_name) || ', ' || quote_literal(r.column_name) || ' WHERE EXISTS (SELECT 1 FROM ' || quote_ident(r.table_schema) || '.' || quote_ident(r.table_name) || ' WHERE ' || quote_ident(r.column_name) || ' IS NULL);'
LOOP
RAISE NOTICE '** % % %', quote_ident(r.table_schema), quote_ident(r.table_name), quote_ident(r.column_name);
END LOOP;
END LOOP;
END
$$;
Osvaldo
If you are running postgresql 9.0+, you can use an anonymous plpgsql block to execute some dynamic SQL.
DO $$
DECLARE
rec RECORD;
v_result INTEGER;
BEGIN
FOR rec IN
SELECT 'select 1 from ' || quote_ident(n.nspname) ||'.'|| quote_ident(c.relname) ||
' where ' || quote_ident(a.attname) || ' IS NULL LIMIT 1' as qry_to_run,
n.nspname||'.'||c.relname as tbl,
a.attname as col
FROM pg_class as c
INNER JOIN pg_attribute as a ON (a.attrelid = c.oid)
INNER JOIN pg_type as t ON (t.oid = a.atttypid)
LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE a.attnum >= 1
AND c.relkind = 'r'
AND pg_catalog.format_type(a.atttypid, a.atttypmod) = 'boolean'
AND n.nspname NOT IN ('pg_catalog','information_schema')
AND a.attnotnull IS NOT TRUE
LOOP
EXECUTE rec.qry_to_run INTO v_result;
IF v_result = 1 THEN
RAISE NOTICE 'Table % has NULLs in the BOOLEAN field %', rec.tbl,rec.col;
v_result := 0;
END IF;
END LOOP;
END;
$$;
This advances #bma's excellent answer to make it shorter, faster and a bit smarter:
DO $$
DECLARE
rec record;
_found boolean;
BEGIN
FOR rec IN
SELECT format('SELECT TRUE FROM %s WHERE %I IS NULL LIMIT 1'
, c.oid::regclass, a.attname) AS qry_to_run
,c.oid::regclass AS tbl
,a.attname AS col
FROM pg_namespace n
JOIN pg_class c ON c.relnamespace = n.oid
JOIN pg_attribute a ON a.attrelid = c.oid
WHERE n.nspname <> 'information_schema'
AND n.nspname NOT LIKE 'pg_%' -- exclude system, temp, toast tbls
AND c.relkind = 'r'
AND a.attnum > 0
AND a.atttypid = 'boolean'::regtype
AND a.attnotnull = FALSE
AND a.attisdropped = FALSE
LOOP
EXECUTE rec.qry_to_run INTO _found;
IF _found THEN
RAISE NOTICE 'Table % has NULLs in the BOOLEAN field %'
, rec.tbl,rec.col;
END IF;
END LOOP;
END
$$;

Running PostgreSQL stored procedures in SQL console

I have following stored procedure
CREATE FUNCTION runMortalityModel(a_user_id integer) RETURNS integer AS $$
DECLARE
t1 RECORD;
t2 RECORD;
numberOfDeaths integer;
BEGIN
SELECT person.id personId, person.age, condprobmin, condprobmax, random() experiment
INTO t1
FROM person, mortality_cond_prob
WHERE (user_id = a_user_id) and
(person.age = mortality_cond_prob.age);
SELECT personId
INTO t2
FROM t1
WHERE (tmp.condprobmin <= experiment) and (experiment <= tmp.condprobmax);
SELECT COUNT(*)
INTO numberOfDeaths
FROM t2;
RAISE 'numberOfDeaths=%', numberOfDeaths;
EXECUTE
'DELETE '
|| 'FROM person '
|| 'WHERE person.id IN '
|| t2;
RETURN numberOfDeaths;
END
$$ LANGUAGE plpgsql;
When I try to run this stored procedure using
SELECT runMortalityModel(1);
I get the error Relation »t1« doesn't exist.
How can I fix it?
Update 1: Changed the stored procedure declaration to
CREATE OR REPLACE FUNCTION runMortalityModel(a_user_id integer) RETURNS integer AS $$
DECLARE
t1 RECORD;
t2 RECORD;
numberOfDeaths integer;
BEGIN
EXECUTE 'SELECT person.id personId, person.age, condprobmin, condprobmax, random() experiment '
|| 'FROM person, mortality_cond_prob '
|| 'WHERE (user_id = ' || a_user_id || ') and '
|| '(person.age = mortality_cond_prob.age)'
INTO t1;
EXECUTE 'SELECT personId '
|| 'FROM ' || t1
|| ' WHERE (tmp.condprobmin <= experiment) and (experiment <= tmp.condprobmax)'
INTO t2;
EXECUTE 'SELECT COUNT(*) '
|| 'FROM ' || t2
INTO numberOfDeaths;
RAISE 'numberOfDeaths=%', numberOfDeaths;
EXECUTE
'DELETE '
|| 'FROM person '
|| 'WHERE person.id IN '
|| t2;
RETURN numberOfDeaths;
END
$$ LANGUAGE plpgsql;
I see several issues with original code:
You're trying to use RECORD variable as a relation, you should do ... FROM (SELECT t1.*) s instead;
I see no point to select 1 record, then do a query on that record and then perform count(*), you will always have either 0 or 1 as a result.
You second version looks much better, go for it.
This one seems to work. If you have better ideas, please tell them.
CREATE FUNCTION runMortalityModel(a_user_id integer) RETURNS integer AS $$
DECLARE
t1 RECORD;
curRecord RECORD;
numberOfDeaths integer;
BEGIN
numberOfDeaths := 0;
FOR curRecord IN
SELECT person.id personId, condprobmin, condprobmax, random() experiment
FROM person, mortality_cond_prob
WHERE (user_id = a_user_id) and
(person.age = mortality_cond_prob.age)
LOOP
IF (curRecord.condprobmin <= curRecord.experiment) AND (curRecord.experiment <= curRecord.condprobmax) THEN
EXECUTE
'DELETE '
|| 'FROM person '
|| 'WHERE person.id = ' || curRecord.personId;
numberOfDeaths := numberOfDeaths + 1;
END IF;
END LOOP;
RETURN numberOfDeaths;
END
$$ LANGUAGE plpgsql;