SQL insert into using CTE - tsql

I am facing a performance issue due to "Insert into" statement in sql. I am using a CTE to select data from multiple tables and insert into other table. It was working just fine until yesterday. Select takes less than a minute to retrieve the data where as insert into taking forever. Can some one please help me in understanding what i am doing wrong. Any help is highly appreciated. Thanks.
Here is my code:
I am using this query in an SP. I am trying to load 220K records to 1.5M records table.
;with CTE_A
AS
(
SELECT A1, A2,...
FROM dbo.A with (nolock)
WHERE A1 = <some condition>
GROUP BY a.A1,a.A2 , a.A3
), CTE_C as
(
SELECT C1, C2,....
FROM dbo.B with (nolock)
WHERE a.C1 = <some condition>
GROUP BY a.c1,a.C2 , a.C3
)
INSERT INTO [dbo].MainTable
SELECT
A1, A2, A3 , C1, C2, C3
FROM
CTE_A ta with (nolock)
LEFT OUTER JOIN
CTE_C tc with (nolock) ON ta.a1 = tc.a1 and ta.b1 = tc.b1 and ta.c1 = tc.c1
LEFT OUTER JOIN
othertable bs with (nolock) ON usd_bs.c = s.c
AND (A1 BETWEEN bs.a1 AND bs.a1)
AND bs.c1 = 1

try this method (temp table instead cte), perfomance must be much higher for your task
IF OBJECT_ID('Tempdb..#CTE_A') IS NOT NULL
DROP TABLE #CTE_A
IF OBJECT_ID('Tempdb..#CTE_C') IS NOT NULL
DROP TABLE #CTE_C
-------------------------------------------------------------
SELECT A1 ,
A2 ,...
INTO #CTE_A --data set into temp table
FROM dbo.A WITH ( NOLOCK )
WHERE A1 = <some condition>
GROUP BY a.A1 ,
a.A2 ,
a.A3
-------------------------------------------------------------
SELECT C1 ,
C2 ,....
FROM dbo.B WITH ( NOLOCK )
INTO #CTE_C --data set into temp table
WHERE a.C1 = <some condition>
GROUP BY a.c1 ,
a.C2 ,
a.C3
INSERT INTO [dbo].MainTable
SELECT A1 ,
A2 ,
A3 ,
C1 ,
C2 ,
C3
FROM #CTE_A AS ta
LEFT JOIN #CTE_C AS tc ON ta.a1 = tc.a1
AND ta.b1 = tc.b1
AND ta.c1 = tc.c1
LEFT JOIN othertable AS bs ON usd_bs.c = s.c
AND ( A1 BETWEEN bs.a1 AND bs.a1 )
AND bs.c1 = 1

Related

Syntax Error: Lateral Join with Delete (PostgreSQL)

PostgreSQL 11.1
AFAIK, this is correct and should run. It Fails with syntax error on Delete. What am I missing?
Thanks for any help.
ERROR: syntax error at or near "DELETE"
LINE 41: DELETE FROM d
WITH _in (tservice, patient_recid, disease_recid, new_disease_recid) AS (
VALUES ('2021-04-21'::timestamp, '23262'::integer, '34978'::integer, '33364'::integer)
)
UPDATE dx d
SET disease_recid = n.new_disease_recid
FROM _in n,
LATERAL ( WITH RECURSIVE readtoend AS(
SELECT recid, newrecid
FROM patients p1
JOIN _in n ON p1.recid = n.patient_recid
UNION
SELECT c.recid, c.newrecid
FROM patients c
INNER JOIN readtoend s ON s.newrecid = c.recid
),
readtostart AS(
SELECT recid, newrecid
FROM patients p1
JOIN _in n ON p1.recid = n.patient_recid
UNION
SELECT c.recid, c.newrecid
FROM patients c
INNER JOIN readtostart s ON s.recid = c.newrecid
)
SELECT recid FROM readtoend
UNION
SELECT recid FROM readtostart
) j,
LATERAL ( WITH _get_existing_target AS(
SELECT d.*
FROM d
WHERE (d.patient_recid, d.disease_recid) = (j.recid, n.new_disease_recid) AND d.tservice <= n.tservice
),
_get_conflicts AS(
SELECT d.*
FROM d
WHERE (d.patient_recid, d.disease_recid) = (j.recid, n.disease_recid) AND EXISTS ( SELECT 1
FROM _get_existing_target x
WHERE d.patient_recid = x.patient_recid AND d.tservice::date = x.tservice::date)
)
DELETE FROM d
USING _get_conflicts f
WHERE d.recid = f.recid
RETURNING d.*
) del
WHERE (d.patient_recid, d.disease_recid) = (j.recid, n.disease_recid) AND d.tservice::date <= n.tservice::date
AND d.recid NOT IN ( SELECT recid FROM del);
You cannot use DELETE ... RETURNING in the FROM list of a query.

PostgreSQL Join with special condition

Lets assume we have the following table1:
1 2 3
a x m
a y m
b z m
I want to do an inner join on the table
INNER JOIN tabel2 ON table1.2 = table2.2
Somehow like this, but additional a condition that the value of table1.1 not unique. Thus on table1.1 = b no inner join will occure in this example.
What is the best way to achieve this?
Using a an aggregate in a sub query is how I would do it
SELECT *
FROM table1
JOIN table2
ON table1."2" = table2."2"
JOIN (
SELECT "1"
FROM table1
GROUP BY "1"
HAVING COUNT(*) > 1
) AS sub_q
ON sub_q."1" = table1."1";
Another option might be a cte or temporary table to hold the rows you're joining on
WITH _cte AS
(
SELECT "1"
FROM table1
GROUP BY "1"
HAVING COUNT(*) > 1
)
SELECT *
FROM table1
JOIN table2
ON table1."2" = table2."2"
JOIN _cte AS cte
ON cte."1" = table1."1";
temp table:
CREATE TEMPORARY TABLE _tab
(
"1" varchar
);
INSERT INTO _tab
SELECT "1"
FROM table1
GROUP BY "1"
HAVING COUNT(*) > 1;
SELECT *
FROM table1
JOIN table2
ON table1."2" = table2."2"
JOIN _tab AS tab
ON tab."1" = table1."1";

Avoiding Order By in T-SQL

Below sample query is a part of my main query. I found SORT operator in below query is consuming 30% of the cost.
To avoid SORT, there is need of creation of Indexes. Is there any other way to optimize this code.
SELECT TOP 1 CONVERT( DATE, T_Date) AS T_Date
FROM TableA
WHERE ID = r.ID
AND Status = 3
AND TableA_ID >ISNULL((
SELECT TOP 1 TableA_ID
FROM TableA
WHERE ID = r.ID
AND Status <> 3
ORDER BY T_Date DESC
), 0)
ORDER BY T_Date ASC
Looks like you can use not exists rather than the sorts. I think you'll probably get a better performance boost by use a CTE or derived table instead of the a scalar subquery.
select *
from r ... left outer join
(
select ID, min(t_date) as min_date from TableA t1
where status = 3 and not exists (
select 1 from TableA t2
where t2.ID = t1.ID
and t2.status <> 3 and t2.t_date > t1.t_date
)
group by ID
) as md on md.ID = r.ID ...
or
select *
from r ... left outer join
(
select t1.ID, min(t1.t_date) as min_date
from TableA t1 left outer join TableA t2
on t2.ID = t1.ID and t2.status <> 3
where t1.status = 3 and t1.t_date < t2.t_date
group by t1.ID
having count(t2.ID) = 0
) as md on md.ID = r.ID ...
It also appears that you're relying on an identity column but it's not clear what those values mean. I'm basically ignoring it and using the date column instead.
Try this:
SELECT TOP 1 CONVERT( DATE, T_Date) AS T_Date
FROM TableA a1
LEFT JOIN (
SELECT ID, MAX(TableA_ID) AS MaxAID
FROM TableA
WHERE Status <> 3
GROUP BY ID
) a2 ON a2.ID = a1.ID AND a1.TableA_ID > coalesce(a2.MAXAID,0)
WHERE a1.ID = r.ID AND a1.Status = 3
ORDER BY T_Date ASC
The use of TOP 1 in combination with the unexplained r alias concern me. There's almost certainly a MUCH better way to get this data into your results that doesn't involve doing this in a sub query (unless this is for an APPLY operation).

Aggregate similar row

Suppose I've a table like this:
NAME REF1 REF2 DRCT
A (null) Ra D1
A Rb (null) D1
A (null) Rc D2
B Rd (null) D3
B (null) Re D3
I want aggregate this table in something like:
NAME REF1 REF2 DRCT
A Rb Ra D1
A (null) Rc D2
B Rd Re D3
As you can see, i want aggregate each row with same name. I've search through COALESCE and various aggregate functions but I haven't found what i was looking for. Any idea?
Assuming that what I ask in my previous comment is true, (only null or a given value for REF1 and REF2 for each NAME, DRCT pair), this seems to work:
select NAME, M_REF1, M_REF2, DRCT
from (
select A.NAME, coalesce(A.REF1, B.REF1) m_REF1,
coalesce(A.REF2, B.REF2) m_REF2, A.REF1 A_REF1, B.REF1 B_REF1,
A.REF2 A_REF2, B.REF2 B_REF2, A.DRCT
from Table1 A JOIN Table1 B on A.NAME = B.NAME AND A.DRCT = B.DRCT)
WHERE A_REF1 = m_REF1 AND B_REF2 = m_REF2
UNION
select A.NAME, A.REF1, A.REF2, A.DRCT
FROM Table1 A JOIN
(select NAME, DRCT, COUNT(*)
from Table1
group by NAME, DRCT
HAVING COUNT(*) = 1) B ON A.NAME = B.NAME AND A.DRCT = B.DRCT;
The union is used because the rows with only one record are not included in the first SELECT.
But this is somewhat simpler, and works too:
select A.NAME, coalesce(A.REF1, B.REF1) M_REF1, coalesce(A.REF2,B.REF2) M_REF2,A.DRCT
from Table1 A LEFT OUTER JOIN Table1 B ON A.DRCT = B.DRCT AND A.NAME = B.NAME
WHERE NVL2(A.REF1,0,1) = 1 AND NVL2(B.REF1,0,1) =0
AND NVL2(A.REF2,0,1) = 0 AND NVL2(B.REF2,0,1) = 1
UNION
select A.NAME, A.REF1, A.REF2, A.DRCT
FROM Table1 A JOIN
(select NAME, DRCT, COUNT(*)
from Table1
group by NAME, DRCT
HAVING COUNT(*) = 1) B ON A.NAME = B.NAME AND A.DRCT = B.DRCT;

Concatenate fields from a joined query

I need to update a field with concatenated results from a T-SQL query that uses an INNER JOIN and a LEFT JOIN. I was able to do this with the STUFF and FOR XML PATH functions with a simpler query, but my efforts at doing the same process with a more elaborate query have not been successful.
Here is the query that gives me the results I need with the ID field going to end up as the grouping and the Step field will be the one where the concatenated values need to be in the one field per one ID.
SELECT sc.ID, sc.STEP
FROM Table1 As sc
INNER JOIN Table2 As f
ON sc.STEP = f.Step AND sc.STEP_TYPE = f.StepType AND
sc.OldStep = f.OldStep
LEFT JOIN Table3 As l
ON sc.ID = l.ID
WHERE f.Group = l.Group AND sc.CompDate IS NULL
That will give me my results broken down into multiple fields per ID
•ID-----STEP
01 - 101
01 - 102
01 - 103
02 - 107
02 - 113
And what I need is:
•ID-----STEP
01 - 101, 102, 103
02 - 107, 113
Here is what i've tried so far:
;With OA As
( SELECT s.ID, STUFF((
SELECT ', ' + sc.STEP
FROM Table1 As sc
WHERE sc.ID = s.ID
ORDER BY sc.ID
FOR XML PATH('')),1,1,'') As Steps
FROM Table1 As s
INNER JOIN Table2 As f
ON s.STEP = f.Step AND s.STEP_TYPE = f.StepType
AND s.OldStep = f.OldStep
LEFT JOIN Table3 As l
ON s.ID = l.ID
WHERE f.Group = l.Group AND s.CompDate IS NULL
GROUP BY s.ID
)
SELECT * FROM OpenAuditSteps
The problem here is that I am getting a concatenation of all the reocrds, not just the ones grouped on the individual ID's. I've tried various ways of arranging the joins, but nothing has worked so far.
You are very nearly there: You already have your first query working. Assuming the results of that go into #Table1 then
SELECT Distinct
sc1.ID,
STUFF (( Select ',' + sc2.STEP
FROM #Table1 AS SC2
WHERE sc2.ID = sc1.ID
FOR XML PATH('')),1,1,'') AS STEPS
FROM #Table1 AS SC1
ORDER BY sc1.ID
So to combine it into one single query using WITH try this:
;WITH IDSteps AS (
SELECT sc.ID, sc.STEP
FROM Table1 As sc
INNER JOIN Table2 As f
ON sc.STEP = f.Step AND sc.STEP_TYPE = f.StepType AND
sc.OldStep = f.OldStep
LEFT JOIN Table3 As l
ON sc.ID = l.ID
WHERE f.Group = l.Group AND sc.CompDate IS NULL
)
SELECT Distinct
sc1.ID,
STUFF (( Select ',' + sc2.STEP
FROM IDSteps AS SC2
WHERE sc2.ID = sc1.ID
FOR XML PATH('')),1,1,'') AS STEPS
FROM IDSteps AS SC1
ORDER BY sc1.ID;